diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index d22bc865..4870d614 100755 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -3519,7 +3519,7 @@ class AppBase: if self.action["app_name"].lower() == "shuffle tools": timeout = 55 - timeout = 30 + #timeout = 30 try: executor = concurrent.futures.ThreadPoolExecutor() diff --git a/backend/go-app/main.go b/backend/go-app/main.go index ad8c63b4..1dac07a9 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -86,59 +86,6 @@ var runningEnvironment = "onprem" var syncUrl = "https://shuffler.io" var dbclient *datastore.Client - -type Userapi struct { - Username string `datastore:"username"` - ApiKey string `datastore:"apikey"` -} - -type ExecutionInfo struct { - TotalApiUsage int64 `json:"total_api_usage" datastore:"total_api_usage"` - TotalWorkflowExecutions int64 `json:"total_workflow_executions" datastore:"total_workflow_executions"` - TotalAppExecutions int64 `json:"total_app_executions" datastore:"total_app_executions"` - TotalCloudExecutions int64 `json:"total_cloud_executions" datastore:"total_cloud_executions"` - TotalOnpremExecutions int64 `json:"total_onprem_executions" datastore:"total_onprem_executions"` - DailyApiUsage int64 `json:"daily_api_usage" datastore:"daily_api_usage"` - DailyWorkflowExecutions int64 `json:"daily_workflow_executions" datastore:"daily_workflow_executions"` - DailyAppExecutions int64 `json:"daily_app_executions" datastore:"daily_app_executions"` - DailyCloudExecutions int64 `json:"daily_cloud_executions" datastore:"daily_cloud_executions"` - DailyOnpremExecutions int64 `json:"daily_onprem_executions" datastore:"daily_onprem_executions"` -} - -// "Execution by status" -// Execution history -//type GlobalStatistics struct { -// BackendExecutions int64 `json:"backend_executions" datastore:"backend_executions"` -// WorkflowCount int64 `json:"workflow_count" datastore:"workflow_count"` -// ExecutionCount int64 `json:"execution_count" datastore:"execution_count"` -// ExecutionSuccessCount int64 `json:"execution_success_count" datastore:"execution_success_count"` -// ExecutionAbortCount int64 `json:"execution_abort_count" datastore:"execution_abort_count"` -// ExecutionFailureCount int64 `json:"execution_failure_count" datastore:"execution_failure_count"` -// ExecutionPendingCount int64 `json:"execution_pending_count" datastore:"execution_pending_count"` -// AppUsageCount int64 `json:"app_usage_count" datastore:"app_usage_count"` -// TotalAppsCount int64 `json:"total_apps_count" datastore:"total_apps_count"` -// SelfMadeAppCount int64 `json:"self_made_app_count" datastore:"self_made_app_count"` -// WebhookUsageCount int64 `json:"webhook_usage_count" datastore:"webhook_usage_count"` -// Baseline map[string]int64 `json:"baseline" datastore:"baseline"` -//} - -type ParsedOpenApi struct { - Body string `datastore:"body,noindex" json:"body"` - ID string `datastore:"id" json:"id"` - Success bool `datastore:"success,omitempty" json:"success,omitempty"` -} - -// Limits set for a user so that they can't do a shitload -type UserLimits struct { - DailyApiUsage int64 `json:"daily_api_usage" datastore:"daily_api_usage"` - DailyWorkflowExecutions int64 `json:"daily_workflow_executions" datastore:"daily_workflow_executions"` - DailyCloudExecutions int64 `json:"daily_cloud_executions" datastore:"daily_cloud_executions"` - DailyTriggers int64 `json:"daily_triggers" datastore:"daily_triggers"` - DailyMailUsage int64 `json:"daily_mail_usage" datastore:"daily_mail_usage"` - MaxTriggers int64 `json:"max_triggers" datastore:"max_triggers"` - MaxWorkflows int64 `json:"max_workflows" datastore:"max_workflows"` -} - type retStruct struct { Success bool `json:"success"` SyncFeatures shuffle.SyncFeatures `json:"sync_features"` @@ -147,43 +94,6 @@ type retStruct struct { Reason string `json:"reason"` } -// Saves some data, not sure what to have here lol -type UserAuth struct { - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - Name string `json:"name" datastore:"name" yaml:"name"` - Workflows []string `json:"workflows" datastore:"workflows"` - Username string `json:"username" datastore:"username"` - Fields []UserAuthField `json:"fields" datastore:"fields"` -} - -type UserAuthField struct { - Key string `json:"key" datastore:"key"` - Value string `json:"value" datastore:"value,noindex"` -} - -// Not environment, but execution environment -//type Environment struct { -// Name string `datastore:"name"` -// Type string `datastore:"type"` -// Registered bool `datastore:"registered"` -// Default bool `datastore:"default" json:"default"` -// Archived bool `datastore:"archived" json:"archived"` -// Id string `datastore:"id" json:"id"` -// OrgId string `datastore:"org_id" json:"org_id"` -//} - -// timeout maybe? idk -type session struct { - Username string `datastore:"Username,noindex"` - Id string `datastore:"Id,noindex"` - Session string `datastore:"session,noindex"` -} - -type loginStruct struct { - Username string `json:"username"` - Password string `json:"password"` -} - type Contact struct { Firstname string `json:"firstname"` Lastname string `json:"lastname"` @@ -3825,7 +3735,7 @@ func remoteOrgJobHandler(org shuffle.Org, interval int) error { //log.Printf("Remote Data: %s", respBody) err = remoteOrgJobController(org, respBody) if err != nil { - log.Printf("[ERROR] Failed job controller run for %s: %s", respBody, err) + //log.Printf("[ERROR] Failed cloud sync job controller run for '%s': %s", respBody, err) return err } return nil @@ -3869,7 +3779,7 @@ func runInitEs(ctx context.Context) { activeOrgs, err := shuffle.GetAllOrgs(ctx) setUsers := false - //log.Printf("ORGS: %d", len(activeOrgs)) + _ = setUsers if err != nil { if fmt.Sprintf("%s", err) == "EOF" { time.Sleep(7 * time.Second) @@ -3924,7 +3834,7 @@ func runInitEs(ctx context.Context) { if len(activeOrgs) == 1 { if len(activeOrgs[0].Users) == 0 { - log.Printf("ORG doesn't have any users??") + log.Printf("[ERROR] Main Org doesn't have any user. Creating.") users, err := shuffle.GetAllUsers(ctx) if err != nil && len(users) == 0 { @@ -3957,10 +3867,9 @@ func runInitEs(ctx context.Context) { if strings.Contains(os.Getenv("SHUFFLE_OPENSEARCH_URL"), "https") { log.Printf("[INFO] Waiting during init to make sure the opensearch instance is up and running with security features properly") - time.Sleep(30 * time.Second) + time.Sleep(15 * time.Second) } - _ = setUsers schedules, err := shuffle.GetAllSchedules(ctx, "ALL") if err != nil { log.Printf("[WARNING] Failed getting schedules during service init: %s", err) @@ -4104,11 +4013,11 @@ func runInitEs(ctx context.Context) { continue } - log.Printf("[DEBUG] Should start schedule for org %s (%s)", org.Name, org.Id) + log.Printf("[DEBUG] Should start cloud schedule for org %s (%s)", org.Name, org.Id) job := func() { err := remoteOrgJobHandler(org, interval) if err != nil { - log.Printf("[ERROR] Failed request with remote org setup (2): %s", err) + log.Printf("[ERROR] Failed request with remote org setup for org %s (2): %s", org.Id, err) } } diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 74676a45..4c29f71e 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -1692,6 +1692,8 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request return shuffle.WorkflowExecution{}, "Cloud not implemented yet", errors.New("Cloud not implemented yet") } + shuffle.IncrementCache(ctx, workflowExecution.OrgId, "workflow_executions_cloud") + // What it needs to know: // 1. Parameters if len(workflowExecution.Workflow.Actions) == 1 { @@ -1705,13 +1707,11 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request // If worker, should this backend be a proxy? I think so. return shuffle.WorkflowExecution{}, "Cloud not implemented yet (2)", errors.New("Cloud not implemented yet") } + } else { + shuffle.IncrementCache(ctx, workflowExecution.OrgId, "workflow_executions_onprem") } - //err = increaseStatisticsField(ctx, "workflow_executions", workflow.ID, 1, workflowExecution.ExecutionOrg) - //if err != nil { - // log.Printf("Failed to increase stats execution stats: %s", err) - //} - + shuffle.IncrementCache(ctx, workflowExecution.OrgId, "workflow_executions") return workflowExecution, "", nil } diff --git a/docker-compose.yml b/docker-compose.yml index 2eb61619..1b93335c 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: frontend: - image: ghcr.io/shuffle/shuffle-frontend:latest + image: ghcr.io/shuffle/shuffle-frontend:nightly container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -61,8 +61,9 @@ services: hostname: shuffle-opensearch container_name: shuffle-opensearch environment: - - bootstrap.memory_lock=true - "OPENSEARCH_JAVA_OPTS=-Xms2048m -Xmx2048m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM + - bootstrap.memory_lock=true + - DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI=true - cluster.initial_master_nodes=shuffle-opensearch - cluster.routing.allocation.disk.threshold_enabled=false - cluster.name=shuffle-cluster diff --git a/frontend/src/components/AppFramework.jsx b/frontend/src/components/AppFramework.jsx index 9750ff7f..ca11154f 100644 --- a/frontend/src/components/AppFramework.jsx +++ b/frontend/src/components/AppFramework.jsx @@ -38,7 +38,7 @@ import { toast } from 'react-toastify'; cytoscape.use(edgehandles); const svgSize = "40px" -const parsedDatatypeImages = { +export const parsedDatatypeImages = { "SIEM": encodeURI(`data:image/svg+xml;utf-8,`), "CASES": encodeURI(`data:image/svg+xml;utf-8,`), @@ -713,7 +713,7 @@ const AppFramework = (props) => { } useEffect(() => { - console.log("DISCWRAP CHANG: ", discoveryWrapper) + //console.log("DISCWRAP CHANG: ", discoveryWrapper) if (discoveryWrapper === undefined || discoveryWrapper.id === "SHUFFLE" || discoveryWrapper.id === undefined || cy === undefined) { setDiscoveryData({}) @@ -874,7 +874,7 @@ const AppFramework = (props) => { }, []) useEffect(() => { - console.log("New selected app: ", newSelectedApp, discoveryData) + //console.log("New selected app: ", newSelectedApp, discoveryData) if (newSelectedApp.objectID === undefined || newSelectedApp.objectID === undefined || newSelectedApp.objectID.length === 0) { return } @@ -2065,8 +2065,8 @@ const AppFramework = (props) => { const foundelement = cy.getElementById(discoveryData.id) if (foundelement !== undefined && foundelement !== null) { - console.log("element: ", foundelement) - console.log("DISC: ", discoveryData) + //console.log("element: ", foundelement) + //console.log("DISC: ", discoveryData) foundelement.data("large_image", parsedDatatypeImages[discoveryData.id.toUpperCase()]) foundelement.data("text_margin_y", "14px") foundelement.data("margin_x", "32px") diff --git a/frontend/src/components/WorkflowTemplatePopup.jsx b/frontend/src/components/WorkflowTemplatePopup.jsx index 769ade88..9db5c484 100644 --- a/frontend/src/components/WorkflowTemplatePopup.jsx +++ b/frontend/src/components/WorkflowTemplatePopup.jsx @@ -12,6 +12,7 @@ import { Drawer, CircularProgress, IconButton, + Tooltip, } from "@mui/material"; import { @@ -34,12 +35,38 @@ const WorkflowTemplatePopup = (props) => { const [workflow, setWorkflow] = useState({}); const [appAuthentication, setAppAuthentication] = React.useState(undefined); + const imagestyleWrapper = { + height: 40, + width: 40, + borderRadius: 40, + border: "1px solid rgba(255,255,255,0.3)", + overflow: "hidden", + display: "flex", + } + + const imagestyleWrapperDefault = { + height: 40, + width: 40, + borderRadius: 40, + border: "1px solid red", + overflow: "hidden", + display: "flex", + } + const imagestyle = { height: 40, width: 40, borderRadius: 40, + border: "1px solid rgba(255,255,255,0.3)", + overflow: "hidden", } + const imagestyleDefault = { + display: "block", + marginLeft: 9, + marginTop: 10, + } + if (title === undefined || title === null || title === "") { console.log("No title for workflow template popup!"); return null @@ -297,7 +324,10 @@ const WorkflowTemplatePopup = (props) => { parsedTitle = title.substring(0, maxlength) + "..." } - parsedTitle.replaceAll("_", " ") + parsedTitle = parsedTitle.replaceAll("_", " ") + + const parsedDescription = description !== undefined && description !== null ? description.replaceAll("_", " ") : "" + return (
@@ -334,7 +364,7 @@ const WorkflowTemplatePopup = (props) => { if (errorMessage !== "") { toast("Already failed to generate workflow for these apps. Please try again later or contact support@shuffler.io.") } else if (isActive) { - toast("Workflow already generated. Please try another template, or continue to the next page.") + toast("Workflow already generated. Please try another workflow template!") // FIXME: Remove these? loadAppAuth() @@ -350,22 +380,32 @@ const WorkflowTemplatePopup = (props) => { >
- - {img2 !== "" ? - - - - + {img1 !== undefined && img1 !== "" && srcapp !== undefined && srcapp !== "" ? + + + + + + : + + } + {img2 !== undefined && img2 !== "" && dstapp !== undefined && dstapp !== "" ? + + + + + + : }
- + {parsedTitle} - {description} + {parsedDescription}
diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 5871c03a..28b02f52 100755 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -228,15 +228,16 @@ const parseCurl = (s) => { // Basically CRUD for each category + special export const appCategories = [ { - "name": "Communication", + "name": "Communication", "color": "#FFC107", "icon": "communication", "action_labels": ["List Messages", "Send Message", "Get Message", "Search messages", "List Attachments", "Get Attachment", "Get Contact"], - }, { + }, + { "name": "SIEM", "color": "#FFC107", "icon": "siem", - "action_labels": ["Search", "List Alerts", "Close Alert", "Get Alert", "Create detection", "Add to lookup list",], + "action_labels": ["Search", "List Alerts", "Close Alert", "Get Alert", "Create detection", "Add to lookup list", "Isolate endpoint",], }, { "name": "Eradication", "color": "#FFC107", diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 4f9ac5b3..d5e88e8c 100755 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -287,6 +287,7 @@ const Apps = (props) => { const [selectedAction, setSelectedAction] = React.useState({}); const [searchBackend, setSearchBackend] = React.useState(false); const [searchableApps, setSearchableApps] = React.useState([]); + const [publishModalOpen, setPublishModalOpen] = React.useState(false); const [openApi, setOpenApi] = React.useState(""); const [openApiData, setOpenApiData] = React.useState(""); @@ -441,6 +442,7 @@ const Apps = (props) => { if (privateapps.length > 0) { if (selectedApp.id === undefined || selectedApp.id === null) { setSelectedApp(privateapps[0]); + setSharingConfiguration(privateapps[0].sharing === true ? "public" : "you") } if ( @@ -646,6 +648,7 @@ const Apps = (props) => { if (selectedApp.id !== data.id) { data.name = newAppname; setSelectedApp(data); + setSharingConfiguration(data.sharing === true ? "public" : "you") if ( data.actions !== undefined && @@ -658,7 +661,7 @@ const Apps = (props) => { } if (data.sharing) { - setSharingConfiguration(isCloud ? "public" : "everyone"); + setSharingConfiguration("public"); } } }} @@ -999,11 +1002,11 @@ const Apps = (props) => { ); }; - const userRoles = ["you", isCloud ? "public" : "everyone"]; + const userRoles = ["you", "public"]; - // Admin in org or creator of app - // FIXME: Missing check for if same creator account - const canEditApp = userdata !== undefined && (userdata.admin === "true" || userdata.id === selectedApp.owner || selectedApp.owner === "" || (userdata.admin === "true" && userdata.active_org.id === selectedApp.reference_org)) || !selectedApp.generated + // Admin in org or creator of app + // FIXME: Missing check for if same creator account + const canEditApp = userdata !== undefined && (userdata.admin === "true" || userdata.id === selectedApp.owner || selectedApp.owner === "" || (userdata.admin === "true" && userdata.active_org.id === selectedApp.reference_org)) || !selectedApp.generated //fetch(globalUrl+"/api/v1/get_openapi/"+urlParams.get("id"), var baseInfo = @@ -1054,6 +1057,7 @@ const Apps = (props) => { console.log("New version: ", newversion); selectedApp.app_version = selectedApp.app_version; setSelectedApp(selectedApp); + setSharingConfiguration(selectedApp.sharing === true ? "public" : "you") if (newversion !== undefined && newversion !== null) { getApp(newversion.id, true); @@ -1156,17 +1160,22 @@ const Apps = (props) => {