From 3969c8e998818e63d39da81eb5717c5530f01100 Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 8 Feb 2021 08:41:22 +0100 Subject: [PATCH] #249: Basic fix for sensitive data export --- README.md | 9 ++- backend/app_sdk/app_base.py | 7 +- backend/go-app/app.yaml | 98 -------------------------- backend/go-app/main.go | 4 +- backend/go-app/walkoff.go | 50 ++++++++++--- frontend/src/views/AngularWorkflow.jsx | 60 ++++++++-------- frontend/src/views/AppCreator.jsx | 2 +- frontend/src/views/Apps.jsx | 9 +-- frontend/src/views/Workflows.jsx | 43 ++++++++--- 9 files changed, 122 insertions(+), 160 deletions(-) delete mode 100644 backend/go-app/app.yaml diff --git a/README.md b/README.md index f9589f40..a996646e 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,10 @@ Please consider [sponsoring](https://github.com/sponsors/frikky) the project if ## Support * [Discord](https://discord.gg/B2CBzUm) +* [Twitter](https://twitter.com/shuffleio) * [Email](mailto:frikky@shuffler.io) * [Open issue](https://github.com/frikky/Shuffle/issues/new) +* [Shuffler.io](https://shuffler.io/contact) ## Blogposts * [1. Introducing Shuffle](https://medium.com/security-operation-capybara/introducing-shuffle-an-open-source-soar-platform-part-1-58a529de7d12) @@ -68,12 +70,9 @@ Below is the folder structure with a short explanation ├── README.md # What you're reading right now ├── backend # Contains backend related code. │   ├── go-app # The backend golang webserver -│   ├── app_gen # Code for app generation outside the Shuffle platform │ └── app_sdk # The SDK used for apps -├── frontend # Contains frontend code. ReactJS and cytoscape. Horrible code :) -├── functions # Contains google cloud function code mainly. -│   ├── static_baseline.py # Static code used by stitcher.go to generate code -│   ├── stitcher.go # Attempts to stitch together an app - part of backend now +├── frontend # Contains frontend code. ReactJS, Material UI and cytoscape +├── functions # Has execution and extension resources, such as the Wazuh integration │   ├── onprem # Code for onprem solutions │  │   ├── Orborus # Distributes execution locations │  │   ├── Worker # Runs a workflow diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 021f58ed..5320ecd9 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -62,7 +62,7 @@ class AppBase: if ret.status_code != 200: self.logger.info(ret.text) except requests.exceptions.ConnectionError as e: - self.logger.exception(e) + self.logger.exception("ConnectionError: %s" % e) return except TypeError as e: self.logger.exception(e) @@ -120,6 +120,11 @@ class AppBase: # 1. For the first array, take the total amount(y) (2x3=6) and divide it by the current array (x): 2. x/y = 3. This means do 3 of each value # 2. For the second array, take the total amount(y) (2x3=6) and divide it by the current array (x): 3. x/y = 2. # 3. What does the 3rd array do? Same, but ehhh? + # + # Example4: + # What if there are multiple loops inside a single item? + # + # paramlist = [] listitems = [] diff --git a/backend/go-app/app.yaml b/backend/go-app/app.yaml deleted file mode 100644 index 59f28320..00000000 --- a/backend/go-app/app.yaml +++ /dev/null @@ -1,98 +0,0 @@ -runtime: go111 - -env_variables: - -automatic_scaling: - max_instances: 1 - min_instances: 1 - -handlers: -- url: /api/(.*) - script: auto - secure: always -- url: /static/js/(.*) - static_files: build/static/js/\1 - upload: build/static/js/(.*) - secure: always -- url: /static/css/(.*) - static_files: build/static/css/\1 - upload: build/static/css/(.*) - secure: always -- url: /images/(.*) - static_files: build/images/\1 - upload: build/images/(.*) - secure: always -- url: /(.*\.(json|ico))$ - static_files: build/\1 - upload: build/.*\.(json|ico)$ - secure: always -- url: /manifest.json - static_files: build/manifest.json - upload: build/manifest.json - secure: always - -# lol.. wildcard doesn't work with /api/(.*) for some reason -- url: / - static_files: build/index.html - upload: build/index.html - secure: always -- url: /home - static_files: build/index.html - upload: build/index.html - secure: always -- url: /passwordreset - static_files: build/index.html - upload: build/index.html - secure: always -- url: /login - static_files: build/index.html - upload: build/index.html - secure: always -- url: /register - static_files: build/index.html - upload: build/index.html - secure: always -- url: /workflows - static_files: build/index.html - upload: build/index.html - secure: always -- url: /workflows/(.*) - static_files: build/index.html - upload: build/index.html - secure: always -- url: /info/(.*) - static_files: build/index.html - upload: build/index.html - secure: always -- url: /docs/(.*) - static_files: build/index.html - upload: build/index.html - secure: always -- url: /docs - static_files: build/index.html - upload: build/index.html - secure: always -- url: /settings - static_files: build/index.html - upload: build/index.html - secure: always -- url: /apps - static_files: build/index.html - upload: build/index.html - secure: always -- url: /contact - static_files: build/index.html - upload: build/index.html - secure: always -- url: /apps/(.*) - static_files: build/index.html - upload: build/index.html - secure: always -- url: /register/(.*) - static_files: build/index.html - upload: build/index.html - secure: always -- url: /passwordreset/(.*) - static_files: build/index.html - upload: build/index.html - secure: always diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 1ffd334f..aef237d7 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -6600,7 +6600,9 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { log.Printf("Failed to increase success execution stats: %s", err) } - cacheKey := fmt.Sprintf("workflowapps-sorted") + cacheKey := fmt.Sprintf("workflowapps-sorted-100") + requestCache.Delete(cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") requestCache.Delete(cacheKey) resp.WriteHeader(200) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index f8b76e75..91d80d1d 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -414,6 +414,7 @@ type Workflow struct { ExecutionEnvironment string `json:"execution_environment" datastore:"execution_environment"` PreviouslySaved bool `json:"first_save" datastore:"first_save"` Categories Categories `json:"categories" datastore:"categories"` + ExampleArgument string `json:"example_argument" datastore:"example_argument,noindex"` } type Category struct { @@ -2093,13 +2094,26 @@ func updateAppAuth(auth AppAuthenticationStorage, workflowId, nodeId string, add } // Identifies what a category defined really is -func handleCategoryIncrease(categories Categories, action Action) Categories { - log.Printf("Action: %s, category: %s", action.AppName, action.Category) +func handleCategoryIncrease(categories Categories, action Action, workflowapps []WorkflowApp) Categories { if action.Category == "" { - log.Printf("Should find app's categories as it's empty during save") + appName := action.AppName + for _, app := range workflowapps { + if appName != strings.ToLower(app.Name) { + continue + } + + if len(app.Categories) > 0 { + log.Printf("[INFO] Setting category for %s: %s", app.Name, app.Categories) + action.Category = app.Categories[0] + break + } + } + + //log.Printf("Should find app's categories as it's empty during save") return categories } + //log.Printf("Action: %s, category: %s", action.AppName, action.Category) // FIXME: Make this an "autodiscover" that's controlled by the category itself // Should just be a list that's looped against :) newCategory := strings.ToLower(action.Category) @@ -2227,6 +2241,8 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { allNodes := []string{} workflow.Categories = Categories{} + workflowapps, apperr := getAllWorkflowApps(ctx, 500) + //log.Printf("Action: %#v", action.Authentication) for _, action := range workflow.Actions { allNodes = append(allNodes, action.ID) @@ -2253,7 +2269,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { action.Errors = []string{} } - workflow.Categories = handleCategoryIncrease(workflow.Categories, action) + workflow.Categories = handleCategoryIncrease(workflow.Categories, action, workflowapps) newActions = append(newActions, action) } @@ -2261,7 +2277,6 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { log.Printf("[WORKFLOW INIT] NOT PREVIOUSLY SAVED - SET ACTION AUTH!") //AuthenticationId string `json:"authentication_id,omitempty" datastore:"authentication_id"` - workflowapps, apperr := getAllWorkflowApps(ctx, 500) allAuths, err := getAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) if err == nil && len(workflowapps) > 0 && apperr == nil { //log.Printf("Setting actions") @@ -2819,8 +2834,11 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { Errors: workflow.Errors, } - cacheKey := fmt.Sprintf("workflowapps-sorted") + cacheKey := fmt.Sprintf("workflowapps-sorted-100") requestCache.Delete(cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") + requestCache.Delete(cacheKey) + log.Printf("[INFO] Saved new version of workflow %s (%s) for org %s", workflow.Name, fileId, workflow.OrgId) resp.WriteHeader(200) newBody, err := json.Marshal(returndata) @@ -4706,7 +4724,9 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { if err != nil { log.Printf("Failed to increase total apps loaded stats: %s", err) } - cacheKey := fmt.Sprintf("workflowapps-sorted") + cacheKey := fmt.Sprintf("workflowapps-sorted-100") + requestCache.Delete(cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") requestCache.Delete(cacheKey) //err = memcache.Delete(request.Context(), sessionToken) @@ -5273,7 +5293,9 @@ func updateWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { return } - cacheKey := fmt.Sprintf("workflowapps-sorted") + cacheKey := fmt.Sprintf("workflowapps-sorted-100") + requestCache.Delete(cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") requestCache.Delete(cacheKey) log.Printf("Changed workflow app %s", app.ID) @@ -6166,7 +6188,9 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, continue } - cacheKey := fmt.Sprintf("workflowapps-sorted") + cacheKey := fmt.Sprintf("workflowapps-sorted-100") + requestCache.Delete(cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") requestCache.Delete(cacheKey) } } else { @@ -6680,7 +6704,9 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { } //memcache.Delete(ctx, "all_apps") - cacheKey := fmt.Sprintf("workflowapps-sorted") + cacheKey := fmt.Sprintf("workflowapps-sorted-100") + requestCache.Delete(cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") requestCache.Delete(cacheKey) resp.WriteHeader(200) @@ -6824,6 +6850,10 @@ func getAllWorkflowApps(ctx context.Context, maxLen int) ([]WorkflowApp, error) break } + if app.Name == "Shuffle Subflow" { + continue + } + found := false //log.Printf("ACTIONS: %d - %s", len(app.Actions), app.Name) for _, innerapp := range apps { diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 9594cce5..99073bc3 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -540,7 +540,7 @@ const AngularWorkflow = (props) => { currentnode.removeClass('awaiting-data-highlight') currentnode.addClass('success-highlight') - if (!visited.includes(item.action.label)) { + if (visited !== undefined && visited !== null && !visited.includes(item.action.label)) { if (executionRunning) { //alert.show("Success in node "+item.action.label) //+" with result "+item.result) @@ -1382,7 +1382,7 @@ const AngularWorkflow = (props) => { //throw BreakException return false } - }); + }) } @@ -1766,13 +1766,13 @@ const AngularWorkflow = (props) => { const stopSchedule = (trigger, triggerindex) => { alert.info("Stopping schedule") fetch(globalUrl+"/api/v1/workflows/"+props.match.params.key+"/schedule/"+trigger.id, { - method: 'DELETE', - headers: { - 'Content-Type': 'application/json', - 'Accept': 'application/json', - }, - credentials: "include", - }) + method: 'DELETE', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + credentials: "include", + }) .then((response) => { if (response.status !== 200) { console.log("Status not 200 for stream results :O!") @@ -1783,21 +1783,16 @@ const AngularWorkflow = (props) => { .then((responseJson) => { // No matter what, it's being stopped. if (!responseJson.success) { - alert.error("Failed to stop schedule: " + responseJson.reason) - - workflow.triggers[triggerindex].status = "stopped" - trigger.status = "stopped" - setSelectedTrigger(trigger) - setWorkflow(workflow) - saveWorkflow(workflow) + alert.WARNING("Failed to stop schedule: " + responseJson.reason) } else { alert.success("Successfully stopped schedule") - workflow.triggers[triggerindex].status = "stopped" - trigger.status = "stopped" - setSelectedTrigger(trigger) - setWorkflow(workflow) - saveWorkflow(workflow) } + + workflow.triggers[triggerindex].status = "stopped" + trigger.status = "stopped" + setSelectedTrigger(trigger) + setWorkflow(workflow) + saveWorkflow(workflow) }) .catch(error => { alert.error(error.toString()) @@ -2535,7 +2530,8 @@ const AngularWorkflow = (props) => { newAppname = newAppname.slice(0, maxlen)+".." } - const image = "url("+app.large_image+")" + //const image = "url("+app.large_image+")" + const image = app.large_image const newAppStyle = JSON.parse(JSON.stringify(paperAppStyle)) const pixelSize = !hover ? "2px" : "4px" newAppStyle.borderLeft = app.is_valid ? `${pixelSize} solid green` : `${pixelSize} solid orange` @@ -2555,7 +2551,7 @@ const AngularWorkflow = (props) => { {setHover(true)}} onMouseOut={() => {setHover(false)}}> -
+ {newAppname} @@ -5530,6 +5526,7 @@ const AngularWorkflow = (props) => { if (trigger.id === undefined) { return } + alert.info("Stopping webhook") fetch(globalUrl+"/api/v1/hooks/"+trigger.id+"/delete", { @@ -5548,18 +5545,23 @@ const AngularWorkflow = (props) => { return response.json() }) .then((responseJson) => { - workflow.triggers[triggerindex].status = "stopped" - trigger.status = "stopped" - setWorkflow(workflow) - setSelectedTrigger(trigger) + if (workflow.triggers[triggerindex] !== undefined) { + workflow.triggers[triggerindex].status = "stopped" + } if (responseJson.success) { //alert.success("Successfully stopped webhook") // Set the status saveWorkflow(workflow) } else { - alert.error("Failed stopping webhook: "+responseJson.reason) + if (responseJson.reason !== undefined) { + alert.error("Failed stopping webhook: "+responseJson.reason) + } } + + trigger.status = "stopped" + setWorkflow(workflow) + setSelectedTrigger(trigger) }) .catch(error => { alert.error(error.toString()) @@ -6937,7 +6939,7 @@ const AngularWorkflow = (props) => { : null const variablesModal = variablesModalOpen ? - { setNewVariableName("") diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index f23d42ba..1a3c1a7c 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -2150,7 +2150,7 @@ const AppCreator = (props) => {

- {name} ({actions === null || actions === undefined ? 0 : actions.length}) + {name} {actions === null || actions === undefined || actions.length === 0 ? null : ({actions.length})}

diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 5ae1ee01..2555459b 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -145,6 +145,7 @@ const Apps = (props) => { const [isDropzone, setIsDropzone] = React.useState(false); const upload = React.useRef(null); const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" ? true : false + const borderRadius = 3 const { start, stop } = useInterval({ duration: 5000, @@ -330,9 +331,9 @@ const Apps = (props) => { //
//
var imageline = data.large_image.length === 0 ? - {data.title} + {data.title} : - {data.title} { + {data.title} { //console.log("IMG LOADED!: ", event.target) }} /> @@ -535,9 +536,9 @@ const Apps = (props) => { : null var imageline = selectedApp.large_image === undefined || selectedApp.large_image.length === 0 ? - {selectedApp.title} + {selectedApp.title} : - {selectedApp.title} + {selectedApp.title} const GetAppExample = () => { if (selectedAction.returns === undefined) { diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index f28ad54a..0f1d6107 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -395,23 +395,44 @@ const Workflows = (props) => { let exportFileDefaultName = data.name+'.json'; data["owner"] = "" - for (var key in data.triggers) { - const trigger = data.triggers[key] - if (trigger.app_name === "Shuffle Workflow") { - if (trigger.parameters.length > 2) { - trigger.parameters[2].value = "" + if (data.triggers !== null && data.triggers !== undefined) { + for (var key in data.triggers) { + const trigger = data.triggers[key] + if (trigger.app_name === "Shuffle Workflow") { + if (trigger.parameters.length > 2) { + trigger.parameters[2].value = "" + } + } + + if (trigger.status == "running") { + trigger.status = "stopped" } - } - - if (trigger.status == "running") { - trigger.status = "stopped" } } - for (var key in data.actions) { - data.actions[key].authentication_id = "" + if (data.actions !== null && data.actions !== undefined) { + for (var key in data.actions) { + data.actions[key].authentication_id = "" + + for (var subkey in data.actions[key].parameters) { + const param = data.actions[key].parameters[subkey] + if (param.name.includes("key") || param.name.includes("user") || param.name.includes("pass") || param.name.includes("api") || param.name.includes("auth") || param.name.includes("secret")) { + param.value = "" + } + } + } } + if (data.workflow_variables !== null && data.workflow_variables !== undefined) { + for (var key in data.workflow_variables) { + const param = data.workflow_variables[key] + if (param.name.includes("key") || param.name.includes("user") || param.name.includes("pass") || param.name.includes("api") || param.name.includes("auth") || param.name.includes("secret")) { + param.value = "" + } + } + } + + //console.log(data) //return data["org"] = []