#249: Basic fix for sensitive data export

README.md

@@ -13,8 +13,10 @@ Please consider [sponsoring](https://github.com/sponsors/frikky) the project if
 
 ## Support
 * [Discord](https://discord.gg/B2CBzUm)
+* [Twitter](https://twitter.com/shuffleio)
 * [Email](mailto:frikky@shuffler.io)
 * [Open issue](https://github.com/frikky/Shuffle/issues/new)
+* [Shuffler.io](https://shuffler.io/contact)
 
 ## Blogposts
 * [1. Introducing Shuffle](https://medium.com/security-operation-capybara/introducing-shuffle-an-open-source-soar-platform-part-1-58a529de7d12)
@@ -68,12 +70,9 @@ Below is the folder structure with a short explanation
 ├── README.md				# What you're reading right now
 ├── backend					# Contains backend related code.
 │   ├── go-app 			# The backend golang webserver
-│   ├── app_gen 		# Code for app generation outside the Shuffle platform
 │   └── app_sdk			# The SDK used for apps
-├── frontend				# Contains frontend code. ReactJS and cytoscape. Horrible code :)
+├── frontend				# Contains frontend code. ReactJS, Material UI and cytoscape
-├── functions				# Contains google cloud function code mainly.
+├── functions				# Has execution and extension resources, such as the Wazuh integration
-│   ├── static_baseline.py	# Static code used by stitcher.go to generate code
-│   ├── stitcher.go		# Attempts to stitch together an app - part of backend now
 │   ├── onprem				# Code for onprem solutions
 │   │   ├── Orborus 	# Distributes execution locations
 │   │   ├── Worker		# Runs a workflow

backend/app_sdk/app_base.py

@@ -62,7 +62,7 @@ class AppBase:
             if ret.status_code != 200:
                 self.logger.info(ret.text)
         except requests.exceptions.ConnectionError as e:
-            self.logger.exception(e)
+            self.logger.exception("ConnectionError: %s" % e)
             return
         except TypeError as e:
             self.logger.exception(e)
@@ -120,6 +120,11 @@ class AppBase:
         # 1. For the first array, take the total amount(y) (2x3=6) and divide it by the current array (x): 2. x/y = 3. This means do 3 of each value
         # 2. For the second array, take the total amount(y) (2x3=6) and divide it by the current array (x): 3. x/y = 2. 
         # 3. What does the 3rd array do? Same, but ehhh?
+        # 
+        # Example4:
+        # What if there are multiple loops inside a single item?
+        # 
+        #
 
         paramlist = []
         listitems = []

backend/go-app/app.yaml

@@ -1,98 +0,0 @@
-runtime: go111
-
-env_variables:
-
-automatic_scaling:
-  max_instances: 1
-  min_instances: 1
-
-handlers:
-- url: /api/(.*)
-  script: auto
-  secure: always
-- url: /static/js/(.*)
-  static_files: build/static/js/\1
-  upload: build/static/js/(.*)
-  secure: always
-- url: /static/css/(.*)
-  static_files: build/static/css/\1
-  upload: build/static/css/(.*)
-  secure: always
-- url: /images/(.*)
-  static_files: build/images/\1
-  upload: build/images/(.*)
-  secure: always
-- url: /(.*\.(json|ico))$
-  static_files: build/\1
-  upload: build/.*\.(json|ico)$
-  secure: always
-- url: /manifest.json
-  static_files: build/manifest.json
-  upload: build/manifest.json
-  secure: always
-
-# lol.. wildcard doesn't work with /api/(.*) for some reason
-- url: /
-  static_files: build/index.html
-  upload: build/index.html
-  secure: always
-- url: /home
-  static_files: build/index.html
-  upload: build/index.html
-  secure: always
-- url: /passwordreset
-  static_files: build/index.html
-  upload: build/index.html
-  secure: always
-- url: /login
-  static_files: build/index.html
-  upload: build/index.html
-  secure: always
-- url: /register
-  static_files: build/index.html
-  upload: build/index.html
-  secure: always
-- url: /workflows
-  static_files: build/index.html
-  upload: build/index.html
-  secure: always
-- url: /workflows/(.*)
-  static_files: build/index.html
-  upload: build/index.html
-  secure: always
-- url: /info/(.*)
-  static_files: build/index.html
-  upload: build/index.html
-  secure: always
-- url: /docs/(.*)
-  static_files: build/index.html
-  upload: build/index.html
-  secure: always
-- url: /docs
-  static_files: build/index.html
-  upload: build/index.html
-  secure: always
-- url: /settings
-  static_files: build/index.html
-  upload: build/index.html
-  secure: always
-- url: /apps
-  static_files: build/index.html
-  upload: build/index.html
-  secure: always
-- url: /contact
-  static_files: build/index.html
-  upload: build/index.html
-  secure: always
-- url: /apps/(.*)
-  static_files: build/index.html
-  upload: build/index.html
-  secure: always
-- url: /register/(.*)
-  static_files: build/index.html
-  upload: build/index.html
-  secure: always
-- url: /passwordreset/(.*)
-  static_files: build/index.html
-  upload: build/index.html
-  secure: always

backend/go-app/main.go

@@ -6600,7 +6600,9 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) {
 		log.Printf("Failed to increase success execution stats: %s", err)
 	}
 
-	cacheKey := fmt.Sprintf("workflowapps-sorted")
+	cacheKey := fmt.Sprintf("workflowapps-sorted-100")
+	requestCache.Delete(cacheKey)
+	cacheKey = fmt.Sprintf("workflowapps-sorted-500")
 	requestCache.Delete(cacheKey)
 
 	resp.WriteHeader(200)

backend/go-app/walkoff.go

@@ -414,6 +414,7 @@ type Workflow struct {
 	ExecutionEnvironment string     `json:"execution_environment" datastore:"execution_environment"`
 	PreviouslySaved      bool       `json:"first_save" datastore:"first_save"`
 	Categories           Categories `json:"categories" datastore:"categories"`
+	ExampleArgument      string     `json:"example_argument" datastore:"example_argument,noindex"`
 }
 
 type Category struct {
@@ -2093,13 +2094,26 @@ func updateAppAuth(auth AppAuthenticationStorage, workflowId, nodeId string, add
 }
 
 // Identifies what a category defined really is
-func handleCategoryIncrease(categories Categories, action Action) Categories {
+func handleCategoryIncrease(categories Categories, action Action, workflowapps []WorkflowApp) Categories {
-	log.Printf("Action: %s, category: %s", action.AppName, action.Category)
 	if action.Category == "" {
-		log.Printf("Should find app's categories as it's empty during save")
+		appName := action.AppName
+		for _, app := range workflowapps {
+			if appName != strings.ToLower(app.Name) {
+				continue
+			}
+
+			if len(app.Categories) > 0 {
+				log.Printf("[INFO] Setting category for %s: %s", app.Name, app.Categories)
+				action.Category = app.Categories[0]
+				break
+			}
+		}
+
+		//log.Printf("Should find app's categories as it's empty during save")
 		return categories
 	}
 
+	//log.Printf("Action: %s, category: %s", action.AppName, action.Category)
 	// FIXME: Make this an "autodiscover" that's controlled by the category itself
 	// Should just be a list that's looped against :)
 	newCategory := strings.ToLower(action.Category)
@@ -2227,6 +2241,8 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
 	allNodes := []string{}
 	workflow.Categories = Categories{}
 
+	workflowapps, apperr := getAllWorkflowApps(ctx, 500)
+
 	//log.Printf("Action: %#v", action.Authentication)
 	for _, action := range workflow.Actions {
 		allNodes = append(allNodes, action.ID)
@@ -2253,7 +2269,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
 			action.Errors = []string{}
 		}
 
-		workflow.Categories = handleCategoryIncrease(workflow.Categories, action)
+		workflow.Categories = handleCategoryIncrease(workflow.Categories, action, workflowapps)
 		newActions = append(newActions, action)
 	}
 
@@ -2261,7 +2277,6 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
 		log.Printf("[WORKFLOW INIT] NOT PREVIOUSLY SAVED - SET ACTION AUTH!")
 		//AuthenticationId string `json:"authentication_id,omitempty" datastore:"authentication_id"`
 
-		workflowapps, apperr := getAllWorkflowApps(ctx, 500)
 		allAuths, err := getAllWorkflowAppAuth(ctx, user.ActiveOrg.Id)
 		if err == nil && len(workflowapps) > 0 && apperr == nil {
 			//log.Printf("Setting actions")
@@ -2819,8 +2834,11 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
 		Errors:  workflow.Errors,
 	}
 
-	cacheKey := fmt.Sprintf("workflowapps-sorted")
+	cacheKey := fmt.Sprintf("workflowapps-sorted-100")
 	requestCache.Delete(cacheKey)
+	cacheKey = fmt.Sprintf("workflowapps-sorted-500")
+	requestCache.Delete(cacheKey)
+
 	log.Printf("[INFO] Saved new version of workflow %s (%s) for org %s", workflow.Name, fileId, workflow.OrgId)
 	resp.WriteHeader(200)
 	newBody, err := json.Marshal(returndata)
@@ -4706,7 +4724,9 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) {
 	if err != nil {
 		log.Printf("Failed to increase total apps loaded stats: %s", err)
 	}
-	cacheKey := fmt.Sprintf("workflowapps-sorted")
+	cacheKey := fmt.Sprintf("workflowapps-sorted-100")
+	requestCache.Delete(cacheKey)
+	cacheKey = fmt.Sprintf("workflowapps-sorted-500")
 	requestCache.Delete(cacheKey)
 
 	//err = memcache.Delete(request.Context(), sessionToken)
@@ -5273,7 +5293,9 @@ func updateWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) {
 		return
 	}
 
-	cacheKey := fmt.Sprintf("workflowapps-sorted")
+	cacheKey := fmt.Sprintf("workflowapps-sorted-100")
+	requestCache.Delete(cacheKey)
+	cacheKey = fmt.Sprintf("workflowapps-sorted-500")
 	requestCache.Delete(cacheKey)
 
 	log.Printf("Changed workflow app %s", app.ID)
@@ -6166,7 +6188,9 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string,
 							continue
 						}
 
-						cacheKey := fmt.Sprintf("workflowapps-sorted")
+						cacheKey := fmt.Sprintf("workflowapps-sorted-100")
+						requestCache.Delete(cacheKey)
+						cacheKey = fmt.Sprintf("workflowapps-sorted-500")
 						requestCache.Delete(cacheKey)
 					}
 				} else {
@@ -6680,7 +6704,9 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) {
 	}
 
 	//memcache.Delete(ctx, "all_apps")
-	cacheKey := fmt.Sprintf("workflowapps-sorted")
+	cacheKey := fmt.Sprintf("workflowapps-sorted-100")
+	requestCache.Delete(cacheKey)
+	cacheKey = fmt.Sprintf("workflowapps-sorted-500")
 	requestCache.Delete(cacheKey)
 
 	resp.WriteHeader(200)
@@ -6824,6 +6850,10 @@ func getAllWorkflowApps(ctx context.Context, maxLen int) ([]WorkflowApp, error)
 				break
 			}
 
+			if app.Name == "Shuffle Subflow" {
+				continue
+			}
+
 			found := false
 			//log.Printf("ACTIONS: %d - %s", len(app.Actions), app.Name)
 			for _, innerapp := range apps {

frontend/src/views/AngularWorkflow.jsx

@@ -540,7 +540,7 @@ const AngularWorkflow = (props) => {
 						currentnode.removeClass('awaiting-data-highlight')
 						currentnode.addClass('success-highlight')
 
-						if (!visited.includes(item.action.label)) {
+						if (visited !== undefined && visited !== null && !visited.includes(item.action.label)) {
 							if (executionRunning) {
 								//alert.show("Success in node "+item.action.label)
 								//+" with result "+item.result)
@@ -1382,7 +1382,7 @@ const AngularWorkflow = (props) => {
 					//throw BreakException
 					return false 
 				}
-			});
+			})
 		}
 
 
@@ -1766,13 +1766,13 @@ const AngularWorkflow = (props) => {
 	const stopSchedule = (trigger, triggerindex) => {
 		alert.info("Stopping schedule")
 		fetch(globalUrl+"/api/v1/workflows/"+props.match.params.key+"/schedule/"+trigger.id, {
-    	  	method: 'DELETE',
+			method: 'DELETE',
-				headers: {
+			headers: {
-					'Content-Type': 'application/json',
+				'Content-Type': 'application/json',
-					'Accept': 'application/json',
+				'Accept': 'application/json',
-				},
+			},
-	  			credentials: "include",
+				credentials: "include",
-    		})
+		})
 		.then((response) => {
 			if (response.status !== 200) {
 				console.log("Status not 200 for stream results :O!")
@@ -1783,21 +1783,16 @@ const AngularWorkflow = (props) => {
 		.then((responseJson) => {
 			// No matter what, it's being stopped.
 			if (!responseJson.success) {
-				alert.error("Failed to stop schedule: " + responseJson.reason)
+				alert.WARNING("Failed to stop schedule: " + responseJson.reason)
-
-				workflow.triggers[triggerindex].status = "stopped" 
-				trigger.status = "stopped" 
-				setSelectedTrigger(trigger)
-				setWorkflow(workflow)
-				saveWorkflow(workflow)
 			} else {
 				alert.success("Successfully stopped schedule")
-				workflow.triggers[triggerindex].status = "stopped" 
-				trigger.status = "stopped" 
-				setSelectedTrigger(trigger)
-				setWorkflow(workflow)
-				saveWorkflow(workflow)
 			}
+
+			workflow.triggers[triggerindex].status = "stopped" 
+			trigger.status = "stopped" 
+			setSelectedTrigger(trigger)
+			setWorkflow(workflow)
+			saveWorkflow(workflow)
 		})
 		.catch(error => {
 			alert.error(error.toString())
@@ -2535,7 +2530,8 @@ const AngularWorkflow = (props) => {
 			newAppname = newAppname.slice(0, maxlen)+".."
 		}
 
-		const image = "url("+app.large_image+")"
+		//const image = "url("+app.large_image+")"
+		const image = app.large_image
 		const newAppStyle = JSON.parse(JSON.stringify(paperAppStyle))
 		const pixelSize = !hover ? "2px" : "4px"
 		newAppStyle.borderLeft = app.is_valid ? `${pixelSize} solid green` : `${pixelSize} solid orange`
@@ -2555,7 +2551,7 @@ const AngularWorkflow = (props) => {
 				<Paper square style={newAppStyle} onMouseOver={() => {setHover(true)}} onMouseOut={() => {setHover(false)}}>
 					<Grid container style={{margin: "10px 10px 10px 15px", flex: "10"}}>
 						<Grid item>
-							<div style={{borderRadius: borderRadius, height: 80, width: 80, backgroundImage: image, backgroundSize: "cover", backgroundRepeat: "no-repeat"}} />
+							<img alt={newAppname} src={image} style={{borderRadius: borderRadius, height: 80, width: 80,}} />
 						</Grid>
 						<Grid style={{display: "flex", flexDirection: "column", marginLeft: "20px", minWidth: 185, maxWidth: 185, overflow: "hidden", maxHeight: 80, }}>
 							<Grid item style={{flex: 1}}>
@@ -5530,6 +5526,7 @@ const AngularWorkflow = (props) => {
 		if (trigger.id === undefined) {
 			return
 		}
+
 		alert.info("Stopping webhook")
 
 		fetch(globalUrl+"/api/v1/hooks/"+trigger.id+"/delete", {
@@ -5548,18 +5545,23 @@ const AngularWorkflow = (props) => {
 			return response.json()
 		})
     .then((responseJson) => {
-			workflow.triggers[triggerindex].status = "stopped"
+			if (workflow.triggers[triggerindex] !== undefined) {
-			trigger.status = "stopped"
+				workflow.triggers[triggerindex].status = "stopped"
-			setWorkflow(workflow)
+			}
-			setSelectedTrigger(trigger)
 
 			if (responseJson.success) {
 				//alert.success("Successfully stopped webhook")
 				// Set the status
 				saveWorkflow(workflow)
 			} else {
-				alert.error("Failed stopping webhook: "+responseJson.reason)
+				if (responseJson.reason !== undefined) {
+					alert.error("Failed stopping webhook: "+responseJson.reason)
+				}
 			}
+
+			trigger.status = "stopped"
+			setWorkflow(workflow)
+			setSelectedTrigger(trigger)
 		})
 		.catch(error => {
 			alert.error(error.toString())
@@ -6937,7 +6939,7 @@ const AngularWorkflow = (props) => {
 		: null
 	
 	const variablesModal = variablesModalOpen ? 
-		<Dialog modal 
+		<Dialog 
 			open={variablesModalOpen} 
 			onClose={() => {
 				setNewVariableName("")

frontend/src/views/AppCreator.jsx

@@ -2150,7 +2150,7 @@ const AppCreator = (props) => {
 						</h2>
 					</Link>
 					<h2>
-						{name} ({actions === null || actions === undefined ? 0 : actions.length})
+						{name} {actions === null || actions === undefined || actions.length === 0 ? null : <span>({actions.length})</span>}
 					</h2>
 				</Breadcrumbs>
 				<Paper style={boxStyle}>

frontend/src/views/Apps.jsx

@@ -145,6 +145,7 @@ const Apps = (props) => {
 	const [isDropzone, setIsDropzone] = React.useState(false);
 	const upload = React.useRef(null);
 	const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" ? true : false
+	const borderRadius = 3
 
 	const { start, stop } = useInterval({
 	  	duration: 5000,
@@ -330,9 +331,9 @@ const Apps = (props) => {
 		//<div style={{backgroundColor: theme.palette.inputColor, height: 100, width: 100, borderRadius: 3, verticalAlign: "middle", textAlign: "center", display: "table-cell"}}>
 		// <div style={{width: "100px", height: "100px", border: "1px solid black", verticalAlign: "middle", textAlign: "center", display: "table-cell"}}>
 		var imageline = data.large_image.length === 0 ?
-			<img alt={data.title} style={{width: 100, height: 100, backgroundColor: theme.palette.inputColor,}} />
+			<img alt={data.title} style={{borderRadius: borderRadius, width: 100, height: 100, backgroundColor: theme.palette.inputColor,}} />
 			: 
-			<img alt={data.title} src={data.large_image} style={{maxWidth: 100, maxHeight: "100%", display: "block", margin: "0 auto"}} onLoad={(event) => {
+			<img alt={data.title} src={data.large_image} style={{borderRadius: borderRadius, maxWidth: 100, minWidth: 100, maxHeight: "100%", display: "block", margin: "0 auto"}} onLoad={(event) => {
 				//console.log("IMG LOADED!: ", event.target)
 			}} />
 
@@ -535,9 +536,9 @@ const Apps = (props) => {
 			: null
 
 		var imageline = selectedApp.large_image === undefined || selectedApp.large_image.length === 0 ?
-			<img alt={selectedApp.title} style={{width: 100, height: 100, backgroundColor: theme.palette.inputColor,}} />
+			<img alt={selectedApp.title} style={{borderRadius: borderRadius, width: 100, height: 100, backgroundColor: theme.palette.inputColor,}} />
 			: 
-			<img alt={selectedApp.title} src={selectedApp.large_image} style={{maxHeight: 100, maxWidth: 100, backgroundColor: theme.palette.inputColor}} />
+			<img alt={selectedApp.title} src={selectedApp.large_image} style={{borderRadius: borderRadius, maxWidth: 100, height: "auto", backgroundColor: theme.palette.inputColor}} />
 
 		const GetAppExample = () => {
 			if (selectedAction.returns === undefined) {

frontend/src/views/Workflows.jsx

@@ -395,23 +395,44 @@ const Workflows = (props) => {
 		let exportFileDefaultName = data.name+'.json';
 
 		data["owner"] = ""
-		for (var key in data.triggers) {
+		if (data.triggers !== null && data.triggers !== undefined) {
-			const trigger = data.triggers[key]
+			for (var key in data.triggers) {
-			if (trigger.app_name === "Shuffle Workflow") {
+				const trigger = data.triggers[key]
-				if (trigger.parameters.length > 2) {
+				if (trigger.app_name === "Shuffle Workflow") {
-					trigger.parameters[2].value = ""
+					if (trigger.parameters.length > 2) {
+						trigger.parameters[2].value = ""
+					}
+				} 
+				
+				if (trigger.status == "running") {
+					trigger.status = "stopped"
 				}
-			} 
-			
-			if (trigger.status == "running") {
-				trigger.status = "stopped"
 			}
 		}
 
-		for (var key in data.actions) {
+		if (data.actions !== null && data.actions !== undefined) {
-			data.actions[key].authentication_id = ""
+			for (var key in data.actions) {
+				data.actions[key].authentication_id = ""
+
+				for (var subkey in data.actions[key].parameters) {
+					const param = data.actions[key].parameters[subkey]
+					if (param.name.includes("key") || param.name.includes("user") || param.name.includes("pass") || param.name.includes("api") || param.name.includes("auth") || param.name.includes("secret")) {
+						param.value = ""
+					}
+				}
+			}
 		}
 
+		if (data.workflow_variables !== null && data.workflow_variables !== undefined) {
+			for (var key in data.workflow_variables) {
+				const param = data.workflow_variables[key]
+				if (param.name.includes("key") || param.name.includes("user") || param.name.includes("pass") || param.name.includes("api") || param.name.includes("auth") || param.name.includes("secret")) {
+					param.value = ""
+				}
+			}
+		}
+
+		//console.log(data)
 		//return
 
 		data["org"] = []