Added major JSON change to action

README.md

@@ -87,8 +87,8 @@ There will be a major overhaul to the backend specifically. I'm currently moving
 - * Full OpenAPI support with authentication schemes in App creator (not Oauth2 yet)
 - * Change workflow name
 - * User run statistics 
+- Extended result data usage, build json with answers, not just "from previous action"
 - Workflows 		- IMPORT DEFAULT WORKFLOWS - Create some towards e.g. TheHive & MISP.
-- Extended data usage - build json with answers
 - Documentation - General documentation /docs rewrite
 - API doc				- 1. In Shuffle. 2. In e.g. python
 - Fix scheduler

frontend/src/AngularWorkflow.js

@@ -1028,8 +1028,6 @@ const AngularWorkflow = (props) => {
 		}, {
 			duration: animationDuration,	
 		})
-
-		console.log(previousnodecolor)
 	}
 
 	const onNodeHover = (event) => {
@@ -2134,11 +2132,15 @@ const AngularWorkflow = (props) => {
 						var staticcolor = "inherit"
 						var actioncolor = "inherit"
 						var varcolor = "inherit"
-						var multiline = false
+						var multiline
 						if (data.multiline !== undefined && data.multiline !== null && data.multiline === true) {
 							multiline = true
 						}
 
+						if (data.value.startsWith("{") && data.value.endsWith("}")) {
+							multiline = true
+						}
+
 						var placeholder = "Static value"
 						if (data.example !== undefined && data.example !== null && data.example.length > 0) {
 							placeholder = data.example
@@ -2165,12 +2167,24 @@ const AngularWorkflow = (props) => {
 								onChange={(event) => {
 									changeActionParameter(event, count)
 								}}
+								onBlur={(event) => {
+									// Super basic check
+									if (event.target.value.startsWith("{")) {
+										console.log("VALIDATING JSON")
+										try {
+											JSON.parse(event.target.value)
+										} catch (e) {
+											alert.error("Failed to parse json")
+										}
+									}
+								}}
 							/>
 
 
 						// Remap data based on variant
 						if (data.variant === "STATIC_VALUE") {
 							staticcolor = "#f85a3e"	
+							console.log("DATA IS STATIC")
 						} else if (data.variant === "ACTION_RESULT") {
 							// Gets the parents of the current node
 							var parents = getParents(selectedAction)
@@ -2390,7 +2404,7 @@ const AngularWorkflow = (props) => {
 			<div style={{marginTop: "20px"}}>
 				Environment:
 				<Select
-					value={selectedActionEnvironment.Name === undefined ? "" : selectedActionEnvironment.Name}
+					value={selectedActionEnvironment === undefined || selectedActionEnvironment.Name === undefined ? "" : selectedActionEnvironment.Name}
 					PaperProps={{
 						style: {
 							backgroundColor: inputColor,
@@ -2597,12 +2611,13 @@ const AngularWorkflow = (props) => {
 	}
 
 	const AppConditionHandler = (props) => {
-  		const { tmpdata, type } = props;
+  	const { tmpdata, type } = props;
 
 		if (tmpdata === undefined) {
 			return tmpdata
 		}
 		const [data, ] = useState(tmpdata)
+		const [multiline, setMultiline] = useState(false)
 
 		if (data.variant === "") {
 			data.variant = "STATIC_VALUE"
@@ -2611,9 +2626,8 @@ const AngularWorkflow = (props) => {
 		var staticcolor = "inherit"
 		var actioncolor = "inherit"
 		var varcolor = "inherit"
-		var multiline = false
 		if (data.multiline !== undefined && data.multiline !== null && data.multiline === true) {
-			multiline = true
+			setMultiline(true)
 		}
 
 		var placeholder = "Static value"
@@ -2635,10 +2649,14 @@ const AngularWorkflow = (props) => {
 				}}
 				fullWidth
 				multiline={multiline}
-				rows="5"
+				rows={5}
 				color="primary"
 				defaultValue={data.value}
 				placeholder={placeholder}
+				onClick={() => {
+					console.log("CHANGE FIELD")
+					setMultiline(!multiline)
+				}}
 				onBlur={(e) => {
 					changeActionVariable(data.action_field, e.target.value)
 				}}
@@ -2701,12 +2719,12 @@ const AngularWorkflow = (props) => {
 				setCurrentView("variables")
 				datafield = 
 				<div>
-				<div>
+					<div>
-					Looks like you don't have any variables yet.  
+						Looks like you don't have any variables yet.  
-				</div>
+					</div>
-				<div style={{width: "100%", margin: "auto"}}>
+					<div style={{width: "100%", margin: "auto"}}>
-					<Button style={{margin: "auto", marginTop: "10px",}} color="primary" variant="outlined" onClick={() => setVariablesModalOpen(true)}>New workflow variable</Button> 				
+						<Button style={{margin: "auto", marginTop: "10px",}} color="primary" variant="outlined" onClick={() => setVariablesModalOpen(true)}>New workflow variable</Button> 				
-				</div>
+					</div>
 				</div>
 			} else {
 				// FIXME - this is a shitty solution that needs re-renders all the time
@@ -4022,6 +4040,7 @@ const AngularWorkflow = (props) => {
 							}}
 							color="secondary"
 							placeholder={"Execution Argument"}
+							defaultValue={executionText}
 							onBlur={(e) => {
 								setExecutionText(e.target.value)
 							}}

frontend/src/App.js

@@ -38,12 +38,12 @@ import AlertTemplate from "react-alert-template-basic";
 import { positions, Provider } from "react-alert";
 
 // Testing - localhost
-//const globalUrl = "http://192.168.3.6:5001"
+const globalUrl = "http://192.168.3.6:5001"
 //console.log("HOST: ", process.env)
 
 
 // Production - backend proxy forwarding in nginx
-const globalUrl = window.location.origin
+//const globalUrl = window.location.origin
 
 const surfaceColor = "#27292D"
 const inputColor = "#383B40"

frontend/src/Workflows.js

@@ -86,7 +86,7 @@ const Workflows = (props) => {
 				if (isLoggedIn) {
 					alert.error("An error occurred while loading workflows")
 				} else {
-					window.location.pathname = "/login"
+					window.location = "/login"
 				}
 
 				return
@@ -493,15 +493,17 @@ const Workflows = (props) => {
 		}
 
 		var t = new Date(data.started_at*1000)
+		var jsonvalid = true
 		var showResult = data.result.trim()
-		if ((showResult.startsWith("{") && showResult.endsWith("}")) || (showResult.startsWith("[{") && showResult.endsWith("}]"))) {
+		showResult = replaceAll(showResult, " None", " \"None\"");
-			//showResult = <JSONPretty 
+		try {
-			//				id="json-pretty" 
+			JSON.parse(showResult)
-			//				theme={JSONPrettyMon} 
+		} catch (e) {
-			//				data={showResult}/>
+			jsonvalid = false
-			
+		}
-			showResult = replaceAll(showResult, " None", " \"None\"");
+
-			console.log(showResult)
+		console.log("VALID: ", jsonvalid)
+		if (jsonvalid) {
 			showResult = <ReactJson 
 				src={JSON.parse(showResult)} 
 				theme="solarized" 
@@ -531,13 +533,13 @@ const Workflows = (props) => {
 				<Grid container style={{margin: "10px 10px 10px 10px", flex: "1"}}>
 					<Grid style={{display: "flex", flexDirection: "column", width: "100%"}}>
 						<Grid item style={{flex: "1"}}>
-							<h4 style={{marginBottom: "0px", marginTop: "10px"}}><b>Status</b>: {data.status}</h4>
+							<h4 style={{marginBottom: "0px", marginTop: "10px"}}><b>Name</b>: {data.action.label}</h4>
 						</Grid>
 						<Grid item style={{flex: "1", justifyContent: "center"}}>
 							App: {data.action.app_name}, Version: {data.action.app_version}
 						</Grid>
 						<Grid item style={{flex: "1", justifyContent: "center"}}>
-							Action: {data.action.name}, Environment: {data.action.environment}
+							Action: {data.action.name}, Environment: {data.action.environment}, Status: {data.status}
 						</Grid>
 						<div style={{display: "flex", flex: "1"}}>
 							<Grid item style={{flex: "10", justifyContent: "center"}}>
@@ -578,25 +580,30 @@ const Workflows = (props) => {
 		var parsedArgument = selectedExecution.execution_argument
 		if (selectedExecution.execution_argument !== undefined && selectedExecution.execution_argument.length > 0) {
 			parsedArgument = replaceAll(parsedArgument, " None", " \"None\"");
+		}	
+
+		var arg = null
+		if (selectedExecution.execution_argument !== undefined && selectedExecution.execution_argument.length > 0) {
+			var jsonvalid = false
+			var showResult = selectedExecution.execution_argument.trim()
+			showResult = replaceAll(showResult, " None", " \"None\"");
+			try {
+				JSON.parse(showResult)
+			} catch (e) {
+				jsonvalid = false
+			}
+
+			arg = jsonvalid ? 
+				<ReactJson 
+					src={JSON.parse(showResult)} 
+					theme="solarized" 
+					collapsed={true}
+					displayDataTypes={false}
+					name={"Execution argument"}
+				/>
+			: showResult
 		}
 
-		const arg = selectedExecution.execution_argument !== undefined && selectedExecution.execution_argument.length > 0 ? 
-			<div>
-				Execution argument: {(parsedArgument.startsWith("{") && parsedArgument.endsWith("}")) || (parsedArgument.startsWith("[{") && parsedArgument.endsWith("}]")) ?
-					<div>
-						<ReactJson 
-							src={JSON.parse(parsedArgument)} 
-							theme="solarized" 
-							collapsed={true}
-							displayDataTypes={false}
-							name={"Execution argument"}
-						/>
-					</div>
-					:
-					selectedExecution.execution_argument
-					}
-			</div>
-			: null
 		/*
 		<div>
 			ID: {selectedExecution.execution_id}

functions/onprem/app_sdk/README.md

@@ -1,3 +1,10 @@
 # app_sdk
 This is the SDK used for apps to behave like they should. 
 To change it in the backend, upload it to Buckets/shuffler.appspot.com/generated_apps/baseline.
+
+## If you want to update apps.. Why doesn't this work every time..?:
+1. Write your code & check if runtime works
+2. Build app_base image
+3. docker rm $(docker ps -aq) # Remove all stopped containers
+4. Delete the specific app's Docker image (docker rmi frikky/shuffle:...)
+5. Rebuild the Docker image (click load in GUI?)

functions/onprem/app_sdk/app_base.py

@@ -1,5 +1,6 @@
 import os
 import sys
+import re
 import time 
 import json
 import logging
@@ -63,7 +64,7 @@ class AppBase:
 
         # Add async logger
         # self.console_logger.handlers[0].stream.set_execution_id()
-        self.logger.info("Before initial stream result")
+        #self.logger.info("Before initial stream result")
         try:
             ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result)
             self.logger.info("Workflow: %d" % ret.status_code)
@@ -72,7 +73,8 @@ class AppBase:
         except requests.exceptions.ConnectionError as e:
             print("Connectionerror: %s" %  e)
             return
-        self.logger.info("AFTER initial stream result")
+        #self.logger.info("AFTER initial stream result")
+        self.logger.info("THIS IS THE NEW UPDATE")
 
         # Verify whether there are any parameters with ACTION_RESULT required
         # If found, we get the full results list from backend
@@ -105,13 +107,102 @@ class AppBase:
 
         self.logger.info("AFTER FULLEXEC stream result")
 
+        # Takes a workflow execution as argument
+        # Returns a string if the result is single, or a list if it's a list
+        # Not implemented: lists
+        def get_json_value(execution_data, input_data):
+            parsersplit = input_data.split(".")
+            actionname = parsersplit[0][1:]
+            print(f"Actionname: {actionname}")
+        
+            # 1. Find the action
+            baseresult = ""
+            try: 
+                if actionname.lower() == "exec": 
+                    baseresult = execution_data["execution_argument"]
+                else:
+                    for result in execution_data["results"]:
+                        if result["action"]["label"].lower() == actionname.lower():
+                            baseresult = result["result"]
+                            break
+            except KeyError as error:
+                print(f"Error: {error}")
+
+            print(f"After first trycatch")
+        
+            # 2. Find the JSON data
+            if len(baseresult) == 0:
+                return ""
+        
+            if len(parsersplit) == 1:
+                return baseresult
+        
+            baseresult = baseresult.replace("\'", "\"")
+            basejson = {}
+            try:
+                basejson = json.loads(baseresult)
+            except json.decoder.JSONDecodeError as e:
+                return baseresult
+        
+            try:
+                for value in parsersplit[1:]:
+                    if value == "#":
+                        print("HANDLE RECURSIVE LOOP")
+                        pass
+                    else:
+                        if isinstance(basejson[value], str):
+                            print(f"LOADING STRING {basejson[value]} AS JSON?")
+                            try:
+                                parsedjson = json.loads(basejson[value])
+                            except json.decoder.JSONDecodeError as e:
+                                print("RETURNING BECAUSE {basejson[value]} is a normal string")  
+                                return basejson
+                        else:
+                            basejson = basejson[value]
+            except KeyError as e:
+                print(f"Keyerror: {e}")
+                return basejson
+            except IndexError as e:
+                print(f"Indexerror: {e}")
+                return basejson
+        
+            return basejson
+
+
+
         def parse_params(action, fullexecution, parameter):
             jsonparsevalue = "$."
+
+            # Regex to find all the things
+            if parameter["variant"] == "STATIC_VALUE":
+                data = parameter["value"]
+                self.logger.debug(f"\n\nHandle static data with JSON: {data}\n\n")
+
+                match = ".*([$]{1}(\w+\.?){1,})"
+                actualitem = re.findall(match, data, re.MULTILINE)
+                self.logger.info("PARSED: %s" % actualitem)
+                if len(actualitem) > 0:
+                    for replace in actualitem:
+                        try:
+                            to_be_replaced = replace[0]
+                        except IndexError:
+                            continue
+
+                        value = get_json_value(fullexecution, to_be_replaced)
+                        
+                        # Check if json inside string
+                        if isinstance(value, str) > 0:
+                            print(f"VALUE: {value}")
+
+                self.logger.info(f"CONVERT DATA FROM {parameter['value']} to {data}")
+                parameter["value"] = data
+
             if parameter["variant"] == "WORKFLOW_VARIABLE":
                 for item in fullexecution["workflow"]["workflow_variables"]:
                     if parameter["action_field"] == item["name"]:
                         parameter["value"] = item["value"]
                         break
+
             elif parameter["variant"] == "ACTION_RESULT":
                 # FIXME - calculate value based on action_field and $if prominent
                 # FIND THE RIGHT LABEL
@@ -258,6 +349,7 @@ class AppBase:
 
                     print(sourcevalue)
                     destinationvalue = condition["destination"]["value"]
+
                     if condition["destination"]["variant"]== "" or condition["destination"]["variant"]== "STATIC_VALUE":
                         condition["destination"]["variant"] = "STATIC_VALUE"
                     else:
@@ -398,7 +490,7 @@ class AppBase:
         action_result["completed_at"] = int(time.time())
 
         # I wonder if this actually works 
-        self.logger.info("Before last stream result")
+        #self.logger.info("Before last stream result")
         try:
             ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result)
             self.logger.info("Result: %d" % ret.status_code)

functions/onprem/app_sdk/update_dockerhub

@@ -1,4 +1,4 @@
 #!/bin/bash
 docker rmi frikky/shuffle:app_sdk
-docker build . -t frikky/shuffle:app_sdk
+docker build . -t frikky/shuffle:app_sdk --no-cache
 docker push frikky/shuffle:app_sdk