Merging fixes from cloud. Detailed changes TBA with next release

backend/app_sdk/app_base.py

@@ -345,6 +345,36 @@ class AppBase:
             
         return new_input
 
+    def prepare_response(self, request):
+        try:
+            parsedheaders = {}
+            for key, value in request.headers.items():
+                parsedheaders[key] = value
+
+            cookies = {}
+            if request.cookies:
+                for key, value in request.cookies.items():
+                    cookies[key] = value
+
+            
+            jsondata = request.text
+            try:
+                jsondata = json.loads(jsondata)
+            except:
+                pass
+
+            return json.dumps({
+                "success": True,
+                "status": request.status_code,
+                "url": request.url,
+                "headers": parsedheaders,
+                "body": jsondata,
+                "cookies":cookies,
+            })
+        except Exception as e:
+            print(f"[WARNING] Failed in request: {e}")
+            return request.text
+
     # FIXME: Add more info like logs in here.
     # Docker logs: https://forums.docker.com/t/docker-logs-inside-the-docker-container/68190/2
     def send_result(self, action_result, headers, stream_path):

backend/tests/files.sh

@@ -17,6 +17,6 @@
 
 
 #curl http://localhost:5001/api/v1/files/create -H "Authorization: Bearer 09627dcb-7e2a-4843-819b-417d268ff840" -d '{"filename": "rule2.yar", "org_id": "11f67b76-6051-4425-b0d6-be23daac6d12", "workflow_id": "global", "namespace": "yara"}'
-curl http://localhost:5001/api/v1/files/file_366ee8d2-1af6-4270-8639-213af30b4a29/upload -H "Authorization: Bearer 09627dcb-7e2a-4843-819b-417d268ff840" -F 'shuffle_file=@upload.sh'
+curl http://localhost:5002/api/v1/files/file_366ee8d2-1af6-4270-8639-213af30b4a29/upload -H "Authorization: Bearer 09627dcb-7e2a-4843-819b-417d268ff840" -F 'shuffle_file=@upload.sh'
 
 #curl http://localhost:5001/api/v1/files/namespaces/yara -H "Authorization: Bearer c5b4c827-65ec-47f4-9e8a-234cdba38959" --output rules.zip

backend/tests/upload.sh

@@ -1 +1,4 @@
-#
+# hello
+this is line 2
+and 3
+Is it a python problem?

frontend/src/components/ParsedAction.jsx

@@ -561,7 +561,20 @@ const ParsedAction = (props) => {
 		const calculateHelpertext = (input_data) => {
 			var helperText = ""
 			var looperText = ""
-			const found = input_data.match(/[$]{1}([a-zA-Z0-9_-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,}/g)
+			//const found = input_data.match(/[$]{1}([a-zA-Z0-9_-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,}/g)
+			var found = input_data.match(/[\\]{0,1}[$]{1}([a-zA-Z0-9_-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,}/g)
+
+			if (found !== null && found !== undefined) {
+				var new_occurences = []
+				for (var key in found) {
+					if (found[key][0] !== "\\") {
+						new_occurences.push(found[key])
+					}
+				}
+
+				console.log("New found: ", new_occurences))
+				found = new_occurences.valueOf()
+			}
 
 			if (found !== null) {
 				try {
@@ -571,7 +584,7 @@ const ParsedAction = (props) => {
 						if ((variableSplit.length-1) > 1) {
 							//console.log("Larger than 1: ", variableSplit)
 							if (looperText.length === 0) {
-								looperText += "PS: Double looping (.#) may cause problems."
+								looperText += "PS: Double looping (.#.#) may cause problems."
 							}
 						}
 

frontend/src/components/ShuffleCodeEditor.jsx

@@ -289,14 +289,23 @@ const CodeEditor = (props) => {
 			var current_code_line = code_lines[i]
 			// console.log(current_code_line)
 
-			var variable_occurence = current_code_line.match(/[$]{1}([a-zA-Z0-9_-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,}/g)
+			var variable_occurence = current_code_line.match(/[\\]{0,1}[$]{1}([a-zA-Z0-9_-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,}/g)
 
 			if (variable_occurence === null || variable_occurence === undefined) {
 				console.log("No variables found. Returning")
 				continue
-				//return
 			}
 
+			var new_occurences = []
+			for (var key in variable_occurence) {
+				if (variable_occurence[key][0] !== "\\") {
+					new_occurences.push(variable_occurence[key])
+				}
+			}
+
+			variable_occurence = new_occurences.valueOf()
+			console.log("Match2: ", variable_occurence)
+
 			// console.log(variable_occurence)
 			// console.log()
 
@@ -309,9 +318,10 @@ const CodeEditor = (props) => {
 			// 	console.log(actionlist[j].autocomplete);
 			// }
 
+			// Finds occurences of dollar signs
 			var dollar_occurence = []
-			for(var ch=0; ch<current_code_line.length; ch++){
+			for(var ch = 0; ch < current_code_line.length; ch++){
-				if(current_code_line[ch] === '$'){
+				if (current_code_line[ch] === '$' && (ch === 0 || current_code_line[ch-1] !== "\\")) {
 					dollar_occurence.push(ch)
 				}
 			}
@@ -319,7 +329,7 @@ const CodeEditor = (props) => {
 
 			var dollar_occurence_len = []
 			try{
-				for(var occ = 0; occ<variable_occurence.length; occ++){
+				for(var occ = 0; occ < variable_occurence.length; occ++){
 					dollar_occurence_len.push(variable_occurence[occ].length)
 				}
 			} catch (e) {}
@@ -328,6 +338,11 @@ const CodeEditor = (props) => {
 
 
 			try{
+				if (variable_occurence.length === 0) {
+					//value.markText({line:i, ch:0}, {line:i, ch:code_lines[i].length-1}, {"css": "background-color: #282828; border-radius: 0px; color: #b8bb26"})
+					//value.markText({line:i, ch:0}, {line:i, ch:code_lines[i].length-1}, {"css": "background-color: #; border-radius: 0px; color: inherit"})
+				}
+
 				// console.log(variable_occurence)
 				for (var occ = 0; occ < variable_occurence.length; occ++){
 					// value.markText({line:i, ch:dollar_occurence[occ]}, {line:i, ch:dollar_occurence_len[occ]+dollar_occurence[occ]}, {"css": "background-color: #8b8e26; border-radius: 4px; color: white"})
@@ -388,6 +403,7 @@ const CodeEditor = (props) => {
 
 	const expectedOutput = (input) => {
 		
+		//const found = input.match(/[$]{1}([a-zA-Z0-9_-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,}/g)
 		const found = input.match(/[$]{1}([a-zA-Z0-9_-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,}/g)
 		//if (found === null || found === undefined) {
 		//	console.log("No output found!")
@@ -441,7 +457,7 @@ const CodeEditor = (props) => {
 										console.log("ERR IN INPUT: ", e)
 									}
 
-									console.log("Got input: ", new_input, actionlist[k].example, typeof new_input)
+									//console.log("Got output for: ", fullpath, new_input, actionlist[k].example, typeof new_input)
 
 									if (typeof new_input === "object") {
 										new_input = JSON.stringify(new_input)
@@ -605,7 +621,12 @@ const CodeEditor = (props) => {
 			open={expansionModalOpen}
 			onClose={() => {
 				console.log("In closer")
-				changeActionParameterCodeMirror({target: {value: ""}}, fieldCount, localcodedata)
+
+				if (changeActionParameterCodeMirror !== undefined) {
+					changeActionParameterCodeMirror({target: {value: ""}}, fieldCount, localcodedata)
+				} else {
+					console.log("No action called changeActionParameterCodeMirror in code editor")
+				}
 				//setExpansionModalOpen(false)
 			}}
 			PaperComponent={PaperComponent}
@@ -1234,7 +1255,7 @@ const CodeEditor = (props) => {
 								<IconButton disabled={executing} color="primary" style={{border: `1px solid ${theme.palette.primary.main}`, marginLeft: 300, padding: 8}} variant="contained" onClick={() => {
 									executeSingleAction(expOutput)
 								}}>
-									<Tooltip title="Try it! This runs Shuffle Tools' 'repeat back to me' action with the expected output." placement="top">
+									<Tooltip title="Try it! This runs the Shuffle Tools 'repeat back to me' action with what you see in the expected output window." placement="top">
 										{executing ? <CircularProgress style={{height: 18, width: 18, }} /> : <PlayArrowIcon style={{height: 18, width: 18, }} /> }
 													 
 									</Tooltip>

frontend/src/views/Admin.jsx

@@ -3932,6 +3932,7 @@ const Admin = (props) => {
                 defaultValue={""}
                 autoFocus
               />}</div>
+
           <CodeEditor
             expansionModalOpen={openEditor}
             setExpansionModalOpen={setOpenEditor}
@@ -3941,6 +3942,7 @@ const Admin = (props) => {
             key = {fileContent} //https://reactjs.org/docs/reconciliation.html#recursing-on-children
             runUpdateText = {runUpdateText}
           />
+
           <Divider
             style={{
               marginTop: 20,

frontend/src/views/AngularWorkflow.jsx

@@ -254,7 +254,6 @@ const AngularWorkflow = (defaultprops) => {
   const yellow = "#FECC00";
   //const theme = useTheme();
 
-  const [bodyWidth, bodyHeight] = useWindowSize();
 
   var to_be_copied = "";
   const [firstrequest, setFirstrequest] = React.useState(true);
@@ -395,6 +394,9 @@ const AngularWorkflow = (defaultprops) => {
   const triggerEnvironments = isCloud ? ["cloud"] : ["onprem", "cloud"];
   const unloadText = "Are you sure you want to leave without saving (CTRL+S)?";
   const classes = useStyles();
+
+  const [bodyWidth, bodyHeight] = useWindowSize()
+	//console.log("Mobile: ", isMobile, bodyWidth, bodyHeight)
 	const cytoscapeWidth = isMobile ? bodyWidth - leftBarSize : bodyWidth - leftBarSize - 25
 
 
@@ -427,8 +429,6 @@ const AngularWorkflow = (defaultprops) => {
       })
       .then((responseJson) => {
 				if (responseJson.success === true) {
-					console.log("RESPONSE FOR APP: ", responseJson.reason);
-					
 					if (responseJson.reason !== undefined && responseJson.reason !== undefined && responseJson.reason.length > 0) {
 						if (!responseJson.reason.includes("404: Not Found") && responseJson.reason.length > 25) {
 							selectedApp.documentation = responseJson.reason
@@ -651,7 +651,6 @@ const AngularWorkflow = (defaultprops) => {
   };
 
   const setNewAppAuth = (appAuthData) => {
-    console.log("DAta: ", appAuthData);
     fetch(globalUrl + "/api/v1/apps/authentication", {
       method: "PUT",
       headers: {
@@ -1511,8 +1510,6 @@ const AngularWorkflow = (defaultprops) => {
           }
 
           if (cy !== undefined) {
-            console.log("NEW AUTH = reset cy's onnodeselect");
-
             // Remove the old listener for select, run with new one
             cy.removeListener("select");
             cy.on("select", "node", (e) => onNodeSelect(e, newauth));
@@ -14180,7 +14177,6 @@ const AngularWorkflow = (defaultprops) => {
     }
 
     const handleSubmitCheck = () => {
-      console.log("NEW AUTH: ", authenticationOption);
       if (authenticationOption.label.length === 0) {
         authenticationOption.label = `Auth for ${selectedApp.name}`;
       }
@@ -14244,7 +14240,6 @@ const AngularWorkflow = (defaultprops) => {
         });
       }
 
-      console.log("FIELDS: ", newFields);
       newAuthOption.fields = newFields;
       setNewAppAuth(newAuthOption);
 

frontend/src/views/AppCreator.jsx

@@ -670,7 +670,8 @@ const AppCreator = (defaultprops) => {
 					if (newaction.url !== undefined && newaction.url !== null && newaction.url.includes("_shuffle_replace_")) {
 						const regex = /_shuffle_replace_\d/i;
 						//console.log("NEW: ", 
-						newaction.url = newaction.url.replace(regex, "")
+						newaction.url = newaction.url.replaceAll(new RegExp(regex, 'g'), "")
+						console.log("Replaced: ", newaction.url) 
 					}
 
           // Finding category
@@ -679,6 +680,12 @@ const AppCreator = (defaultprops) => {
             var categoryindex = -1;
             // Stupid way of finding a category/grouping
             for (var key in pathsplit) {
+							if (pathsplit[key].includes("_shuffle_replace_")) {
+								const regex = /_shuffle_replace_\d/i;
+								//console.log("NEW: ", 
+								pathsplit[key] = pathsplit[key].replaceAll(new RegExp(regex, 'g'), "")
+							}
+
               if (
                 pathsplit[key].length > 0 &&
                 pathsplit[key] !== "v1" &&
@@ -1697,7 +1704,6 @@ const AppCreator = (defaultprops) => {
 			// Basic way to allow multiple of the same path 
 			var pathjoin = item.url+"_"+item.method.toLowerCase()
 			if (handledPaths.includes(pathjoin)) {
-				console.log("ALREADY INCLUDED: ", pathjoin)
 
 				// Max 100 of same lol
 				for (var i = 0; i < 100; i++) {
@@ -1708,7 +1714,6 @@ const AppCreator = (defaultprops) => {
 						continue
 					}
 
-					console.log("FOUND NEW: ", item.url)
 					break
 				}
 			}
@@ -1978,6 +1983,37 @@ const AppCreator = (defaultprops) => {
       	  item.body.length > 0
       	) {
 					console.log("GOT BODY: ", item.url, item.method)
+
+					// Replacing dollarsign insertions that aren't escaped
+					// This is to stop it from messing with systems in Shuffle.
+					// This MAY cause it to be a little weird in other systems however,
+					// but it's the only way we can properly support e.g. GraphQL
+					// with good examples
+					var newbody = ""
+					for (var key in item.body) {
+						if (item.body[key] === "$") {
+							if (key > 0) {
+								//console.log("Found: ", item.body[key-1])
+								if (item.body[key-1] !== "\\") {
+									newbody += "\\"
+								} 
+
+								newbody += item.body[key]
+							} else {
+								newbody += "\\"
+								newbody += item.body[key]
+							}
+							//newbody += item.body[key]
+						} else {
+							newbody += item.body[key]
+						}
+					}
+
+					if (newbody !== item.body) {
+						//console.log("New body: ", newbody)
+						item.body = newbody
+					}
+
 					//var pathjoin = item.url+"_"+item.method.toLowerCase()
 
       	  const required = false;
@@ -4656,6 +4692,12 @@ const AppCreator = (defaultprops) => {
         <div style={{ marginTop: 10, marginBottom: 10 }}>
           {projectCategories.map((tag, index) => {
             const newname = tag.charAt(0).toUpperCase() + tag.slice(1);
+
+						//var regex = /_shuffle_replace_\d/i;
+						////console.log("NEW: ", 
+						//newname = newname.replaceAll(regex, "")
+						//console.log("Replaced: ", newname) 
+
             return (
               <Chip
                 key={index}
@@ -4888,6 +4930,7 @@ const AppCreator = (defaultprops) => {
     imageUploadError.length > 0 ? (
       <div style={{ marginTop: 10 }}>Error: {imageUploadError}</div>
     ) : null;
+
   const imageUploadModalView = openImageModal ? (
     <Dialog
       open={openImageModal}

frontend/src/views/Apps.jsx

@@ -77,10 +77,12 @@ export const FindJsonPath = (path, inputdata) => {
 	var parsedValues = [];
 
 	if (inputdata === undefined || inputdata === null) {
+		console.log("Input is ", inputdata, ". Returning.")
 		return inputdata
 	}
 
 	if (typeof inputdata !== "object") {
+		console.log("Input is NOT an object. Returning.")
 		return inputdata
 	}
 
@@ -90,6 +92,7 @@ export const FindJsonPath = (path, inputdata) => {
 	}
 
 	if (keysplit.length === 0) {
+		console.log("Couldn't find key: length is 0 for keysplit.")
 		return inputdata
 	}
 
@@ -114,9 +117,11 @@ export const FindJsonPath = (path, inputdata) => {
 		}
 	}
 
+	var found = false
 	for (const [key, value] of Object.entries(inputdata)) {
-
+		const newkey = key.valueOf().toLowerCase().replaceAll(" ", "_")
-		if (key === keysplit[0]) {
+		if (key === keysplit[0] || newkey === keysplit[0]) {
+			found = true 
 
 			// Return if no more keys
 			// Else, dig deeper