Major changes to multi-file in Shuffle. Not finished yet.

backend/app_sdk/app_base.py

@@ -57,47 +57,68 @@ class AppBase:
     # - How can you download / stream a file? 
     # - Can you decide if you want a stream or the files directly?
     def get_file(self, value):
-        print("INSIDE GET_FILE")
-
         full_execution = self.full_execution
         org_id = full_execution["workflow"]["execution_org"]["id"]
 
         print("SHOULD GET FILES BASED ON ORG %s, workflow %s and value(s) %s" % (org_id, full_execution["workflow"]["id"], value))
 
-        get_path = "/api/v1/files/%s?execution_id=%s" % (value, full_execution["execution_id"])
+        if isinstance(value, list):
-        headers = {
+            print("IS LIST!")
-            "Content-Type": "application/json",     
+            #if len(value) == 1:
-            "Authorization": "Bearer %s" % self.authorization
+            #    value = value[0]
-        }
+        else:
+            value = [value]
 
-        ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers)
+        returns = []
-        print("RET1: %s" % ret1.text)
+        for item in value:
-        if ret1.status_code != 200:
+            print("VALUE: %s" % item)
+            if len(item) != 36:
+                print("Bad length for value")
+                continue
+                #return {
+                #    "filename": "",
+                #    "data": "",
+                #    "success": False,
+                #}
+
+            get_path = "/api/v1/files/%s?execution_id=%s" % (item, full_execution["execution_id"])
+            headers = {
+                "Content-Type": "application/json",     
+                "Authorization": "Bearer %s" % self.authorization
+            }
+
+            ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers)
+            print("RET1: %s" % ret1.text)
+            if ret1.status_code != 200:
+                returns.append({
+                    "filename": "",
+                    "data": "",
+                    "success": False,
+                })
+                continue
+
+            content_path = "/api/v1/files/%s/content?execution_id=%s" % (item, full_execution["execution_id"])
+            ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers)
+            print("Ret2: %s" % ret2.text)
+            if ret2.status_code == 200:
+                tmpdata = ret1.json()
+                returndata = {
+                    "success": True,
+                    "filename": tmpdata["filename"],
+                    "data": ret2.content,
+                }
+                returns.append(returndata)
+
+        if len(returns) == 0:
             return {
-                "filename": "",
-                "data": "",
                 "success": False,
+                "filename": "",
+                "data": b"",
             }
-
+        elif len(returns) == 1:
-        content_path = "/api/v1/files/%s/content?execution_id=%s" % (value, full_execution["execution_id"])
+            return returns[0]
-        ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers)
+        else:
-        print("Ret2: %s" % ret2.text)
+            return returns
-        if ret2.status_code == 200:
-            tmpdata = ret1.json()
-            returndata = {
-                "success": True,
-                "filename": tmpdata["filename"],
-                "data": ret2.content,
-            }
-            # open('facebook.ico', 'wb').write(r.content)
-
-            return returndata
-
-        return {
-            "success": False,
-            "filename": "",
-            "data": b"",
-        }
 
     # Sets files in the backend
     def set_files(self, infiles):
@@ -522,6 +543,7 @@ class AppBase:
                         # Magical way of returning which makes app sdk identify 
                         # it as multi execution
                         return newvalue, True
+
                     elif len(actualitem) > 0:
                         # FIXME: This is absolutely not perfect. 
                         print("In recursion v2: ", actualitem)
@@ -704,7 +726,7 @@ class AppBase:
                     print("SET DATA WRAPPER TO %s!" % parsersplit[-1])
                     parseditem = "${%s%s}$" % (parsersplit[-1], json.dumps(data))
 
-            print("Before last return")
+            print("Before last return with %s" % appendresult)
             return str(parseditem)+str(appendresult), is_loop
 
         # Parses parameters sent to it and returns whether it did it successfully with the values found
@@ -1205,15 +1227,22 @@ class AppBase:
                                 if listitem in filteredlist:
                                     continue
 
-                                filteredlist.append(listitem)
+                                # FIXME: Subsub required?. Recursion! 
+                                # Basically multiply what we have with the outer loop?
+                                # 
+                                if isinstance(listitem, list):
+                                    for subitem in listitem:
+                                        filteredlist.append(subitem)
+                                else:
+                                    filteredlist.append(listitem)
 
                             #print("New list length: %d" % len(filteredlist))
                             if len(filteredlist) > 1:
                                 print("Calculating new multi-loop length with %d lists" % len(filteredlist))
                                 tmplength = 1
                                 for innerlist in filteredlist:
-                                    print("List length: %d. %d*%d" % (len(innerlist), len(innerlist), tmplength))
                                     tmplength = len(innerlist)*tmplength
+                                    print("List length: %d. %d*%d" % (tmplength, len(innerlist), tmplength))
 
                                 minlength = tmplength
 
@@ -1257,6 +1286,7 @@ class AppBase:
                                 except ValueError:
                                     result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres)
                                     print("Can't handle type %s value from function" % (type(newres)))
+
                             print("POST NEWRES RESULT: ", result)
                         else:
                             print("APP_SDK DONE: Starting MULTI execution (length: %d) with values %s" % (minlength, multi_parameters))
@@ -1286,7 +1316,6 @@ class AppBase:
                                         if isinstance(value, list):
                                             try:
                                                 newvalue = value[i]
-                                                print("NEWVALUE: %s" % newvalue)
                                             except IndexError:
                                                 pass
 

backend/go-app/files.go

@@ -1,7 +1,7 @@
 package main
 
 /*
-	Handles files within Workflows.
+	Handles files within Workflows.of Shuffle
 */
 
 import (

backend/go-app/main.go

@@ -6813,7 +6813,7 @@ func remoteOrgJobHandler(org Org, interval int) error {
 
 	respBody, err := ioutil.ReadAll(newresp.Body)
 	if err != nil {
-		log.Printf("Failed body read in job sync: %s", err)
+		log.Printf("[ERROR] Failed body read in job sync: %s", err)
 		return err
 	}
 
@@ -6821,7 +6821,7 @@ func remoteOrgJobHandler(org Org, interval int) error {
 
 	err = remoteOrgJobController(org, respBody)
 	if err != nil {
-		log.Printf("Failed job controller run: %s", err)
+		log.Printf("[ERROR] Failed job controller run for %s: %s", respBody, err)
 		return err
 	}
 	return nil

backend/go-app/walkoff.go

@@ -5765,7 +5765,7 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) {
 	}
 
 	// Query for the specifci workflowId
-	q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(20)
+	q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(30)
 	var workflowExecutions []WorkflowExecution
 	_, err = dbclient.GetAll(ctx, q, &workflowExecutions)
 	if err != nil {

docker-compose.yml

@@ -2,7 +2,7 @@ version: '3'
 services:
   frontend:
     #build: ./frontend
-    image: ghcr.io/frikky/shuffle-frontend:0.8.0
+    image: ghcr.io/frikky/shuffle-frontend:0.8.3
     container_name: shuffle-frontend
     hostname: shuffle-frontend
     ports:
@@ -17,7 +17,7 @@ services:
       - backend
   backend:
     build: ./backend
-    image: ghcr.io/frikky/shuffle-backend:0.8.11
+    image: ghcr.io/frikky/shuffle-backend:0.8.3
     container_name: shuffle-backend
     hostname: ${BACKEND_HOSTNAME}
     # Here for debugging:

frontend/src/views/Apps.jsx

@@ -116,7 +116,7 @@ const Apps = (props) => {
 	const [apps, setApps] = React.useState([])
 	const [filteredApps, setFilteredApps] = React.useState([])
 	const [validation, setValidation] = React.useState(false)
-	const [isLoading, setIsLoading] = React.useState(false)
+	const [isLoading, setIsLoading] = React.useState(true)
 	const [appSearchLoading, setAppSearchLoading] = React.useState(false)
 	const [selectedAction, setSelectedAction] = React.useState({})
 	const [searchBackend, setSearchBackend] = React.useState(false)
@@ -207,6 +207,7 @@ const Apps = (props) => {
 	  			credentials: "include",
     		})
 		.then((response) => {
+			setIsLoading(false)
 			if (response.status !== 200) {
 				console.log("Status not 200 for apps :O!")
 			}
@@ -231,6 +232,7 @@ const Apps = (props) => {
     })
 		.catch(error => {
 			alert.error(error.toString())
+			setIsLoading(false)
 		});
 	}
 
@@ -742,7 +744,7 @@ const Apps = (props) => {
 						<Divider style={{height: 1, backgroundColor: dividerColor, marginTop: 20, marginBottom: 20}} />
 						<div style={{}}>
 							<Button
-								variant="text"
+								variant="contained"
 								component="label"
 								color="primary"
 								style={{marginRight: 10, }}
@@ -932,11 +934,14 @@ const Apps = (props) => {
 								}
 							</Paper>
 						: 
-						<Paper square style={uploadViewPaperStyle}>
+						isLoading ? 
-							<h4 style={{margin: 10}}>
+							<CircularProgress style={{width: 40, height: 40, margin: "auto"}}/>
-								No apps have been created, uploaded or downloaded yet. Click "Load existing apps" above to get the baseline. This may take a while as its building docker images.
+							:
-							</h4>
+							<Paper square style={uploadViewPaperStyle}>
-						</Paper>
+								<h4 style={{margin: 10}}>
+									No apps have been created, uploaded or downloaded yet. Click "Load existing apps" above to get the baseline. This may take a while as its building docker images.
+								</h4>
+							</Paper>
 						}
 					</div>
 				</div>

frontend/src/views/Workflows.jsx

@@ -74,6 +74,7 @@ const Workflows = (props) => {
 	const [update, setUpdate] = React.useState("test");
 	const [deleteModalOpen, setDeleteModalOpen] = React.useState(false);
 	const [editingWorkflow, setEditingWorkflow] = React.useState({})
+	const [executionLoading, setExecutionLoading] = React.useState(false)
 	const { start, stop } = useInterval({
 	  	duration: 5000,
 	  	startImmediate: false,
@@ -245,6 +246,7 @@ const Workflows = (props) => {
 	}
 
 	const getWorkflowExecution = (id) => {
+		setExecutionLoading(true)
 		fetch(globalUrl+"/api/v1/workflows/"+id+"/executions", {
     	  method: 'GET',
 				headers: {
@@ -254,6 +256,7 @@ const Workflows = (props) => {
 	  			credentials: "include",
     		})
 		.then((response) => {
+			setExecutionLoading(false)
 			if (response.status !== 200) {
 				console.log("Status not 200 for WORKFLOW EXECUTION :O!")
 			}
@@ -275,6 +278,7 @@ const Workflows = (props) => {
 			}
 		})
 		.catch(error => {
+			setExecutionLoading(false)
 			alert.error(error.toString())
 		});
 	}
@@ -858,10 +862,17 @@ const Workflows = (props) => {
 				</div> 
 			)
 		}
+
 		return (
-			<h4>
+			executionLoading ?
-				There are no executiondetails yet. Click "execute" to run your first one.
+				<div style={{marginTop: 25, textAlign: "center"}}>
-			</h4>
+					<CircularProgress />
+				</div>
+				: 
+				<h4>
+					There are no executiondetails yet. Click "execute" to run your first one.
+				</h4>
+			
 		)
 	}
 
@@ -880,9 +891,14 @@ const Workflows = (props) => {
 			)
 		} 
 		return (
-			<h4>
+			executionLoading ?
-				There are no executions for this workflow yet
+				<div style={{marginTop: 25, textAlign: "center"}}>
-			</h4>
+					<CircularProgress />
+				</div>
+				:
+				<h4>
+					There are no executions for this workflow yet
+				</h4>
 		)
 	}
 
@@ -1111,107 +1127,113 @@ const Workflows = (props) => {
 			workflowViewStyle.display = "none"
 		}
 
-		const workflowView = workflows.length > 0 ? 
+		const WorkflowView = () => {
-		<div style={viewStyle}>	
+			if (workflows.length === 0) {
-			<div style={workflowViewStyle}>
+				return (
-				<div style={{display: "flex"}}>
+					<div style={emptyWorkflowStyle}>	
-					<div style={{flex: "4"}}>
+						<Paper style={boxStyle}>
-						<h2>Workflows</h2> 
+							<div>
+								<h2>Welcome to Shuffle</h2>
+							</div>
+							<div>
+								<p>
+									<b>Shuffle</b> is a flexible, easy to use, automation platform allowing users to integrate their services and devices freely. It's made to significantly reduce the amount of manual labor, and is focused on security applications. <a href="/docs/about" style={{textDecoration: "none", color: "#f85a3e"}}>Click here to learn more.</a>
+								</p>
+							</div>
+							<div>
+								If you want to jump straight into it, click here to create your first workflow: 
+							</div>
+							<div>
+								<Button color="primary" style={{marginTop: "20px",}} variant="outlined" onClick={() => setModalOpen(true)}>New workflow</Button> 				
+							</div>
+						</Paper>
 					</div>
-					<div style={{marginTop: 20}}>
+				)
-					 	<Tooltip color="primary" title={"Create new workflow"} placement="top">
+			}
-							<Button color="primary" style={{}} variant="text" onClick={() => setModalOpen(true)}><AddIcon /></Button> 				
-						</Tooltip>
-						{/*
-					 	<Tooltip color="primary" title={"Import workflows"} placement="top">
-							<Button color="primary" style={{}} variant="text" onClick={() => upload.click()}>
-								<PublishIcon />
-							</Button> 				
-						</Tooltip>
-						*/}
-						<Tooltip color="primary" title={`Download ALL workflows (${workflows.length})`} placement="top">
-							<Button color="primary" style={{}} variant="text" onClick={() => {
-								exportAllWorkflows()
-							}}>
-								<GetAppIcon />
-							</Button> 				
-						</Tooltip>
-					 	<Tooltip color="primary" title={"Import workflows"} placement="top">
-							<Button color="primary" style={{}} variant="text" onClick={() => setLoadWorkflowsModalOpen(true)}>
-								<CloudDownloadIcon />
-							</Button> 				
-						</Tooltip>
-						<input hidden type="file" multiple="multiple" ref={(ref) => upload = ref} onChange={importFiles} />
-					</div>
-				</div>
-				<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
 
-				<div style={scrollStyle}>
+			return (
-					{workflows.map((data, index) => {
+			<div style={viewStyle}>	
-						return (
+				<div style={workflowViewStyle}>
-							<WorkflowPaper key={index} data={data} />
+					<div style={{display: "flex"}}>
-						)
+						<div style={{flex: "4"}}>
-					})}
+							<h2>Workflows</h2> 
-				</div>
+						</div>
-			</div>
+						<div style={{marginTop: 20}}>
-			<div style={{flex: viewSize.executionsView, marginLeft: "10px", marginRight: "10px"}}>
+						 	<Tooltip color="primary" title={"Create new workflow"} placement="top">
-				<div style={{display: "flex"}}>
+								<Button color="primary" style={{}} variant="text" onClick={() => setModalOpen(true)}><AddIcon /></Button> 				
-					<div style={{flex: "10"}}>
+							</Tooltip>
-						<h2>Executions: {selectedWorkflow.name}</h2> 
+							{/*
+						 	<Tooltip color="primary" title={"Import workflows"} placement="top">
+								<Button color="primary" style={{}} variant="text" onClick={() => upload.click()}>
+									<PublishIcon />
+								</Button> 				
+							</Tooltip>
+							*/}
+							<Tooltip color="primary" title={`Download ALL workflows (${workflows.length})`} placement="top">
+								<Button color="primary" style={{}} variant="text" onClick={() => {
+									exportAllWorkflows()
+								}}>
+									<GetAppIcon />
+								</Button> 				
+							</Tooltip>
+						 	<Tooltip color="primary" title={"Import workflows"} placement="top">
+								<Button color="primary" style={{}} variant="text" onClick={() => setLoadWorkflowsModalOpen(true)}>
+									<CloudDownloadIcon />
+								</Button> 				
+							</Tooltip>
+							<input hidden type="file" multiple="multiple" ref={(ref) => upload = ref} onChange={importFiles} />
+						</div>
 					</div>
-					<div style={{flex: "1"}}>
+					<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
-						<Button color="primary" style={{marginTop: "20px"}} variant="text" onClick={() => {
-								alert.info("Refreshing executions"); 
-								getWorkflowExecution(selectedWorkflow.id)
-							}}>
-							<CachedIcon />
-						</Button> 				
-					</div>
-				</div>
-				<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
-				<div style={scrollStyle}>
-					<ExecutionsView />
-				</div>
-			</div>
-			<div style={{flex: viewSize.executionResults, marginLeft: "10px", marginRight: "10px", minWidth: "33%"}}>
-				<div style={{display: "flex"}}>
-					<div style={{flex: "1"}}>
-						<h2>Execution Timeline</h2>
-					</div>
-					<div style={{flex: 1}}>
-				    	<FormControlLabel
-							style={{color: "white", marginBottom: "0px", marginTop: "10px"}}
-							label={<div style={{color: "white"}}>Collapse results</div>}
-							control={<Switch checked={collapseJson} onChange={() => {setCollapseJson(!collapseJson)}} />}
-						/>
-					</div>
-				</div>
-				<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
-				<div style={scrollStyle}>
-					<ExecutionDetails />
-				</div>
-			</div>
-		</div>
-		: 
-		<div style={emptyWorkflowStyle}>	
-			<Paper style={boxStyle}>
-				<div>
-					<h2>Welcome to Shuffle</h2>
-				</div>
-				<div>
-					<p>
-						<b>Shuffle</b> is a flexible, easy to use, automation platform allowing users to integrate their services and devices freely. It's made to significantly reduce the amount of manual labor, and is focused on security applications. <a href="/docs/about" style={{textDecoration: "none", color: "#f85a3e"}}>Click here to learn more.</a>
-					</p>
-				</div>
-				<div>
-					If you want to jump straight into it, click here to create your first workflow: 
-				</div>
-				<div>
-					<Button color="primary" style={{marginTop: "20px",}} variant="outlined" onClick={() => setModalOpen(true)}>New workflow</Button> 				
-				</div>
-			</Paper>
-		</div>
 
+					<div style={scrollStyle}>
+						{workflows.map((data, index) => {
+							return (
+								<WorkflowPaper key={index} data={data} />
+							)
+						})}
+					</div>
+				</div>
+				<div style={{flex: viewSize.executionsView, marginLeft: "10px", marginRight: "10px"}}>
+					<div style={{display: "flex"}}>
+						<div style={{flex: "10"}}>
+							<h2>Executions: {selectedWorkflow.name}</h2> 
+						</div>
+						<div style={{flex: "1"}}>
+							<Button color="primary" style={{marginTop: "20px"}} variant="text" onClick={() => {
+									alert.info("Refreshing executions"); 
+									getWorkflowExecution(selectedWorkflow.id)
+								}}>
+								<CachedIcon />
+							</Button> 				
+						</div>
+					</div>
+					<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
+					<div style={scrollStyle}>
+						<ExecutionsView />
+					</div>
+				</div>
+				<div style={{flex: viewSize.executionResults, marginLeft: "10px", marginRight: "10px", minWidth: "33%"}}>
+					<div style={{display: "flex"}}>
+						<div style={{flex: "1"}}>
+							<h2>Execution Timeline</h2>
+						</div>
+						<div style={{flex: 1}}>
+					    	<FormControlLabel
+								style={{color: "white", marginBottom: "0px", marginTop: "10px"}}
+								label={<div style={{color: "white"}}>Collapse results</div>}
+								control={<Switch checked={collapseJson} onChange={() => {setCollapseJson(!collapseJson)}} />}
+							/>
+						</div>
+					</div>
+					<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
+					<div style={scrollStyle}>
+						<ExecutionDetails />
+					</div>
+				</div>
+			</div>
+		)
+	}
 
 	const importWorkflowsFromUrl = (url) => {
 		console.log("IMPORT WORKFLOWS FROM ", downloadUrl)
@@ -1384,13 +1406,17 @@ const Workflows = (props) => {
 
 	const loadedCheck = isLoaded && isLoggedIn && workflowDone ? 
 		<div>
-			{workflowView}
+			<WorkflowView />
 			{modalView}
 			{deleteModal}
 			{workflowDownloadModalOpen}
 		</div>
 		:
-		<div>
+		<div style={{paddingTop: 250, width: 250, margin: "auto", textAlign: "center"}}>
+			<CircularProgress />
+			<Typography>
+				Loading Workflows
+			</Typography>
 		</div>