#141: Fixed more general use-case issues for files

backend/app_sdk/app_base.py

@@ -148,6 +148,7 @@ class AppBase:
             print("")
 
 
+        self.full_execution = fullexecution
         self.logger.info("AFTER FULLEXEC stream result")
 
         # Gets the value at the parenthesis level you want
@@ -478,7 +479,7 @@ class AppBase:
 
             #Actionname: Start_node
 
-            print(f"Actionname: {actionname_lower}")
+            print(f"\nActionname: {actionname_lower}")
 
             # 1. Find the action
             baseresult = ""
@@ -549,22 +550,30 @@ class AppBase:
             except KeyError as error:
                 print(f"KeyError in JSON: {error}")
         
-            print(f"After first trycatch")
+            print(f"After first trycatch. Baseresult: ", baseresult)
         
             # 2. Find the JSON data
             if len(baseresult) == 0:
                 return ""+appendresult, False
         
+            print("After second return")
             if len(parsersplit) == 1:
                 return str(baseresult)+str(appendresult), False
         
             baseresult = baseresult.replace("\'", "\"")
+            baseresult = baseresult.replace(" True,", " true,")
+            baseresult = baseresult.replace(" False", " false,")
+
+            print("After third parser return - Formatted: ", baseresult)
             basejson = {}
             try:
                 basejson = json.loads(baseresult)
             except json.decoder.JSONDecodeError as e:
+                print("Parser issue with JSON: %s" % e)
                 return str(baseresult)+str(appendresult), False
 
+            print("After fourth parser return as JSON")
+        
             data, is_loop = recurse_json(basejson, parsersplit[1:])
             parseditem = data
             if is_loop:
@@ -577,6 +586,7 @@ class AppBase:
                     print("SET DATA WRAPPER TO %s!" % parsersplit[-1])
                     parseditem = "${%s%s}$" % (parsersplit[-1], json.dumps(data))
 
+            print("Before last return")
             return str(parseditem)+str(appendresult), is_loop
 
         # Parses parameters sent to it and returns whether it did it successfully with the values found
@@ -854,8 +864,9 @@ class AppBase:
                 returndata = {
                     "success": True,
                     "filename": tmpdata["filename"],
-                    "data": ret2.text,
+                    "data": ret2.content,
                 }
+                # open('facebook.ico', 'wb').write(r.content)
 
                 return returndata
 
@@ -1093,6 +1104,20 @@ class AppBase:
                                             #replacement = parse_wrapper_start(replacement)
                                             tmpitem = tmpitem.replace(key, replacement, -1)
 
+
+                                        # This code handles files.
+                                        isfile = False
+                                        try:
+                                            if parameter["schema"]["type"] == "file" and len(value) > 0:
+                                                print("SHOULD HANDLE FILE IN MULTI. Get based on value %s" % parameter["value"]) 
+                                                file_value = get_files(fullexecution, tmpitem)
+                                                print("FILE VALUE FOR VAL %s: %s" % (tmpitem, file_value))
+                                                resultarray.append(file_value)
+
+                                        except KeyError as e:
+                                            print("SCHEMA ERROR IN FILE HANDLING: %s" % e)
+
+                                        if not isfile:
                                             resultarray.append(tmpitem)
 
                                     # With this parameter ready, add it to... a greater list of parameters. Rofl
@@ -1114,9 +1139,9 @@ class AppBase:
                                 # This code handles files.
                                 try:
                                     if parameter["schema"]["type"] == "file" and len(value) > 0:
-                                        print("SHOULD HANDLE FILE. Get based on value %s" % parameter["value"]) 
+                                        print("\n SHOULD HANDLE FILE. Get based on value %s. <--- is this a valid ID?" % parameter["value"]) 
                                         file_value = get_files(fullexecution, value)
-                                        print("FILE VALUE: %s" % file_value)
+                                        print("FILE VALUE: %s \n" % file_value)
 
                                         params[parameter["name"]] = file_value 
                                         multi_parameters[parameter["name"]] = file_value 

backend/go-app/files.go

@@ -37,6 +37,7 @@ type File struct {
 	WorkflowId   string   `json:"workflow_id" datastore:"workflow_id"`
 	Workflows    []string `json:"workflows" datastore:"workflows"`
 	DownloadPath string   `json:"download_path" datastore:"download_path"`
+	Md5sum       string   `json:"md5_sum" datastore:"md5_sum"`
 }
 
 var basepath = os.Getenv("SHUFFLE_FILE_LOCATION")
@@ -66,11 +67,14 @@ func fileAuthentication(request *http.Request) (string, error) {
 		// This is annoying af and is done because of maxlength lol
 		newApikey := apikeyCheck[1]
 		if newApikey != workflowExecution.Authorization {
-			log.Printf("[ERROR] Bad apikey for execution %s. %s vs %s", executionId[0], apikey, workflowExecution.Authorization)
+			//log.Printf("[ERROR] Bad apikey for execution %s. %s vs %s", executionId[0], apikey, workflowExecution.Authorization)
+			log.Printf("[ERROR] Bad apikey for execution %s.", executionId[0])
+			//%s vs %s", executionId[0], apikey, workflowExecution.Authorization)
 			return "", errors.New("Bad authorization key")
 		}
 
-		log.Printf("[INFO] Authorization is correct for execution %s! %s vs %s. Setting Org", executionId, apikey, workflowExecution.Authorization)
+		log.Printf("[INFO] Authorization is correct for execution %s!", executionId[0])
+		//%s vs %s. Setting Org", executionId, apikey, workflowExecution.Authorization)
 		if len(workflowExecution.ExecutionOrg) > 0 {
 			return workflowExecution.ExecutionOrg, nil
 		} else if len(workflowExecution.Workflow.ExecutingOrg.Id) > 0 {
@@ -115,7 +119,14 @@ func handleGetFileMeta(resp http.ResponseWriter, request *http.Request) {
 		fileId = strings.Split(fileId, "?")[0]
 	}
 
-	log.Printf("\n\n[INFO] User is trying to delete file %s\n\n", fileId)
+	if len(fileId) != 36 {
+		log.Printf("Bad format for fileId %s", fileId)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`))
+		return
+	}
+
+	log.Printf("\n\n[INFO] User is trying to GET File Meta for %s\n\n", fileId)
 
 	// 1. Check user directly
 	// 2. Check workflow execution authorization
@@ -136,7 +147,7 @@ func handleGetFileMeta(resp http.ResponseWriter, request *http.Request) {
 	}
 
 	// 1. Verify if the user has access to the file: org_id and workflow
-	log.Printf("[INFO] Should DELETE file %s if user has access", fileId)
+	log.Printf("[INFO] Should GET FILE META for %s if user has access", fileId)
 	ctx := context.Background()
 	file, err := getFile(ctx, fileId)
 	if err != nil {
@@ -172,7 +183,7 @@ func handleGetFileMeta(resp http.ResponseWriter, request *http.Request) {
 		return
 	}
 
-	log.Printf("[INFO] Successfully deleted file %s", fileId)
+	log.Printf("[INFO] Successfully got file meta for %s", fileId)
 	resp.WriteHeader(200)
 	resp.Write([]byte(newBody))
 }
@@ -200,6 +211,13 @@ func handleDeleteFile(resp http.ResponseWriter, request *http.Request) {
 		fileId = strings.Split(fileId, "?")[0]
 	}
 
+	if len(fileId) != 36 {
+		log.Printf("Bad format for fileId %s", fileId)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`))
+		return
+	}
+
 	log.Printf("\n\n[INFO] User is trying to delete file %s\n\n", fileId)
 
 	// 1. Check user directly
@@ -318,7 +336,14 @@ func handleGetFileContent(resp http.ResponseWriter, request *http.Request) {
 		fileId = location[4]
 	}
 
-	log.Printf("\n\nUser is trying to get file %s\n\n", fileId)
+	if len(fileId) != 36 {
+		log.Printf("Bad format for fileId %s", fileId)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`))
+		return
+	}
+
+	log.Printf("\n\nUser is trying to download file %s\n\n", fileId)
 
 	// 1. Check user directly
 	// 2. Check workflow execution authorization
@@ -346,11 +371,11 @@ func handleGetFileContent(resp http.ResponseWriter, request *http.Request) {
 	}
 
 	// 1. Verify if the user has access to the file: org_id and workflow
-	log.Printf("Should get file %s", fileId)
+	log.Printf("[INFO] Should get file %s", fileId)
 	ctx := context.Background()
 	file, err := getFile(ctx, fileId)
 	if err != nil {
-		log.Printf("File %s not found: %s", fileId, err)
+		log.Printf("[ERROR] File %s not found: %s", fileId, err)
 		resp.WriteHeader(401)
 		resp.Write([]byte(`{"success": false}`))
 		return
@@ -376,7 +401,7 @@ func handleGetFileContent(resp http.ResponseWriter, request *http.Request) {
 	}
 
 	if file.Status != "active" {
-		log.Printf("File status isn't active. Can't continue.")
+		log.Printf("[ERROR] File status isn't active, but %s. Can't continue.", file.Status)
 		resp.WriteHeader(401)
 		resp.Write([]byte(`{"success": false, "reason": "The file isn't ready to be downloaded yet. Status required: active"}`))
 		return
@@ -384,7 +409,7 @@ func handleGetFileContent(resp http.ResponseWriter, request *http.Request) {
 
 	// Fixme: More auth: org and workflow!
 	downloadPath := file.DownloadPath
-	log.Printf("Downloadpath: %s", downloadPath)
+	log.Printf("[INFO] Downloadpath: %s", downloadPath)
 	Openfile, err := os.Open(downloadPath)
 	defer Openfile.Close() //Close after function return
 	if err != nil {
@@ -438,6 +463,13 @@ func handleUploadFile(resp http.ResponseWriter, request *http.Request) {
 		fileId = location[4]
 	}
 
+	if len(fileId) != 36 {
+		log.Printf("Bad format for fileId %s", fileId)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`))
+		return
+	}
+
 	// 1. Check user directly
 	// 2. Check workflow execution authorization
 	user, err := handleApiAuthentication(resp, request)
@@ -494,7 +526,6 @@ func handleUploadFile(resp http.ResponseWriter, request *http.Request) {
 	}
 
 	request.ParseMultipartForm(32 << 20)
-	//var buf bytes.Buffer
 	parsedFile, _, err := request.FormFile("shuffle_file")
 	if err != nil {
 		log.Printf("[ERROR] Couldn't upload file: %s", err)
@@ -513,16 +544,23 @@ func handleUploadFile(resp http.ResponseWriter, request *http.Request) {
 		return
 	}
 
-	//io.Copy(&buf, parsedFile)
+	// Can be used for validation files for change
-	//contents := buf.String()
+	/*
-	//buf.Reset()
+		var buf bytes.Buffer
+		io.Copy(&buf, parsedFile)
+		contents := buf.Bytes()
+		md5 := md5sum(contents)
+		buf.Reset()
+	*/
+	md5 := ""
+
 	f, err := os.OpenFile(file.DownloadPath, os.O_WRONLY|os.O_CREATE, os.ModePerm)
 	if err != nil {
 		// Rolling back file
 		file.Status = "created"
 		setFile(ctx, *file)
 
-		log.Printf("Failed creating file: %s", err)
+		log.Printf("[ERROR] Failed uploading and creating file: %s", err)
 		resp.WriteHeader(500)
 		resp.Write([]byte(`{"success": false}`))
 		return
@@ -532,7 +570,9 @@ func handleUploadFile(resp http.ResponseWriter, request *http.Request) {
 	io.Copy(f, parsedFile)
 
 	// FIXME: Set this one to 200 anyway? Can't download file then tho..
+	log.Printf("[INFO] MD5 for file %s is %s", file.Filename, md5)
 	file.Status = "active"
+	file.Md5sum = md5
 	err = setFile(ctx, *file)
 	if err != nil {
 		log.Printf("[ERROR] Failed setting file back to active")

backend/go-app/main.go

@@ -7119,6 +7119,7 @@ func runInit(ctx context.Context) {
 			}
 		}
 
+		/*
 			fileq := datastore.NewQuery("Files").Limit(1)
 			count, err := dbclient.Count(ctx, fileq)
 			log.Printf("FILECOUNT: %d", count)
@@ -7150,6 +7151,7 @@ func runInit(ctx context.Context) {
 					log.Printf("Created file %s in init", newFile.DownloadPath)
 				}
 			}
+		*/
 
 		var allworkflowapps []AppAuthenticationStorage
 		q = datastore.NewQuery("workflowappauth")
@@ -8045,10 +8047,10 @@ func initHandlers() {
 	// https://developer.box.com/reference/get-files-id-content/
 	// 1. Creating the "get file" option. Make it possible to run this in the frontend.
 	r.HandleFunc("/api/v1/files/{fileId}/content", handleGetFileContent).Methods("GET", "OPTIONS")
-	r.HandleFunc("/api/v1/files/{fileId}", handleDeleteFile).Methods("DELETE", "OPTIONS")
 	r.HandleFunc("/api/v1/files/create", handleCreateFile).Methods("POST", "OPTIONS")
 	r.HandleFunc("/api/v1/files/{fileId}/upload", handleUploadFile).Methods("POST", "OPTIONS")
 	r.HandleFunc("/api/v1/files/{fileId}", handleGetFileMeta).Methods("GET", "OPTIONS")
+	r.HandleFunc("/api/v1/files/{fileId}", handleDeleteFile).Methods("DELETE", "OPTIONS")
 
 	http.Handle("/", r)
 }

backend/go-app/walkoff.go

@@ -2795,7 +2795,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf
 					continue
 				}
 
-				log.Printf("Should set %s to SKIPPED as it's NOT a childnode of the startnode.", action.ID)
+				log.Printf("[WARNING] Set %s to SKIPPED as it's NOT a childnode of the startnode.", action.ID)
 				defaultResults = append(defaultResults, ActionResult{
 					Action:        action,
 					ExecutionId:   workflowExecution.ExecutionId,

docker-compose.yml

@@ -1,7 +1,7 @@
 version: '3'
 services:
   frontend:
-    build: ./frontend
+    #build: ./frontend
     image: ghcr.io/frikky/shuffle-frontend:0.8.0
     container_name: shuffle-frontend
     hostname: shuffle-frontend
@@ -16,8 +16,8 @@ services:
     depends_on:
       - backend
   backend:
-    #build: ./backend
+    build: ./backend
-    image: ghcr.io/frikky/shuffle-backend:0.8.1
+    image: ghcr.io/frikky/shuffle-backend:0.8.11
     container_name: shuffle-backend
     hostname: ${BACKEND_HOSTNAME}
     # Here for debugging:

frontend/src/views/AngularWorkflow.jsx

@@ -5354,7 +5354,7 @@ const AngularWorkflow = (props) => {
 		return null 
 	}
 
-	const cytoscapeViewWidths = 700
+	const cytoscapeViewWidths = 750
 	const bottomBarStyle = {
 		position: "fixed", 
 		right: 20, 
@@ -5412,6 +5412,62 @@ const AngularWorkflow = (props) => {
 		return null
 	}
 
+	const FileMenu = () => {
+		const [newAnchor, setNewAnchor] = React.useState(null);
+		const [showShuffleMenu, setShowShuffleMenu] = React.useState(false)
+
+		{ /*const [showShuffleMenu, setShowShuffleMenu] = React.useState(true) */}
+		return (
+			<div style={{"display": "inline-block"}}>
+				<Menu
+					id="long-menu"
+					anchorEl={newAnchor}
+					open={showShuffleMenu}
+					onClose={() => {
+						setShowShuffleMenu(false)
+					}}
+				>
+					<div style={{margin: 15, color: "white", maxWidth: 250, minWidth: 250, }}>
+						<h4>This menu is used to control the workflow itself.</h4>
+						<Divider style={{backgroundColor: "white", marginTop: 10, marginBottom: 10,}}/>
+						<FormControlLabel
+							style={{marginBottom: 15, color: "white",}}
+							label={<div style={{color: "white"}}>Exit on Error</div>}
+							control={
+								<Switch checked={workflow.configuration.exit_on_error} onChange={() => {
+									workflow.configuration.exit_on_error = !workflow.configuration.exit_on_error
+									setWorkflow(workflow)
+									setUpdate("exit_on_error_"+workflow.configuration.exit_on_error ? "true" : "false")
+									setShowShuffleMenu(false)
+								}} />
+							}
+						/>
+						<FormControlLabel
+							style={{marginBottom: 15, color: "white",}}
+							label={<div style={{color: "white"}}>Start from top</div>}
+							control={
+								<Switch checked={workflow.configuration.start_from_top} onChange={() => {
+									workflow.configuration.start_from_top = !workflow.configuration.start_from_top
+									setWorkflow(workflow)
+									setUpdate("start_from_top_"+workflow.configuration.start_from_top ? "true" : "false")
+									setShowShuffleMenu(false)
+								}} />
+							}
+						/>
+					</div>
+				</Menu>
+				<Tooltip color="secondary" title="Workflow settings" placement="top-start">
+					<Button color="primary" style={{height: 50, marginLeft: 10, }} variant="outlined" onClick={(event) => {
+						setShowShuffleMenu(!showShuffleMenu)
+						setNewAnchor(event.currentTarget)
+					}}>
+						<SettingsIcon />
+					</Button>
+				</Tooltip>
+			</div>
+		)
+	}
+
 	const WorkflowMenu = () => {
 		const [newAnchor, setNewAnchor] = React.useState(null);
 		const [showShuffleMenu, setShowShuffleMenu] = React.useState(false)
@@ -5539,6 +5595,7 @@ const AngularWorkflow = (props) => {
 							<DirectionsRunIcon />
 						</Button>
 					</Tooltip>
+					{/* <FileMenu />	*/}
 					<WorkflowMenu />	
 				</div>
 			</div>

frontend/src/views/Workflows.jsx

@@ -360,7 +360,7 @@ const Workflows = (props) => {
 				data.triggers[key].status = "stopped"
 			}
 		}
-
+		data["org"] = []
 		data.execution_org = {"id": ""}
 		console.log(data)