#65: Added number(), parse() and more for data parsing

backend/app_sdk/app_base.py

@@ -102,6 +102,208 @@ class AppBase:
 
         self.logger.info("AFTER FULLEXEC stream result")
 
+        # Gets the value at the paranthesis level you want
+        def parse_nested_param(string, level):
+            """
+            Generate strings contained in nested (), indexing i = level
+            """
+            if len(re.findall("\(", string)) == len(re.findall("\)", string)):
+                LeftRightIndex = [x for x in zip(
+                [Left.start()+1 for Left in re.finditer('\(', string)], 
+                reversed([Right.start() for Right in re.finditer('\)', string)]))]
+        
+            elif len(re.findall("\(", string)) > len(re.findall("\)", string)):
+                return parse_nested_param(string + ')', level)
+            elif len(re.findall("\(", string)) < len(re.findall("\)", string)):
+                return parse_nested_param('(' + string, level)
+        
+            else:
+                return 'Failed to parse params'
+        
+            try:
+                return [string[LeftRightIndex[level][0]:LeftRightIndex[level][1]]]
+            except IndexError:
+                return [string[LeftRightIndex[level+1][0]:LeftRightIndex[level+1][1]]]
+        
+        # Finds the deepest level paranthesis in a string
+        def maxDepth(S): 
+            current_max = 0
+            max = 0
+            n = len(S) 
+          
+            # Traverse the input string 
+            for i in range(n): 
+                if S[i] == '(': 
+                    current_max += 1
+          
+                    if current_max > max: 
+                        max = current_max 
+                elif S[i] == ')': 
+                    if current_max > 0: 
+                        current_max -= 1
+                    else: 
+                        return -1
+          
+            # finally check for unbalanced string 
+            if current_max != 0: 
+                return -1
+          
+            return max-1
+        
+        # Specific type parsing
+        def parse_type(data, thistype): 
+            if data == None:
+                return "Empty"
+        
+            if "int" in thistype or "number" in thistype:
+                try:
+                    return int(data)
+                except ValueError:
+                    print("ValueError while casting %s" % data)
+                    return data
+            if "lower" in thistype:
+                return data.lower()
+            if "upper" in thistype:
+                return data.upper()
+            if "trim" in thistype:
+                return data.strip()
+            if "strip" in thistype:
+                return data.strip()
+            if "split" in thistype:
+                return data.split()
+            if "len" in thistype or "length" in thistype:
+                return len(data)
+            if "parse" in thistype:
+                splitvalues = []
+                default_error = """Error. Expected syntax: parse(["hello","test1"],0:1)""" 
+                if "," in data:
+                    splitvalues = data.split(",")
+
+                    for item in range(len(splitvalues)):
+                        splitvalues[item] = splitvalues[item].strip()
+                else:
+                    return default_error 
+
+                lastsplit = []
+                if ":" in splitvalues[-1]:
+                    lastsplit = splitvalues[-1].split(":")
+                else:
+                    try:
+                        lastsplit = [int(splitvalues[-1])]
+                    except ValueError:
+                        return default_error
+
+                try:
+                    parsedlist = ",".join(splitvalues[0:-1])
+                    if len(lastsplit) > 1:
+                        tmp = json.loads(parsedlist)[int(lastsplit[0]):int(lastsplit[1])]
+                    else:
+                        tmp = json.loads(parsedlist)[lastsplit[0]]
+
+                    print(tmp)
+                    return tmp
+                except IndexError as e:
+                    return default_error
+        
+        # Parses the INNER value and recurses until everything is done
+        def parse_wrapper(data):
+            try:
+                if "(" not in data or ")" not in data:
+                    return data
+            except TypeError:
+                return data
+        
+            print("Running %s" % data)
+        
+            # Look for the INNER wrapper first, then move out
+            wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length"]
+            found = False
+            for wrapper in wrappers:
+                if wrapper not in data.lower():
+                    continue
+        
+                found = True
+                break
+        
+            if not found:
+                return data
+        
+            # Do stuff here.
+            innervalue = parse_nested_param(data, maxDepth(data)-0)
+            outervalue = parse_nested_param(data, maxDepth(data)-1)
+            print("INNER: ", outervalue)
+            print("OUTER: ", outervalue)
+        
+            if outervalue != innervalue:
+                #print("Outer: ", outervalue, " inner: ", innervalue)
+                for key in range(len(innervalue)):
+                    # Replace OUTERVALUE[key] with INNERVALUE[key] in data.
+                    print("Replace %s with %s in %s" % (outervalue[key], innervalue[key], data))
+                    data = data.replace(outervalue[key], innervalue[key])
+            else:
+                for thistype in wrappers:
+                    if thistype.lower() not in data.lower():
+                        continue
+        
+                    parsed_value = parse_type(innervalue[0], thistype.lower())
+                    return parsed_value
+        
+            print("DATA: %s\n" % data)
+            return parse_wrapper(data)
+
+        def parse_wrapper_start(data):
+            newdata = []
+            newstring = ""
+            record = True
+            paranCnt = 0
+            for char in data:
+                if char == "(":
+                    paranCnt += 1
+        
+                    if not record:
+                        record = True 
+        
+                if record:
+                    newstring += char
+        
+                if paranCnt == 0 and char == " ":
+                    newdata.append(newstring)
+                    newstring = ""
+                    record = True
+        
+                if char == ")":
+                    paranCnt -= 1
+        
+                    if paranCnt == 0:
+                        record = False
+        
+            if len(newstring) > 0:
+                newdata.append(newstring)
+        
+            print(newdata)
+            parsedlist = []
+            non_string = False
+            for item in newdata:
+                ret = parse_wrapper(item)
+                if not isinstance(ret, str):
+                    non_string = True
+        
+                parsedlist.append(ret)
+        
+            if len(parsedlist) > 0 and not non_string:
+                return " ".join(parsedlist)
+            elif len(parsedlist) == 1 and non_string:
+                return parsedlist[0]
+            else:
+                print("Casting back to string because multi: ", parsedlist)
+                newlist = []
+                for item in parsedlist:
+                    try:
+                        newlist.append(str(item))
+                    except ValueError:
+                        newlist.append("parsing_error")
+                return " ".join(newlist)
+
         # Takes a workflow execution as argument
         # Returns a string if the result is single, or a list if it's a list
         def get_json_value(execution_data, input_data):
@@ -185,7 +387,9 @@ class AppBase:
         
             return basejson
 
+
         def parse_params(action, fullexecution, parameter):
+            # Skip if it starts with $?
             jsonparsevalue = "$."
             match = ".*([$]{1}([a-zA-Z0-9()# _-]+\.?){1,})"
 
@@ -331,7 +535,8 @@ class AppBase:
                         if check:
                             return False, "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check)
 
-                    print(sourcevalue)
+
+                    sourcevalue = parse_wrapper_start(sourcevalue)
                     destinationvalue = condition["destination"]["value"]
 
                     if condition["destination"]["variant"]== "" or condition["destination"]["variant"]== "STATIC_VALUE":
@@ -341,6 +546,7 @@ class AppBase:
                         if check:
                             return False, "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check)
 
+                    destinationvalue = parse_wrapper_start(destinationvalue)
                     available_checks = [
                         "=",
                         "equals",
@@ -444,7 +650,6 @@ class AppBase:
                             if check:
                                 raise "Value check error: %s" % Exception(check)
 
-
                             # Custom format for ${name[0,1,2,...]}$
                             submatch = "([${]{2}([0-9a-zA-Z_-]+)(\[.*\])[}$]{2})"
                             actualitem = re.findall(submatch, value, re.MULTILINE)
@@ -484,20 +689,10 @@ class AppBase:
                                 # With this parameter ready, add it to... a greater list of parameters. Rofl
                                 multi_parameters[parameter["name"]] = resultarray
                             else:
-                                # Testing casting
+                                # Parses things like int(value)
-                                if value.lower() == "true":
+                                self.logger.info("Parsing wrapper data")
-                                    value = True
+                                value = parse_wrapper_start(value)
-                                elif value.lower() == "false":
+                                self.logger.info("Parsing done: ", value)
-                                    value = False
-                                else:
-                                    constructors = [int]
-                                    for c in constructors:
-                                        try:
-                                            value = c(value)
-                                            print("Successfully parsed %s to %s", value, c)
-                                        except ValueError:
-                                            print("Failed to parse %s to %s", value, c)
-                                            pass
 
                                 params[parameter["name"]] = value
                                 multi_parameters[parameter["name"]] = value 
@@ -511,8 +706,11 @@ class AppBase:
                             if isinstance(newres, str):
                                 result += newres
                             else:
-                                print("Can't handle type %s value from function" % (type(newres)))
+                                try:
-                                result += "Can't handle %s type from function" % type(newres)
+                                    result += str(result)
+                                except ValueError:
+                                    result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres)
+                                    print("Can't handle type %s value from function" % (type(newres)))
                         else:
                             print("APP_SDK DONE: Starting multi execution with", multi_parameters)
                             # 1. Use number of executions based on longest array
@@ -579,7 +777,7 @@ class AppBase:
             print(f"Failed to execute: {e}")
             self.logger.exception(f"Failed to execute {e}-{action['id']}")
             action_result["status"] = "FAILURE" 
-            action_result["result"] = "Exception: %s" % e
+            action_result["result"] = "General exception: %s" % e
 
         action_result["completed_at"] = int(time.time())
 

backend/app_sdk/build.sh

@@ -1,4 +1,9 @@
 #!/bin/bash
-docker rmi frikky/shuffle:app_sdk --force
+NAME=app_sdk
-docker build . -t frikky/shuffle:app_sdk
+VERSION=0.2.0
-docker push frikky/shuffle:app_sdk
+
+docker rmi frikky/shuffle:$NAME --force
+docker build . -t frikky/shuffle:$NAME -t frikky/$NAME:$VERSION
+
+docker push frikky/shuffle:$NAME
+docker push frikky/$NAME:$VERSION

backend/go-app/docker.go

@@ -190,12 +190,14 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin
 		buildOptions,
 	)
 	//log.Printf("IMAGERESPONSE: %#v", imageBuildResponse.Body)
+	defer imageBuildResponse.Body.Close()
+	_, newerr := io.Copy(os.Stdout, imageBuildResponse.Body)
+	if newerr != nil {
+		log.Printf("Failed reading Docker build STDOUT: %s", newerr)
+	}
 
 	if err != nil {
 		// Read the STDOUT from the build process
-		defer imageBuildResponse.Body.Close()
-		_, newerr := io.Copy(os.Stdout, imageBuildResponse.Body)
-		log.Printf("Failed reading Docker build STDOUT: %s", newerr)
 
 		return err
 	}

backend/go-app/main.go

@@ -6130,7 +6130,8 @@ func init() {
 	var err error
 	ctx := context.Background()
 
-	log.Printf("Running INIT process")
+	log.Printf("Starting Shuffle backend - initializing database connection")
+	// option.WithoutAuthentication
 	dbclient, err = datastore.NewClient(ctx, gceProject)
 	if err != nil {
 		panic(fmt.Sprintf("DBclient error during init: %s", err))

backend/go-app/walkoff.go

@@ -89,6 +89,7 @@ type WorkflowApp struct {
 	Activated   bool   `json:"activated" yaml:"activated" required:false datastore:"activated"`
 	Tested      bool   `json:"tested" yaml:"tested" required:false datastore:"tested"`
 	Owner       string `json:"owner" datastore:"owner" yaml:"owner"`
+	Hash        string `json:"hash" datastore:"hash" yaml:"hash"` // api.yaml+dockerfile+src/app.py for apps
 	PrivateID   string `json:"private_id" yaml:"private_id" required:false datastore:"private_id"`
 	Description string `json:"description" datastore:"description,noindex" required:false yaml:"description"`
 	Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"`
@@ -3853,22 +3854,9 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
 			// Check the file
 			filename := file.Name()
 			if filename == "Dockerfile" {
-				// Quick Dockerfile check
+				// Set up to make md5 and check if the app is new (api.yaml+src/app.py+Dockerfile)
-				//dockerfile := fmt.Sprintf("%sDockerfile", extra)
+				// Check if Dockerfile, app.py or api.yaml has changed. Hash?
-				//log.Printf("Handling Dockerfile %s", dockerfile)
+				//log.Printf("Handle Dockerfile in location %s", extra)
-				//dockerdata, err := ioutil.ReadFile(dockerfile)
-				//if err != nil {
-				//	log.Printf("Failed to read dockerfile")
-				//	continue
-				//}
-
-				//if len(dockerdata) == 0 {
-				//	log.Printf("Dockerfile is empty")
-				//	continue
-				//}
-
-				log.Printf("Handle Dockerfile in location %s", extra)
-
 				// Try api.yaml and api.yml
 				fullPath := fmt.Sprintf("%s%s", extra, "api.yaml")
 				fileReader, err := fs.Open(fullPath)
@@ -3881,21 +3869,55 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
 					}
 				}
 
-				readFile, err := ioutil.ReadAll(fileReader)
+				appfileData, err := ioutil.ReadAll(fileReader)
 				if err != nil {
 					log.Printf("Failed reading %s: %s", fullPath, err)
 					continue
 				}
 
-				if len(readFile) == 0 {
+				if len(appfileData) == 0 {
 					log.Printf("Failed reading %s - length is 0.", fullPath)
 					continue
 				}
 
-				var workflowapp WorkflowApp
+				// func md5sum(data []byte) string {
-				err = gyaml.Unmarshal(readFile, &workflowapp)
+				// Make hash
+				appPython := fmt.Sprintf("%s/src/app.py", extra)
+				appPythonReader, err := fs.Open(appPython)
 				if err != nil {
-					log.Printf("Failed unmarshaling %s: %s", fullPath, err)
+					log.Printf("Failed to read %s", appPython)
+					continue
+				}
+
+				appPythonData, err := ioutil.ReadAll(appPythonReader)
+				if err != nil {
+					log.Printf("Failed reading %s: %s", appPython, err)
+					continue
+				}
+
+				dockerFp := fmt.Sprintf("%s/Dockerfile", extra)
+				dockerfile, err := fs.Open(dockerFp)
+				if err != nil {
+					log.Printf("Failed to read %s", appPython)
+					continue
+				}
+
+				dockerfileData, err := ioutil.ReadAll(dockerfile)
+				if err != nil {
+					log.Printf("Failed to read dockerfile")
+					continue
+				}
+
+				combined := []byte{}
+				combined = append(combined, appfileData...)
+				combined = append(combined, appPythonData...)
+				combined = append(combined, dockerfileData...)
+				md5 := md5sum(combined)
+
+				var workflowapp WorkflowApp
+				err = gyaml.Unmarshal(appfileData, &workflowapp)
+				if err != nil {
+					log.Printf("Failed unmarshaling workflowapp %s: %s", fullPath, err)
 					continue
 				}
 
@@ -3915,15 +3937,27 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
 				}
 
 				// Make an option to override existing apps?
+				//Hash string `json:"hash" datastore:"hash" yaml:"hash"` // api.yaml+dockerfile+src/app.py for apps
 				removeApps := []string{}
+				skip := false
 				for _, app := range allapps {
 					if app.Name == workflowapp.Name && app.AppVersion == workflowapp.AppVersion {
-						//log.Printf("App upload for %s:%s already exists.", app.Name, app.AppVersion)
+						// FIXME: Check if there's a new APP_SDK as well.
-						log.Printf("Overriding app %s:%s as it exists.", app.Name, app.AppVersion)
+						// Skip this check if app_sdk is new.
+						if app.Hash == md5 && app.Hash != "" {
+							skip = true
+							break
+						}
+
+						//log.Printf("Overriding app %s:%s as it exists but has different hash.", app.Name, app.AppVersion)
 						removeApps = append(removeApps, app.ID)
 					}
 				}
 
+				if skip {
+					continue
+				}
+
 				err = checkWorkflowApp(workflowapp)
 				if err != nil {
 					log.Printf("%s for app %s:%s", err, workflowapp.Name, workflowapp.AppVersion)
@@ -3939,21 +3973,17 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
 					}
 				}
 
-				//if workflowapp.Environment == "" {
-				//	workflowapp.Environment = baseEnvironment
-				//}
-
 				workflowapp.ID = uuid.NewV4().String()
 				workflowapp.IsValid = true
 				workflowapp.Verified = true
 				workflowapp.Sharing = true
 				workflowapp.Downloaded = true
+				workflowapp.Hash = md5
 
 				err = setWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID)
 				if err != nil {
 					log.Printf("Failed setting workflowapp: %s", err)
 					continue
-					//return err
 				}
 
 				err = increaseStatisticsField(ctx, "total_apps_created", workflowapp.ID, 1)
@@ -3975,6 +4005,8 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
 				} else {
 					if len(tags) > 0 {
 						log.Printf("Successfully built image %s", tags[0])
+					} else {
+						log.Printf("Successfully built image docker img")
 					}
 				}
 			}

frontend/src/AngularWorkflow.js

@@ -4493,10 +4493,16 @@ const AngularWorkflow = (props) => {
 						executionData.results.map(data => {
 							var showResult = data.result.trim()
 							showResult.split(" None").join(" \"None\"")
-							//showResult = replaceAll(showResult, " None", " \"None\"")
+							console.log("RESULT: ", showResult)
+
+							// showResult = replaceAll(showResult, " None", " \"None\"")
+							// Super basic check.
 							var jsonvalid = true
 							try {
-								JSON.parse(showResult)
+								const tmp = String(JSON.parse(showResult))
+								if (!tmp.includes("{") && !tmp.includes("[")) {
+									jsonvalid = false
+								}
 							} catch (e) {
 								jsonvalid = false
 							}

frontend/src/Workflows.js

@@ -518,7 +518,10 @@ const Workflows = (props) => {
 		var showResult = data.result.trim()
 		showResult = replaceAll(showResult, " None", " \"None\"");
 		try {
-			JSON.parse(showResult)
+			const tmp = String(JSON.parse(showResult))
+			if (!tmp.includes("{") && !tmp.includes("[")) {
+				jsonvalid = false
+			}
 		} catch (e) {
 			jsonvalid = false
 		}