From 482b498a988afd26ea5a31e5e4262f6eeca95f46 Mon Sep 17 00:00:00 2001 From: frikky Date: Thu, 18 Jun 2020 11:15:52 +0200 Subject: [PATCH] #65: Added number(), parse() and more for data parsing --- backend/app_sdk/app_base.py | 236 +++++++++++++++++++++++++++++--- backend/app_sdk/build.sh | 11 +- backend/go-app/docker.go | 8 +- backend/go-app/main.go | 3 +- backend/go-app/walkoff.go | 88 ++++++++---- frontend/src/AngularWorkflow.js | 10 +- frontend/src/Workflows.js | 5 +- 7 files changed, 304 insertions(+), 57 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 51209a70..c78197f4 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -102,6 +102,208 @@ class AppBase: self.logger.info("AFTER FULLEXEC stream result") + # Gets the value at the paranthesis level you want + def parse_nested_param(string, level): + """ + Generate strings contained in nested (), indexing i = level + """ + if len(re.findall("\(", string)) == len(re.findall("\)", string)): + LeftRightIndex = [x for x in zip( + [Left.start()+1 for Left in re.finditer('\(', string)], + reversed([Right.start() for Right in re.finditer('\)', string)]))] + + elif len(re.findall("\(", string)) > len(re.findall("\)", string)): + return parse_nested_param(string + ')', level) + elif len(re.findall("\(", string)) < len(re.findall("\)", string)): + return parse_nested_param('(' + string, level) + + else: + return 'Failed to parse params' + + try: + return [string[LeftRightIndex[level][0]:LeftRightIndex[level][1]]] + except IndexError: + return [string[LeftRightIndex[level+1][0]:LeftRightIndex[level+1][1]]] + + # Finds the deepest level paranthesis in a string + def maxDepth(S): + current_max = 0 + max = 0 + n = len(S) + + # Traverse the input string + for i in range(n): + if S[i] == '(': + current_max += 1 + + if current_max > max: + max = current_max + elif S[i] == ')': + if current_max > 0: + current_max -= 1 + else: + return -1 + + # finally check for unbalanced string + if current_max != 0: + return -1 + + return max-1 + + # Specific type parsing + def parse_type(data, thistype): + if data == None: + return "Empty" + + if "int" in thistype or "number" in thistype: + try: + return int(data) + except ValueError: + print("ValueError while casting %s" % data) + return data + if "lower" in thistype: + return data.lower() + if "upper" in thistype: + return data.upper() + if "trim" in thistype: + return data.strip() + if "strip" in thistype: + return data.strip() + if "split" in thistype: + return data.split() + if "len" in thistype or "length" in thistype: + return len(data) + if "parse" in thistype: + splitvalues = [] + default_error = """Error. Expected syntax: parse(["hello","test1"],0:1)""" + if "," in data: + splitvalues = data.split(",") + + for item in range(len(splitvalues)): + splitvalues[item] = splitvalues[item].strip() + else: + return default_error + + lastsplit = [] + if ":" in splitvalues[-1]: + lastsplit = splitvalues[-1].split(":") + else: + try: + lastsplit = [int(splitvalues[-1])] + except ValueError: + return default_error + + try: + parsedlist = ",".join(splitvalues[0:-1]) + if len(lastsplit) > 1: + tmp = json.loads(parsedlist)[int(lastsplit[0]):int(lastsplit[1])] + else: + tmp = json.loads(parsedlist)[lastsplit[0]] + + print(tmp) + return tmp + except IndexError as e: + return default_error + + # Parses the INNER value and recurses until everything is done + def parse_wrapper(data): + try: + if "(" not in data or ")" not in data: + return data + except TypeError: + return data + + print("Running %s" % data) + + # Look for the INNER wrapper first, then move out + wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length"] + found = False + for wrapper in wrappers: + if wrapper not in data.lower(): + continue + + found = True + break + + if not found: + return data + + # Do stuff here. + innervalue = parse_nested_param(data, maxDepth(data)-0) + outervalue = parse_nested_param(data, maxDepth(data)-1) + print("INNER: ", outervalue) + print("OUTER: ", outervalue) + + if outervalue != innervalue: + #print("Outer: ", outervalue, " inner: ", innervalue) + for key in range(len(innervalue)): + # Replace OUTERVALUE[key] with INNERVALUE[key] in data. + print("Replace %s with %s in %s" % (outervalue[key], innervalue[key], data)) + data = data.replace(outervalue[key], innervalue[key]) + else: + for thistype in wrappers: + if thistype.lower() not in data.lower(): + continue + + parsed_value = parse_type(innervalue[0], thistype.lower()) + return parsed_value + + print("DATA: %s\n" % data) + return parse_wrapper(data) + + def parse_wrapper_start(data): + newdata = [] + newstring = "" + record = True + paranCnt = 0 + for char in data: + if char == "(": + paranCnt += 1 + + if not record: + record = True + + if record: + newstring += char + + if paranCnt == 0 and char == " ": + newdata.append(newstring) + newstring = "" + record = True + + if char == ")": + paranCnt -= 1 + + if paranCnt == 0: + record = False + + if len(newstring) > 0: + newdata.append(newstring) + + print(newdata) + parsedlist = [] + non_string = False + for item in newdata: + ret = parse_wrapper(item) + if not isinstance(ret, str): + non_string = True + + parsedlist.append(ret) + + if len(parsedlist) > 0 and not non_string: + return " ".join(parsedlist) + elif len(parsedlist) == 1 and non_string: + return parsedlist[0] + else: + print("Casting back to string because multi: ", parsedlist) + newlist = [] + for item in parsedlist: + try: + newlist.append(str(item)) + except ValueError: + newlist.append("parsing_error") + return " ".join(newlist) + # Takes a workflow execution as argument # Returns a string if the result is single, or a list if it's a list def get_json_value(execution_data, input_data): @@ -185,7 +387,9 @@ class AppBase: return basejson + def parse_params(action, fullexecution, parameter): + # Skip if it starts with $? jsonparsevalue = "$." match = ".*([$]{1}([a-zA-Z0-9()# _-]+\.?){1,})" @@ -331,7 +535,8 @@ class AppBase: if check: return False, "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check) - print(sourcevalue) + + sourcevalue = parse_wrapper_start(sourcevalue) destinationvalue = condition["destination"]["value"] if condition["destination"]["variant"]== "" or condition["destination"]["variant"]== "STATIC_VALUE": @@ -341,6 +546,7 @@ class AppBase: if check: return False, "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check) + destinationvalue = parse_wrapper_start(destinationvalue) available_checks = [ "=", "equals", @@ -444,7 +650,6 @@ class AppBase: if check: raise "Value check error: %s" % Exception(check) - # Custom format for ${name[0,1,2,...]}$ submatch = "([${]{2}([0-9a-zA-Z_-]+)(\[.*\])[}$]{2})" actualitem = re.findall(submatch, value, re.MULTILINE) @@ -484,20 +689,10 @@ class AppBase: # With this parameter ready, add it to... a greater list of parameters. Rofl multi_parameters[parameter["name"]] = resultarray else: - # Testing casting - if value.lower() == "true": - value = True - elif value.lower() == "false": - value = False - else: - constructors = [int] - for c in constructors: - try: - value = c(value) - print("Successfully parsed %s to %s", value, c) - except ValueError: - print("Failed to parse %s to %s", value, c) - pass + # Parses things like int(value) + self.logger.info("Parsing wrapper data") + value = parse_wrapper_start(value) + self.logger.info("Parsing done: ", value) params[parameter["name"]] = value multi_parameters[parameter["name"]] = value @@ -511,8 +706,11 @@ class AppBase: if isinstance(newres, str): result += newres else: - print("Can't handle type %s value from function" % (type(newres))) - result += "Can't handle %s type from function" % type(newres) + try: + result += str(result) + except ValueError: + result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres) + print("Can't handle type %s value from function" % (type(newres))) else: print("APP_SDK DONE: Starting multi execution with", multi_parameters) # 1. Use number of executions based on longest array @@ -579,7 +777,7 @@ class AppBase: print(f"Failed to execute: {e}") self.logger.exception(f"Failed to execute {e}-{action['id']}") action_result["status"] = "FAILURE" - action_result["result"] = "Exception: %s" % e + action_result["result"] = "General exception: %s" % e action_result["completed_at"] = int(time.time()) diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 980e4d41..f9dadcad 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,4 +1,9 @@ #!/bin/bash -docker rmi frikky/shuffle:app_sdk --force -docker build . -t frikky/shuffle:app_sdk -docker push frikky/shuffle:app_sdk +NAME=app_sdk +VERSION=0.2.0 + +docker rmi frikky/shuffle:$NAME --force +docker build . -t frikky/shuffle:$NAME -t frikky/$NAME:$VERSION + +docker push frikky/shuffle:$NAME +docker push frikky/$NAME:$VERSION diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index da553183..b578e415 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -190,12 +190,14 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin buildOptions, ) //log.Printf("IMAGERESPONSE: %#v", imageBuildResponse.Body) + defer imageBuildResponse.Body.Close() + _, newerr := io.Copy(os.Stdout, imageBuildResponse.Body) + if newerr != nil { + log.Printf("Failed reading Docker build STDOUT: %s", newerr) + } if err != nil { // Read the STDOUT from the build process - defer imageBuildResponse.Body.Close() - _, newerr := io.Copy(os.Stdout, imageBuildResponse.Body) - log.Printf("Failed reading Docker build STDOUT: %s", newerr) return err } diff --git a/backend/go-app/main.go b/backend/go-app/main.go index eae1259e..372c0203 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -6130,7 +6130,8 @@ func init() { var err error ctx := context.Background() - log.Printf("Running INIT process") + log.Printf("Starting Shuffle backend - initializing database connection") + // option.WithoutAuthentication dbclient, err = datastore.NewClient(ctx, gceProject) if err != nil { panic(fmt.Sprintf("DBclient error during init: %s", err)) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 65823b77..b85d7c2a 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -89,6 +89,7 @@ type WorkflowApp struct { Activated bool `json:"activated" yaml:"activated" required:false datastore:"activated"` Tested bool `json:"tested" yaml:"tested" required:false datastore:"tested"` Owner string `json:"owner" datastore:"owner" yaml:"owner"` + Hash string `json:"hash" datastore:"hash" yaml:"hash"` // api.yaml+dockerfile+src/app.py for apps PrivateID string `json:"private_id" yaml:"private_id" required:false datastore:"private_id"` Description string `json:"description" datastore:"description,noindex" required:false yaml:"description"` Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"` @@ -3853,22 +3854,9 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin // Check the file filename := file.Name() if filename == "Dockerfile" { - // Quick Dockerfile check - //dockerfile := fmt.Sprintf("%sDockerfile", extra) - //log.Printf("Handling Dockerfile %s", dockerfile) - //dockerdata, err := ioutil.ReadFile(dockerfile) - //if err != nil { - // log.Printf("Failed to read dockerfile") - // continue - //} - - //if len(dockerdata) == 0 { - // log.Printf("Dockerfile is empty") - // continue - //} - - log.Printf("Handle Dockerfile in location %s", extra) - + // Set up to make md5 and check if the app is new (api.yaml+src/app.py+Dockerfile) + // Check if Dockerfile, app.py or api.yaml has changed. Hash? + //log.Printf("Handle Dockerfile in location %s", extra) // Try api.yaml and api.yml fullPath := fmt.Sprintf("%s%s", extra, "api.yaml") fileReader, err := fs.Open(fullPath) @@ -3881,21 +3869,55 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin } } - readFile, err := ioutil.ReadAll(fileReader) + appfileData, err := ioutil.ReadAll(fileReader) if err != nil { log.Printf("Failed reading %s: %s", fullPath, err) continue } - if len(readFile) == 0 { + if len(appfileData) == 0 { log.Printf("Failed reading %s - length is 0.", fullPath) continue } - var workflowapp WorkflowApp - err = gyaml.Unmarshal(readFile, &workflowapp) + // func md5sum(data []byte) string { + // Make hash + appPython := fmt.Sprintf("%s/src/app.py", extra) + appPythonReader, err := fs.Open(appPython) if err != nil { - log.Printf("Failed unmarshaling %s: %s", fullPath, err) + log.Printf("Failed to read %s", appPython) + continue + } + + appPythonData, err := ioutil.ReadAll(appPythonReader) + if err != nil { + log.Printf("Failed reading %s: %s", appPython, err) + continue + } + + dockerFp := fmt.Sprintf("%s/Dockerfile", extra) + dockerfile, err := fs.Open(dockerFp) + if err != nil { + log.Printf("Failed to read %s", appPython) + continue + } + + dockerfileData, err := ioutil.ReadAll(dockerfile) + if err != nil { + log.Printf("Failed to read dockerfile") + continue + } + + combined := []byte{} + combined = append(combined, appfileData...) + combined = append(combined, appPythonData...) + combined = append(combined, dockerfileData...) + md5 := md5sum(combined) + + var workflowapp WorkflowApp + err = gyaml.Unmarshal(appfileData, &workflowapp) + if err != nil { + log.Printf("Failed unmarshaling workflowapp %s: %s", fullPath, err) continue } @@ -3915,15 +3937,27 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin } // Make an option to override existing apps? + //Hash string `json:"hash" datastore:"hash" yaml:"hash"` // api.yaml+dockerfile+src/app.py for apps removeApps := []string{} + skip := false for _, app := range allapps { if app.Name == workflowapp.Name && app.AppVersion == workflowapp.AppVersion { - //log.Printf("App upload for %s:%s already exists.", app.Name, app.AppVersion) - log.Printf("Overriding app %s:%s as it exists.", app.Name, app.AppVersion) + // FIXME: Check if there's a new APP_SDK as well. + // Skip this check if app_sdk is new. + if app.Hash == md5 && app.Hash != "" { + skip = true + break + } + + //log.Printf("Overriding app %s:%s as it exists but has different hash.", app.Name, app.AppVersion) removeApps = append(removeApps, app.ID) } } + if skip { + continue + } + err = checkWorkflowApp(workflowapp) if err != nil { log.Printf("%s for app %s:%s", err, workflowapp.Name, workflowapp.AppVersion) @@ -3939,21 +3973,17 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin } } - //if workflowapp.Environment == "" { - // workflowapp.Environment = baseEnvironment - //} - workflowapp.ID = uuid.NewV4().String() workflowapp.IsValid = true workflowapp.Verified = true workflowapp.Sharing = true workflowapp.Downloaded = true + workflowapp.Hash = md5 err = setWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) if err != nil { log.Printf("Failed setting workflowapp: %s", err) continue - //return err } err = increaseStatisticsField(ctx, "total_apps_created", workflowapp.ID, 1) @@ -3975,6 +4005,8 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin } else { if len(tags) > 0 { log.Printf("Successfully built image %s", tags[0]) + } else { + log.Printf("Successfully built image docker img") } } } diff --git a/frontend/src/AngularWorkflow.js b/frontend/src/AngularWorkflow.js index 4a8fb1a7..4f17c815 100644 --- a/frontend/src/AngularWorkflow.js +++ b/frontend/src/AngularWorkflow.js @@ -4493,10 +4493,16 @@ const AngularWorkflow = (props) => { executionData.results.map(data => { var showResult = data.result.trim() showResult.split(" None").join(" \"None\"") - //showResult = replaceAll(showResult, " None", " \"None\"") + console.log("RESULT: ", showResult) + + // showResult = replaceAll(showResult, " None", " \"None\"") + // Super basic check. var jsonvalid = true try { - JSON.parse(showResult) + const tmp = String(JSON.parse(showResult)) + if (!tmp.includes("{") && !tmp.includes("[")) { + jsonvalid = false + } } catch (e) { jsonvalid = false } diff --git a/frontend/src/Workflows.js b/frontend/src/Workflows.js index 18f82242..daeba584 100644 --- a/frontend/src/Workflows.js +++ b/frontend/src/Workflows.js @@ -518,7 +518,10 @@ const Workflows = (props) => { var showResult = data.result.trim() showResult = replaceAll(showResult, " None", " \"None\""); try { - JSON.parse(showResult) + const tmp = String(JSON.parse(showResult)) + if (!tmp.includes("{") && !tmp.includes("[")) { + jsonvalid = false + } } catch (e) { jsonvalid = false }