From eed0d96ed078b84a81926d1231239eb82d7d0db6 Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 28 Jun 2020 19:42:08 +0200 Subject: [PATCH] Pushing for blogpost 4 update --- backend/app_sdk/app_base.py | 33 ++++++++++++++++++----------- backend/go-app/walkoff.go | 2 +- backend/tests/forparser.py | 4 ++-- docker-compose.yml | 2 +- frontend/src/AngularWorkflow.js | 13 ++++++------ functions/onprem/orborus/orborus.go | 5 ++++- 6 files changed, 36 insertions(+), 23 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index c39233a8..89f1f0a7 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -438,10 +438,9 @@ class AppBase: print("Lower keyerror: %s" % e) return "KeyError: Couldn't find key: %s" % e - return basejson - - + return basejson + # Parses parameters sent to it and returns whether it did it successfully with the values found def parse_params(action, fullexecution, parameter): # Skip if it starts with $? jsonparsevalue = "$." @@ -748,7 +747,7 @@ class AppBase: #submatch = "([${]{2}([0-9a-zA-Z_-]+)(\[.*\])[}$]{2})" submatch = "([${]{2}([0-9a-zA-Z_-]+)(\[.*\])[}$]{2})" actualitem = re.findall(submatch, value, re.MULTILINE) - print("Multicheck: %s", actualitem) + print("Multicheck ", actualitem) if len(actualitem) > 0: multiexecution = True @@ -765,9 +764,12 @@ class AppBase: except IndexError: continue - itemlist = json.loads(actualitem) - if len(itemlist) > minlength: - minlength = len(itemlist) + try: + itemlist = json.loads(actualitem) + if len(itemlist) > minlength: + minlength = len(itemlist) + except json.decoder.JSONDecodeError as e: + print("JSON Error: %s in %s" % (e, actualitem)) replacements[to_be_replaced] = actualitem @@ -777,7 +779,13 @@ class AppBase: for i in range(0, minlength): tmpitem = json.loads(json.dumps(parameter["value"])) for key, value in replacements.items(): - replacement = json.loads(value)[i] + replacement = json.dumps(json.loads(value)[i]) + if replacement.startswith("\"") and replacement.endswith("\""): + replacement = replacement[1:len(replacement)-1] + #except json.decoder.JSONDecodeError as e: + + print("REPLACING %s with %s" % (key, replacement)) + #replacement = parse_wrapper_start(replacement) tmpitem = tmpitem.replace(key, replacement, -1) resultarray.append(tmpitem) @@ -786,7 +794,7 @@ class AppBase: multi_parameters[parameter["name"]] = resultarray else: # Parses things like int(value) - self.logger.info("Parsing wrapper data") + self.logger.info("Parsing wrapper data for %s" % value) value = parse_wrapper_start(value) params[parameter["name"]] = value @@ -806,6 +814,7 @@ class AppBase: except ValueError: result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres) print("Can't handle type %s value from function" % (type(newres))) + print("POST NEWRES: ", newres) else: print("APP_SDK DONE: Starting MULTI execution with", multi_parameters) # 1. Use number of executions based on longest array @@ -830,7 +839,8 @@ class AppBase: #print("Running with params %s" % baseparams) ret = await func(**baseparams) - print("Inner ret: %s" % ret) + ret = ret.replace("\"", "\\\"", -1) + print("Inner ret parsed: %s" % ret) try: results.append(json.loads(ret)) @@ -850,8 +860,7 @@ class AppBase: print("Normal result?") result = results - print("RESULT: %s" % result) - + print("RESULT: %s" % result) action_result["status"] = "SUCCESS" action_result["result"] = str(result) if action_result["result"] == "": diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 5aa2aa45..36b78bd4 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -1573,7 +1573,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // doesn't check sharing=true // Have to do it like this to add the user's apps log.Println("Apps set starting") - log.Printf("EXIT ON ERROR: %#v", workflow.Configuration.ExitOnError) + //log.Printf("EXIT ON ERROR: %#v", workflow.Configuration.ExitOnError) workflowApps := []WorkflowApp{} //memcacheName = "all_apps" //if item, err := memcache.Get(ctx, memcacheName); err == memcache.ErrCacheMiss { diff --git a/backend/tests/forparser.py b/backend/tests/forparser.py index ec748b56..1b898c6d 100644 --- a/backend/tests/forparser.py +++ b/backend/tests/forparser.py @@ -1,7 +1,7 @@ import re import json -fullexecution = {"type":"workflow","status":"FINISHED","start":"40447f30-fa44-4a4f-a133-4ee710368737","execution_argument":"","execution_id":"083eaa87-17ff-4aba-996c-83245051cf3d","execution_source":"default","workflow_id":"d7b73e8a-08fe-460e-987b-971cb6f1857f","last_node":"40447f30-fa44-4a4f-a133-4ee710368737","authorization":"b1264cd6-ed7a-4839-8caf-187eee8804b9","result":"TypeError: list indices must be integers or slices, not str","started_at":1593236788,"completed_at":1593236790,"project_id":"shuffle","locations":["europe-west2"],"workflow":{"actions":[{"app_name":"Testing","app_version":"1.0.0","app_id":"5411f573-9bba-44c4-a8d3-0e2bb704546d","errors":"null","id":"40447f30-fa44-4a4f-a133-4ee710368737","is_valid":True,"isStartNode":True,"sharing":True,"private_id":"","label":"Hello this is a name","small_image":"","large_image":"","environment":"Shuffle","name":"repeat_back_to_me","parameters":[{"description":"The message to repeat","id":"","name":"call","example":"REPEATING: Hello world","value":"$this is a test.name is not the same as $this is a test.name2 \n\n\nNot list $this is a test.loop.# either","multiline":True,"action_field":"","variant":"STATIC_VALUE","required":True,"schema":{"type":"string"}}],"execution_variable":{"description":"","id":"","name":"","value":""},"position":{"x":360.5,"y":454.5},"priority":0}],"branches":"null","triggers":"null","schedules":"null","configuration":{"exit_on_error":False,"start_from_top":False},"id":"d7b73e8a-08fe-460e-987b-971cb6f1857f","is_valid":True,"name":"App sdk parser testing","description":"","start":"40447f30-fa44-4a4f-a133-4ee710368737","owner":"43c36230-0a6e-40fc-aebc-a8ef57c81a88","sharing":"private","execution_org":{"name":"","org":"","users":"null","id":""},"workflow_variables":[{"description":"","id":"a034abee-5a5f-4347-9e58-6d2e58ce70f2","name":"This is a test","value":"{\"name\": \"testing\", \"name2\": \"testing3\", \"loop\": [{\"data\": \"1.1.1.1\", \"data_type\": \"ip\"}, {\"data\": \"2.2.2.2\", \"data_type\": \"ip\"}]}"}]},"results":[{"action":{"app_name":"Testing","app_version":"1.0.0","app_id":"5411f573-9bba-44c4-a8d3-0e2bb704546d","errors":"null","id":"40447f30-fa44-4a4f-a133-4ee710368737","is_valid":True,"isStartNode":True,"sharing":True,"private_id":"","label":"Hello this is a name","small_image":"","large_image":"","environment":"Shuffle","name":"repeat_back_to_me","parameters":[{"description":"The message to repeat","id":"","name":"call","example":"","value":"testing is not the same as testing2 \n\n\nNot list $this is a test.loop.# either","multiline":"false","action_field":"","variant":"STATIC_VALUE","required":True,"schema":{"type":"string"}}],"execution_variable":{"description":"","id":"","name":"","value":""},"position":{"x":360.5,"y":454.5},"priority":0},"execution_id":"083eaa87-17ff-4aba-996c-83245051cf3d","authorization":"b1264cd6-ed7a-4839-8caf-187eee8804b9","result":"TypeError: list indices must be integers or slices, not str","started_at":1593236790,"completed_at":1593236790,"status":"FAILURE"}]} +fullexecution = {"type":"workflow","status":"FINISHED","start":"40447f30-fa44-4a4f-a133-4ee710368737","execution_argument":"","execution_id":"083eaa87-17ff-4aba-996c-83245051cf3d","execution_source":"default","workflow_id":"d7b73e8a-08fe-460e-987b-971cb6f1857f","last_node":"40447f30-fa44-4a4f-a133-4ee710368737","authorization":"b1264cd6-ed7a-4839-8caf-187eee8804b9","result":"TypeError: list indices must be integers or slices, not str","started_at":1593236788,"completed_at":1593236790,"project_id":"shuffle","locations":["europe-west2"],"workflow":{"actions":[{"app_name":"Testing","app_version":"1.0.0","app_id":"5411f573-9bba-44c4-a8d3-0e2bb704546d","errors":"null","id":"40447f30-fa44-4a4f-a133-4ee710368737","is_valid":True,"isStartNode":True,"sharing":True,"private_id":"","label":"Hello this is a name","small_image":"","large_image":"","environment":"Shuffle","name":"repeat_back_to_me","parameters":[{"description":"The message to repeat","id":"","name":"call","example":"REPEATING: Hello world","value":"$this is a test.name is not the same as $this is a test.name2 \n\n\nNot list $this is a test.loop.# either","multiline":True,"action_field":"","variant":"STATIC_VALUE","required":True,"schema":{"type":"string"}}],"execution_variable":{"description":"","id":"","name":"","value":""},"position":{"x":360.5,"y":454.5},"priority":0}],"branches":"null","triggers":"null","schedules":"null","configuration":{"exit_on_error":False,"start_from_top":False},"id":"d7b73e8a-08fe-460e-987b-971cb6f1857f","is_valid":True,"name":"App sdk parser testing","description":"","start":"40447f30-fa44-4a4f-a133-4ee710368737","owner":"43c36230-0a6e-40fc-aebc-a8ef57c81a88","sharing":"private","execution_org":{"name":"","org":"","users":"null","id":""},"workflow_variables":[{"description":"","id":"a034abee-5a5f-4347-9e58-6d2e58ce70f2","name":"This is a test","value":"[{\"data\": \"1.1.1.1\", \"data_type\": \"ip\"}]"}]},"results":[{"action":{"app_name":"Testing","app_version":"1.0.0","app_id":"5411f573-9bba-44c4-a8d3-0e2bb704546d","errors":"null","id":"40447f30-fa44-4a4f-a133-4ee710368737","is_valid":True,"isStartNode":True,"sharing":True,"private_id":"","label":"Hello this is a name","small_image":"","large_image":"","environment":"Shuffle","name":"repeat_back_to_me","parameters":[{"description":"The message to repeat","id":"","name":"call","example":"","value":"testing is not the same as testing2 \n\n\nNot list $this is a test.loop.# either","multiline":"false","action_field":"","variant":"STATIC_VALUE","required":True,"schema":{"type":"string"}}],"execution_variable":{"description":"","id":"","name":"","value":""},"position":{"x":360.5,"y":454.5},"priority":0},"execution_id":"083eaa87-17ff-4aba-996c-83245051cf3d","authorization":"b1264cd6-ed7a-4839-8caf-187eee8804b9","result":"TypeError: list indices must be integers or slices, not str","started_at":1593236790,"completed_at":1593236790,"status":"FAILURE"}]} # Takes a workflow execution as argument @@ -135,7 +135,7 @@ def get_json_value(execution_data, input_data): parameter = { "value": """{ - "data8": "Not list $this is a test.loop either with the items $this is a test.loop.#.data_type" + "data8": "Not list $this is a test.#.data either with the items $this is a test.#.data_type" }""" } diff --git a/docker-compose.yml b/docker-compose.yml index c13e60fc..b74d7a3c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,7 @@ services: depends_on: - backend backend: - build: ./backend + #build: ./backend image: frikky/shuffle:backend container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} diff --git a/frontend/src/AngularWorkflow.js b/frontend/src/AngularWorkflow.js index db735647..ff9d1d5c 100644 --- a/frontend/src/AngularWorkflow.js +++ b/frontend/src/AngularWorkflow.js @@ -35,6 +35,7 @@ import Switch from '@material-ui/core/Switch'; import ReactJson from 'react-json-view' import { useBeforeunload } from 'react-beforeunload'; +import CachedIcon from '@material-ui/icons/Cached'; import DirectionsRunIcon from '@material-ui/icons/DirectionsRun'; import PolymerIcon from '@material-ui/icons/Polymer'; import CreateIcon from '@material-ui/icons/Create'; @@ -4305,11 +4306,9 @@ const AngularWorkflow = (props) => { /> - + @@ -4562,6 +4561,7 @@ const AngularWorkflow = (props) => { onClick={() => { getWorkflowExecution(props.match.params.key) }} color="primary"> + Refresh executions @@ -4664,7 +4664,7 @@ const AngularWorkflow = (props) => { : executionData.results.map(data => { - if (!showSkippedActions && (data.status === "SKIPPED" || data.status === "FAILURE")) { + if (executionData.results.length !== 1 && !showSkippedActions && (data.status === "SKIPPED" || data.status === "FAILURE")) { return null } @@ -4685,9 +4685,10 @@ const AngularWorkflow = (props) => { const curapp = apps.find(a => a.name === data.action.app_name && a.app_version === data.action.app_version) const imgsize = 50 + const statusColor = data.status === "FINISHED" || data.status === "SUCCESS" ? "green" : data.status === "ABORTED" || data.status === "FAILURE" ? "red" : "orange" const actionimg = curapp === null ? null : - {data.action.app_name} + {data.action.app_name} return (
diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 9cd19826..aed440b6 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -432,7 +432,6 @@ func zombiecheck() error { stopContainers := []string{} removeContainers := []string{} for _, container := range containers { - // Skip random containers. Only handle things related to Shuffle. if !strings.Contains(container.Image, baseimagename) { shuffleFound := false @@ -447,6 +446,8 @@ func zombiecheck() error { if !shuffleFound { continue } + //} else { + // log.Printf("NAME: %s", container.Image) } for _, name := range container.Names { @@ -455,6 +456,8 @@ func zombiecheck() error { continue } + log.Printf("NAME: %s", name) + // Need to check time here too because a container can be removed the same instant as its created currenttime := time.Now().Unix() if container.State != "running" && currenttime-container.Created > int64(workerTimeout) {