marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pushing for blogpost 4 update

Browse Source
This commit is contained in:
frikky
2020-06-28 19:42:08 +02:00
parent dfc639c92b
commit eed0d96ed0
6 changed files with 36 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app_sdk/app_base.py
+21 -12
View File
@@ -438,10 +438,9 @@ class AppBase:
print("Lower keyerror: %s" % e) print("Lower keyerror: %s" % e)
return "KeyError: Couldn't find key: %s" % e return "KeyError: Couldn't find key: %s" % e
return basejson return basejson
# Parses parameters sent to it and returns whether it did it successfully with the values found
def parse_params(action, fullexecution, parameter): def parse_params(action, fullexecution, parameter):
# Skip if it starts with $? # Skip if it starts with $?
jsonparsevalue = "$." jsonparsevalue = "$."
@@ -748,7 +747,7 @@ class AppBase:
#submatch = "([${]{2}([0-9a-zA-Z_-]+)(\[.*\])[}$]{2})" #submatch = "([${]{2}([0-9a-zA-Z_-]+)(\[.*\])[}$]{2})"
submatch = "([${]{2}([0-9a-zA-Z_-]+)(\[.*\])[}$]{2})" submatch = "([${]{2}([0-9a-zA-Z_-]+)(\[.*\])[}$]{2})"
actualitem = re.findall(submatch, value, re.MULTILINE) actualitem = re.findall(submatch, value, re.MULTILINE)
print("Multicheck: %s", actualitem) print("Multicheck ", actualitem)
if len(actualitem) > 0: if len(actualitem) > 0:
multiexecution = True multiexecution = True
@@ -765,9 +764,12 @@ class AppBase:
except IndexError: except IndexError:
continue continue
itemlist = json.loads(actualitem) try:
if len(itemlist) > minlength: itemlist = json.loads(actualitem)
minlength = len(itemlist) if len(itemlist) > minlength:
minlength = len(itemlist)
except json.decoder.JSONDecodeError as e:
print("JSON Error: %s in %s" % (e, actualitem))
replacements[to_be_replaced] = actualitem replacements[to_be_replaced] = actualitem
@@ -777,7 +779,13 @@ class AppBase:
for i in range(0, minlength): for i in range(0, minlength):
tmpitem = json.loads(json.dumps(parameter["value"])) tmpitem = json.loads(json.dumps(parameter["value"]))
for key, value in replacements.items(): for key, value in replacements.items():
replacement = json.loads(value)[i] replacement = json.dumps(json.loads(value)[i])
if replacement.startswith("\"") and replacement.endswith("\""):
replacement = replacement[1:len(replacement)-1]
#except json.decoder.JSONDecodeError as e:
print("REPLACING %s with %s" % (key, replacement))
#replacement = parse_wrapper_start(replacement)
tmpitem = tmpitem.replace(key, replacement, -1) tmpitem = tmpitem.replace(key, replacement, -1)
resultarray.append(tmpitem) resultarray.append(tmpitem)
@@ -786,7 +794,7 @@ class AppBase:
multi_parameters[parameter["name"]] = resultarray multi_parameters[parameter["name"]] = resultarray
else: else:
# Parses things like int(value) # Parses things like int(value)
self.logger.info("Parsing wrapper data") self.logger.info("Parsing wrapper data for %s" % value)
value = parse_wrapper_start(value) value = parse_wrapper_start(value)
params[parameter["name"]] = value params[parameter["name"]] = value
@@ -806,6 +814,7 @@ class AppBase:
except ValueError: except ValueError:
result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres) result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres)
print("Can't handle type %s value from function" % (type(newres))) print("Can't handle type %s value from function" % (type(newres)))
print("POST NEWRES: ", newres)
else: else:
print("APP_SDK DONE: Starting MULTI execution with", multi_parameters) print("APP_SDK DONE: Starting MULTI execution with", multi_parameters)
# 1. Use number of executions based on longest array # 1. Use number of executions based on longest array
@@ -830,7 +839,8 @@ class AppBase:
#print("Running with params %s" % baseparams) #print("Running with params %s" % baseparams)
ret = await func(**baseparams) ret = await func(**baseparams)
print("Inner ret: %s" % ret) ret = ret.replace("\"", "\\\"", -1)
print("Inner ret parsed: %s" % ret)
try: try:
results.append(json.loads(ret)) results.append(json.loads(ret))
@@ -850,8 +860,7 @@ class AppBase:
print("Normal result?") print("Normal result?")
result = results result = results
print("RESULT: %s" % result) print("RESULT: %s" % result)
action_result["status"] = "SUCCESS" action_result["status"] = "SUCCESS"
action_result["result"] = str(result) action_result["result"] = str(result)
if action_result["result"] == "": if action_result["result"] == "":
backend/go-app/walkoff.go
+1 -1
View File
@@ -1573,7 +1573,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
// doesn't check sharing=true // doesn't check sharing=true
// Have to do it like this to add the user's apps // Have to do it like this to add the user's apps
log.Println("Apps set starting") log.Println("Apps set starting")
log.Printf("EXIT ON ERROR: %#v", workflow.Configuration.ExitOnError) //log.Printf("EXIT ON ERROR: %#v", workflow.Configuration.ExitOnError)
workflowApps := []WorkflowApp{} workflowApps := []WorkflowApp{}
//memcacheName = "all_apps" //memcacheName = "all_apps"
//if item, err := memcache.Get(ctx, memcacheName); err == memcache.ErrCacheMiss { //if item, err := memcache.Get(ctx, memcacheName); err == memcache.ErrCacheMiss {
backend/tests/forparser.py
+2 -2
View File
@@ -1,7 +1,7 @@
import re import re
import json import json
fullexecution = {"type":"workflow","status":"FINISHED","start":"40447f30-fa44-4a4f-a133-4ee710368737","execution_argument":"","execution_id":"083eaa87-17ff-4aba-996c-83245051cf3d","execution_source":"default","workflow_id":"d7b73e8a-08fe-460e-987b-971cb6f1857f","last_node":"40447f30-fa44-4a4f-a133-4ee710368737","authorization":"b1264cd6-ed7a-4839-8caf-187eee8804b9","result":"TypeError: list indices must be integers or slices, not str","started_at":1593236788,"completed_at":1593236790,"project_id":"shuffle","locations":["europe-west2"],"workflow":{"actions":[{"app_name":"Testing","app_version":"1.0.0","app_id":"5411f573-9bba-44c4-a8d3-0e2bb704546d","errors":"null","id":"40447f30-fa44-4a4f-a133-4ee710368737","is_valid":True,"isStartNode":True,"sharing":True,"private_id":"","label":"Hello this is a name","small_image":"","large_image":"","environment":"Shuffle","name":"repeat_back_to_me","parameters":[{"description":"The message to repeat","id":"","name":"call","example":"REPEATING: Hello world","value":"$this is a test.name is not the same as $this is a test.name2 \n\n\nNot list $this is a test.loop.# either","multiline":True,"action_field":"","variant":"STATIC_VALUE","required":True,"schema":{"type":"string"}}],"execution_variable":{"description":"","id":"","name":"","value":""},"position":{"x":360.5,"y":454.5},"priority":0}],"branches":"null","triggers":"null","schedules":"null","configuration":{"exit_on_error":False,"start_from_top":False},"id":"d7b73e8a-08fe-460e-987b-971cb6f1857f","is_valid":True,"name":"App sdk parser testing","description":"","start":"40447f30-fa44-4a4f-a133-4ee710368737","owner":"43c36230-0a6e-40fc-aebc-a8ef57c81a88","sharing":"private","execution_org":{"name":"","org":"","users":"null","id":""},"workflow_variables":[{"description":"","id":"a034abee-5a5f-4347-9e58-6d2e58ce70f2","name":"This is a test","value":"{\"name\": \"testing\", \"name2\": \"testing3\", \"loop\": [{\"data\": \"1.1.1.1\", \"data_type\": \"ip\"}, {\"data\": \"2.2.2.2\", \"data_type\": \"ip\"}]}"}]},"results":[{"action":{"app_name":"Testing","app_version":"1.0.0","app_id":"5411f573-9bba-44c4-a8d3-0e2bb704546d","errors":"null","id":"40447f30-fa44-4a4f-a133-4ee710368737","is_valid":True,"isStartNode":True,"sharing":True,"private_id":"","label":"Hello this is a name","small_image":"","large_image":"","environment":"Shuffle","name":"repeat_back_to_me","parameters":[{"description":"The message to repeat","id":"","name":"call","example":"","value":"testing is not the same as testing2 \n\n\nNot list $this is a test.loop.# either","multiline":"false","action_field":"","variant":"STATIC_VALUE","required":True,"schema":{"type":"string"}}],"execution_variable":{"description":"","id":"","name":"","value":""},"position":{"x":360.5,"y":454.5},"priority":0},"execution_id":"083eaa87-17ff-4aba-996c-83245051cf3d","authorization":"b1264cd6-ed7a-4839-8caf-187eee8804b9","result":"TypeError: list indices must be integers or slices, not str","started_at":1593236790,"completed_at":1593236790,"status":"FAILURE"}]} fullexecution = {"type":"workflow","status":"FINISHED","start":"40447f30-fa44-4a4f-a133-4ee710368737","execution_argument":"","execution_id":"083eaa87-17ff-4aba-996c-83245051cf3d","execution_source":"default","workflow_id":"d7b73e8a-08fe-460e-987b-971cb6f1857f","last_node":"40447f30-fa44-4a4f-a133-4ee710368737","authorization":"b1264cd6-ed7a-4839-8caf-187eee8804b9","result":"TypeError: list indices must be integers or slices, not str","started_at":1593236788,"completed_at":1593236790,"project_id":"shuffle","locations":["europe-west2"],"workflow":{"actions":[{"app_name":"Testing","app_version":"1.0.0","app_id":"5411f573-9bba-44c4-a8d3-0e2bb704546d","errors":"null","id":"40447f30-fa44-4a4f-a133-4ee710368737","is_valid":True,"isStartNode":True,"sharing":True,"private_id":"","label":"Hello this is a name","small_image":"","large_image":"","environment":"Shuffle","name":"repeat_back_to_me","parameters":[{"description":"The message to repeat","id":"","name":"call","example":"REPEATING: Hello world","value":"$this is a test.name is not the same as $this is a test.name2 \n\n\nNot list $this is a test.loop.# either","multiline":True,"action_field":"","variant":"STATIC_VALUE","required":True,"schema":{"type":"string"}}],"execution_variable":{"description":"","id":"","name":"","value":""},"position":{"x":360.5,"y":454.5},"priority":0}],"branches":"null","triggers":"null","schedules":"null","configuration":{"exit_on_error":False,"start_from_top":False},"id":"d7b73e8a-08fe-460e-987b-971cb6f1857f","is_valid":True,"name":"App sdk parser testing","description":"","start":"40447f30-fa44-4a4f-a133-4ee710368737","owner":"43c36230-0a6e-40fc-aebc-a8ef57c81a88","sharing":"private","execution_org":{"name":"","org":"","users":"null","id":""},"workflow_variables":[{"description":"","id":"a034abee-5a5f-4347-9e58-6d2e58ce70f2","name":"This is a test","value":"[{\"data\": \"1.1.1.1\", \"data_type\": \"ip\"}]"}]},"results":[{"action":{"app_name":"Testing","app_version":"1.0.0","app_id":"5411f573-9bba-44c4-a8d3-0e2bb704546d","errors":"null","id":"40447f30-fa44-4a4f-a133-4ee710368737","is_valid":True,"isStartNode":True,"sharing":True,"private_id":"","label":"Hello this is a name","small_image":"","large_image":"","environment":"Shuffle","name":"repeat_back_to_me","parameters":[{"description":"The message to repeat","id":"","name":"call","example":"","value":"testing is not the same as testing2 \n\n\nNot list $this is a test.loop.# either","multiline":"false","action_field":"","variant":"STATIC_VALUE","required":True,"schema":{"type":"string"}}],"execution_variable":{"description":"","id":"","name":"","value":""},"position":{"x":360.5,"y":454.5},"priority":0},"execution_id":"083eaa87-17ff-4aba-996c-83245051cf3d","authorization":"b1264cd6-ed7a-4839-8caf-187eee8804b9","result":"TypeError: list indices must be integers or slices, not str","started_at":1593236790,"completed_at":1593236790,"status":"FAILURE"}]}
# Takes a workflow execution as argument # Takes a workflow execution as argument
@@ -135,7 +135,7 @@ def get_json_value(execution_data, input_data):
parameter = { parameter = {
"value": """{ "value": """{
"data8": "Not list $this is a test.loop either with the items $this is a test.loop.#.data_type" "data8": "Not list $this is a test.#.data either with the items $this is a test.#.data_type"
}""" }"""
} }
docker-compose.yml
+1 -1
View File
@@ -14,7 +14,7 @@ services:
depends_on: depends_on:
- backend - backend
backend: backend:
build: ./backend #build: ./backend
image: frikky/shuffle:backend image: frikky/shuffle:backend
container_name: shuffle-backend container_name: shuffle-backend
hostname: ${BACKEND_HOSTNAME} hostname: ${BACKEND_HOSTNAME}
frontend/src/AngularWorkflow.js
+7 -6
View File
@@ -35,6 +35,7 @@ import Switch from '@material-ui/core/Switch';
import ReactJson from 'react-json-view' import ReactJson from 'react-json-view'
import { useBeforeunload } from 'react-beforeunload'; import { useBeforeunload } from 'react-beforeunload';
import CachedIcon from '@material-ui/icons/Cached';
import DirectionsRunIcon from '@material-ui/icons/DirectionsRun'; import DirectionsRunIcon from '@material-ui/icons/DirectionsRun';
import PolymerIcon from '@material-ui/icons/Polymer'; import PolymerIcon from '@material-ui/icons/Polymer';
import CreateIcon from '@material-ui/icons/Create'; import CreateIcon from '@material-ui/icons/Create';
@@ -4305,11 +4306,9 @@ const AngularWorkflow = (props) => {
/> />
</div> </div>
</Menu> </Menu>
<Tooltip color="secondary" title="Show settings" placement="top-start"> <Tooltip color="secondary" title="Workflow settings" placement="top-start">
<Button color="primary" style={{height: 50, marginLeft: 10, }} variant="outlined" onClick={(event) => { <Button color="primary" style={{height: 50, marginLeft: 10, }} variant="outlined" onClick={(event) => {
setShowShuffleMenu(!showShuffleMenu) setShowShuffleMenu(!showShuffleMenu)
console.log("EVENT: ", event)
console.log("EVENT: ", event.currentTarget)
setNewAnchor(event.currentTarget) setNewAnchor(event.currentTarget)
}}> }}>
<SettingsIcon /> <SettingsIcon />
@@ -4365,7 +4364,7 @@ const AngularWorkflow = (props) => {
/> />
</Tooltip> </Tooltip>
<Tooltip color="primary" title="Save (ctrl+s)" placement="top"> <Tooltip color="primary" title="Save (ctrl+s)" placement="top">
<Button color="primary" style={{height: 50, marginLeft: 10, }} variant="outlined" onClick={() => saveWorkflow()}> <Button color="primary" style={{height: 50, marginLeft: 10, }} variant={lastSaved ? "outlined" : "contained"} onClick={() => saveWorkflow()}>
<SaveIcon /> <SaveIcon />
</Button> </Button>
</Tooltip> </Tooltip>
@@ -4562,6 +4561,7 @@ const AngularWorkflow = (props) => {
onClick={() => { onClick={() => {
getWorkflowExecution(props.match.params.key) getWorkflowExecution(props.match.params.key)
}} color="primary"> }} color="primary">
<CachedIcon style={{marginRight: 10}}/>
Refresh executions Refresh executions
</Button> </Button>
<Divider style={{backgroundColor: "white", marginTop: 10, marginBottom: 10,}}/> <Divider style={{backgroundColor: "white", marginTop: 10, marginBottom: 10,}}/>
@@ -4664,7 +4664,7 @@ const AngularWorkflow = (props) => {
<CircularProgress /> <CircularProgress />
: :
executionData.results.map(data => { executionData.results.map(data => {
if (!showSkippedActions && (data.status === "SKIPPED" || data.status === "FAILURE")) { if (executionData.results.length !== 1 && !showSkippedActions && (data.status === "SKIPPED" || data.status === "FAILURE")) {
return null return null
} }
@@ -4685,9 +4685,10 @@ const AngularWorkflow = (props) => {
const curapp = apps.find(a => a.name === data.action.app_name && a.app_version === data.action.app_version) const curapp = apps.find(a => a.name === data.action.app_name && a.app_version === data.action.app_version)
const imgsize = 50 const imgsize = 50
const statusColor = data.status === "FINISHED" || data.status === "SUCCESS" ? "green" : data.status === "ABORTED" || data.status === "FAILURE" ? "red" : "orange"
const actionimg = curapp === null ? const actionimg = curapp === null ?
null : null :
<img alt={data.action.app_name} src={curapp.large_image} style={{marginRight: 20, width: imgsize, height: imgsize}} /> <img alt={data.action.app_name} src={curapp.large_image} style={{marginRight: 20, width: imgsize, height: imgsize, border: `2px solid ${statusColor}`}} />
return ( return (
<div style={{marginBottom: 40,}}> <div style={{marginBottom: 40,}}>
functions/onprem/orborus/orborus.go
+4 -1
View File
@@ -432,7 +432,6 @@ func zombiecheck() error {
stopContainers := []string{} stopContainers := []string{}
removeContainers := []string{} removeContainers := []string{}
for _, container := range containers { for _, container := range containers {
// Skip random containers. Only handle things related to Shuffle. // Skip random containers. Only handle things related to Shuffle.
if !strings.Contains(container.Image, baseimagename) { if !strings.Contains(container.Image, baseimagename) {
shuffleFound := false shuffleFound := false
@@ -447,6 +446,8 @@ func zombiecheck() error {
if !shuffleFound { if !shuffleFound {
continue continue
} }
//} else {
// log.Printf("NAME: %s", container.Image)
} }
for _, name := range container.Names { for _, name := range container.Names {
@@ -455,6 +456,8 @@ func zombiecheck() error {
continue continue
} }
log.Printf("NAME: %s", name)
// Need to check time here too because a container can be removed the same instant as its created // Need to check time here too because a container can be removed the same instant as its created
currenttime := time.Now().Unix() currenttime := time.Now().Unix()
if container.State != "running" && currenttime-container.Created > int64(workerTimeout) { if container.State != "running" && currenttime-container.Created > int64(workerTimeout) {