marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

FEATURE: Added sub-workflow trigger

Browse Source
This commit is contained in:
frikky
2021-01-03 16:54:31 +01:00
parent a7e0b2706b
commit faae62e1ab
12 changed files with 495 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app_sdk/app_base.py
+71 -28
View File
@@ -25,6 +25,7 @@ class AppBase:
self.authorization = os.getenv("AUTHORIZATION", "")
self.current_execution_id = os.getenv("EXECUTIONID", "")
self.full_execution = os.getenv("FULL_EXECUTION", "")
self.result_wrapper_count = 0
if isinstance(self.action, str):
self.action = json.loads(self.action)
@@ -111,22 +112,52 @@ class AppBase:
#check_value = "$Filter_list_testing.wrapper.#.tmp"
#self.action = action
loopnames = []
for key, value in baseparams.items():
check_value = ""
for param in self.action["parameters"]:
if param["name"] == key:
#print("PARAM: %s" % param)
check_value = param["value"]
break
# self.result_wrapper_count = 0
octothorpe_count = param["value"].count(".#")
if octothorpe_count > self.result_wrapper_count:
self.result_wrapper_count = octothorpe_count
print("NEW OCTOTHORPE WRAPPER: %d" % octothorpe_count)
# This whole thing is hard.
# item = [{"data": "1.2.3.4", "dataType": "ip"}]
# $item = DONT loop items.
# $item.# = Loop items
# $item.#.data = Loop items
# With a single item, this is fine.
# item = [{"list": [{"data": "1.2.3.4", "dataType": "ip"}]}]
# $item = DONT loop items
# $item.# = Loop items
# $item.#.list = DONT loop items
# $item.#.list.# = Loop items
# $item.#.list.#.data = Loop items
# If the item itself is a list.. hmm
# FIXME: Check the above, and fix so that nested looped items can be
# Skipped if wanted
print("\nCHECK: %s" % check_value)
should_merge = False
if "#" in check_value:
should_merge = True
if isinstance(value, list):
if len(value) <= 1:
if len(value) == 1:
baseparams[key] = value[0]
#if "#" in check_value:
# should_merge = True
else:
if not check_value.endswith("#"):
if not should_merge:
print("Adding WITHOUT looping list")
else:
if len(value) not in listlengths:
@@ -239,36 +270,48 @@ class AppBase:
results = []
if has_loop:
print("Should run inner loop: %s" % newparams)
print("[WARNING] Should run inner loop: %s" % newparams)
ret = await self.run_recursed_items(func, newparams, loop_wrapper)
else:
print("Should run with params (inner): %s" % newparams)
print("[INFO] Should run multiplier check with params (inner): %s" % newparams)
# 1. Find the loops that are required and create new multipliers
# If here: check for multipliers within this scope.
ret = []
param_multiplier = await self.get_param_multipliers(newparams)
print("Multiplier length: %d" % len(param_multiplier))
print("[INFO] Multiplier length: %d" % len(param_multiplier))
for subparams in param_multiplier:
tmp = await func(**subparams)
try:
tmp = await func(**subparams)
except:
tmp = "An error occured for value %s" % subparams
print("Return from execution: %s" % ret)
print("RET from execution: %s" % ret)
new_value = tmp
if tmp == None:
ret.append("")
new_value = ""
elif isinstance(tmp, dict):
ret.append(tmp)
new_value = json.dumps(tmp)
elif isinstance(tmp, list):
ret.append(tmp)
else:
#tmp = tmp.replace("\"", "\\\"", -1)
new_value = json.dumps(tmp)
#else:
#tmp = tmp.replace("\"", "\\\"", -1)
try:
ret.append(json.loads(tmp))
except json.decoder.JSONDecodeError as e:
try:
new_value = json.loads(new_value)
except json.decoder.JSONDecodeError as e:
pass
except TypeError as e:
pass
except:
pass
#print("Json: %s" % e)
ret.append(tmp)
#ret.append(tmp)
#if self.result_wrapper_count > 0:
# ret.append("["*(self.result_wrapper_count-1)+new_value+"]"*(self.result_wrapper_count-1))
#else:
ret.append(new_value)
print("Ret length: %d" % len(ret))
if len(ret) == 1:
@@ -293,6 +336,10 @@ class AppBase:
except json.decoder.JSONDecodeError as e:
#print("Json: %s" % e)
results.append(ret)
except TypeError as e:
results.append(ret)
except:
results.append(ret)
if len(results) == 1:
results = results[0]
@@ -424,7 +471,6 @@ class AppBase:
upload_path = "/api/v1/files/%s/upload?execution_id=%s" % (cur_id, full_execution["execution_id"])
print("Create path: %s" % create_path)
# FIXME: Typical failure here if data is returned badly formatted
files={"shuffle_file": (filename, curfile["data"])}
#open(filename,'rb')}
@@ -436,8 +482,6 @@ class AppBase:
return file_ids
async def execute_action(self, action):
# FIXME - add request for the function STARTING here. Use "results stream" or something
# PAUSED, AWAITING_DATA, PENDING, COMPLETED, ABORTED, EXECUTING, SUCCESS, FAILURE
# !!! Let this line stay - its used for some horrible codegeneration / stitching !!! #
#STARTCOPY
@@ -451,7 +495,7 @@ class AppBase:
"status": "EXECUTING"
}
self.action = action
self.action = copy.deepcopy(action)
self.logger.info("ACTION RESULT (start): %s", action_result)
if len(self.action) == 0:
@@ -809,8 +853,7 @@ class AppBase:
return newvalue, True
elif len(actualitem) > 0:
# FIXME: This is absolutely not perfect.
print("In recursion v2: ", actualitem)
print("[INFO] In recursion v2: ", actualitem)
is_loop = True
newvalue = []
@@ -819,13 +862,14 @@ class AppBase:
# Means it's a single item -> continue
if seconditem == "":
print("In first - handling %s" % seconditem)
print("[INFO] In first - handling %s" % firstitem)
tmpitem = basejson[int(firstitem)]
try:
newvalue, is_loop = recurse_json(tmpitem, parsersplit[outercnt+1:])
except IndexError:
newvalue, is_loop = (tmpitem, parsersplit[outercnt+1:])
else:
print("[INFO] In ELSE - handling %s and %s" % (firstitem, seconditem))
if seconditem == "max":
seconditem = len(basejson)
if seconditem == "min":
@@ -850,7 +894,6 @@ class AppBase:
return newvalue, is_loop
# FIXME: Add specific loop for other indexes
else:
#print("BEFORE NORMAL VALUE: ", basejson, value)
if len(value) == 0:
@@ -1603,7 +1646,7 @@ class AppBase:
#})
print("[INFO] APP_SDK DONE: Starting NORMAL execution of function")
#print("[INFO] Running with params (0): %s" % params)
print("[INFO] Running with params (0): %s" % params)
newres = await func(**params)
print("[INFO] Returned from execution.")
if isinstance(newres, tuple):
backend/app_sdk/build.sh
+1 -1
View File
@@ -1,6 +1,6 @@
#!/bin/bash
NAME=shuffle-app_sdk
VERSION=0.8.43
VERSION=0.8.45
docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force
docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
backend/go-app/docker.go
+105
View File
@@ -788,3 +788,108 @@ func hookTest() {
log.Printf("Success! - %s", returnHook.Id)
}
}
//https://stackoverflow.com/questions/23935141/how-to-copy-docker-images-from-one-host-to-another-without-using-a-repository
func getDockerImage(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
// Just here to verify that the user is logged in
_, err := handleApiAuthentication(resp, request)
if err != nil {
log.Printf("Api authentication failed in validate swagger: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`))
return
}
type requestCheck struct {
Name string `datastore:"name" json:"name" yaml:"name"`
}
//body = []byte(`swagger: "2.0"`)
//body = []byte(`swagger: '1.0'`)
//newbody := string(body)
//newbody = strings.TrimSpace(newbody)
//body = []byte(newbody)
//log.Println(string(body))
//tmpbody, err := yaml.YAMLToJSON(body)
//log.Println(err)
//log.Println(string(tmpbody))
// This has to be done in a weird way because Datastore doesn't
// support map[string]interface and similar (openapi3.Swagger)
var version requestCheck
err = json.Unmarshal(body, &version)
if err != nil {
resp.WriteHeader(422)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed JSON marshalling: %s"}`, err)))
return
}
log.Printf("Image to load: %s", version.Name)
//cli, err := client.NewEnvClient()
//if err != nil {
// log.Println("Unable to create docker client")
// return err
//}
dockercli, err := client.NewEnvClient()
if err != nil {
log.Printf("Unable to create docker client: %s", err)
resp.WriteHeader(422)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed JSON marshalling: %s"}`, err)))
return
}
ctx := context.Background()
images, err := dockercli.ImageList(ctx, types.ImageListOptions{
All: true,
})
img := types.ImageSummary{}
tagFound := ""
for _, image := range images {
for _, tag := range image.RepoTags {
log.Printf("Image: %s", tag)
if strings.ToLower(tag) == strings.ToLower(version.Name) {
img = image
tagFound = tag
break
}
}
}
if len(img.ID) == 0 {
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "Couldn't find image %s"}`, version.Name)))
return
}
_ = tagFound
/*
log.Printf("IMg: %#v", img)
pullOptions := types.ImagePullOptions{}
log.Printf("[INFO] Pulling image %s", image)
reader, err := dockercli.ImagePull(ctx, tag, pullOptions)
if err != nil {
log.Printf("[ERROR] Failed getting image %s: %s", image, err)
}
io.Copy(os.Stdout, r)
*/
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "message": "Downloading image %s"}`, version.Name)))
}
backend/go-app/main.go
+3
View File
@@ -8214,6 +8214,9 @@ func initHandlers() {
r.HandleFunc("/api/v1/orgs/{orgId}", handleEditOrg).Methods("POST", "OPTIONS")
//r.HandleFunc("/api/v1/orgs/{orgId}", handleEditOrg).Methods("POST", "OPTIONS")
// Docker orborus specific
//r.HandleFunc("/api/v1/get_docker_image", getDockerImage).Methods("POST", "OPTIONS")
// Important for email, IDS etc. Create this by:
// PS: For cloud, this has to use cloud storage.
// https://developer.box.com/reference/get-files-id-content/
backend/go-app/walkoff.go
+21 -2
View File
@@ -1244,12 +1244,16 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl
if result.Action.Name == "User Input" && result.Action.AppName == "User Input" {
log.Printf("Found User Input node - prepare cloud?")
extraInputs += 1
} else if result.Action.Name == "run_subflow" && result.Action.AppName == "shuffle-subflow" {
log.Printf("[INFO] Found Shuffle Workflow node")
extraInputs += 1
}
}
//log.Printf("LENGTH: %d - %d", len(workflowExecution.Results), len(workflowExecution.Workflow.Actions))
if len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions)+extraInputs {
log.Printf("\nIN HERE WITH RESULTS %d vs %d\n", len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)+extraInputs)
finished := true
lastResult := ""
@@ -2065,6 +2069,8 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
} else if schedule.Id == "" {
trigger.Status = "stopped"
}
} else if trigger.TriggerType == "SUBFLOW" {
//log.Printf("Found subflow: %#v", trigger.Parameters)
} else if trigger.TriggerType == "WEBHOOK" && trigger.Status != "uninitialized" {
hook, err := getHook(ctx, trigger.ID)
if err != nil {
@@ -2718,7 +2724,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf
var execution ExecutionRequest
err = json.Unmarshal(body, &execution)
if err != nil {
log.Printf("Failed execution POST unmarshaling - continuing anyway: %s", err)
log.Printf("[WARNING] Failed execution POST unmarshaling - continuing anyway: %s", err)
//return WorkflowExecution{}, "", err
}
@@ -4689,9 +4695,10 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) {
//log.Printf("Length: %d", len(workflowapps))
// FIXME - this is really garbage, but is here to protect again null values etc.
skipApps := []string{"Shuffle Subflow"}
newapps := []WorkflowApp{}
baseApps := []WorkflowApp{}
for _, workflowapp := range workflowapps {
if !workflowapp.Activated && workflowapp.Generated {
continue
@@ -4701,6 +4708,18 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) {
continue
}
continueOuter := false
for _, skip := range skipApps {
if workflowapp.Name == skip {
continueOuter = true
break
}
}
if continueOuter {
continue
}
//workflowapp.Environment = "cloud"
newactions := []WorkflowAppAction{}
for _, action := range workflowapp.Actions {
docker-compose.yml
+3 -3
View File
@@ -45,7 +45,7 @@ services:
- database
orborus:
#build: ./functions/onprem/orborus
image: ghcr.io/frikky/shuffle-orborus:0.8.40
image: ghcr.io/frikky/shuffle-orborus:0.8.42
container_name: shuffle-orborus
hostname: shuffle-orborus
networks:
@@ -53,8 +53,8 @@ services:
volumes:
- /var/run/docker.sock:/var/run/docker.sock
environment:
- SHUFFLE_APP_SDK_VERSION=0.8.3
- SHUFFLE_WORKER_VERSION=0.8.3
- SHUFFLE_APP_SDK_VERSION=0.8.45
- SHUFFLE_WORKER_VERSION=0.8.4
- ORG_ID=${ORG_ID}
- ENVIRONMENT_NAME=${ENVIRONMENT_NAME}
- BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT}
frontend/src/defaultCytoscapeStyle.js
+2
View File
@@ -74,6 +74,8 @@ const data = [{
'shape': 'octagon',
'border-color': 'orange',
'background-color': '#213243',
'background-width': '100%',
'background-height': '100%',
},
},
{
frontend/src/views/AngularWorkflow.jsx
+271 -22
View File
File diff suppressed because one or more lines are too long
frontend/src/views/SettingsPage.jsx
+2 -2
View File
@@ -85,7 +85,7 @@ const Settings = (props) => {
const generateApikey = () => {
fetch(globalUrl+"/api/v1/generateapikey", {
method: 'GET',
method: 'GET',
headers: {
'Content-Type': 'application/json',
'Accept': 'application/json',
@@ -99,7 +99,7 @@ const Settings = (props) => {
return response.json()
})
.then((responseJson) => {
.then((responseJson) => {
setUserSettings(responseJson)
})
.catch(error => {
functions/onprem/orborus/build.sh
+1 -1
View File
@@ -1,5 +1,5 @@
NAME=shuffle-orborus
VERSION=0.8.41
VERSION=0.8.42
echo "Running docker build with $NAME:$VERSION"
#docker rmi frikky/shuffle:$NAME --force
functions/onprem/orborus/orborus.go
+12 -4
View File
@@ -244,11 +244,11 @@ func initializeImages() {
ctx := context.Background()
if appSdkVersion == "" {
appSdkVersion = "0.8.3"
appSdkVersion = "0.8.45"
log.Printf("[WARNING] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %s", appSdkVersion)
}
if workerVersion == "" {
workerVersion = "0.8.3"
workerVersion = "0.8.41"
log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %s", workerVersion)
}
@@ -633,6 +633,8 @@ func zombiecheck(ctx context.Context, workerTimeout int) error {
All: true,
})
log.Printf("Len: %d", len(containers))
if err != nil {
log.Printf("[ERROR] Failed creating Containerlist: %s", err)
return err
@@ -642,7 +644,7 @@ func zombiecheck(ctx context.Context, workerTimeout int) error {
stopContainers := []string{}
removeContainers := []string{}
log.Printf("Workertimeout: %d", int64(workerTimeout))
log.Printf("[INFO] Baseimage: %s, Workertimeout: %d", baseimagename, int64(workerTimeout))
for _, container := range containers {
// Skip random containers. Only handle things related to Shuffle.
if !strings.Contains(container.Image, baseimagename) {
@@ -656,10 +658,14 @@ func zombiecheck(ctx context.Context, workerTimeout int) error {
// Check image name
if !shuffleFound {
log.Printf("Skipping: %s, %s", container.Labels, container.Image)
continue
}
//} else {
// log.Printf("NAME: %s", container.Image)
} else {
//log.Printf("Img: %s", container.Image)
//log.Printf("Names: %s", container.Names)
}
for _, name := range container.Names {
@@ -673,6 +679,7 @@ func zombiecheck(ctx context.Context, workerTimeout int) error {
// Need to check time here too because a container can be removed the same instant as its created
if container.State != "running" && currenttime-container.Created > int64(workerTimeout) {
log.Printf("Should remove above container because not running and old")
removeContainers = append(removeContainers, container.ID)
containerNames[container.ID] = name
}
@@ -680,6 +687,7 @@ func zombiecheck(ctx context.Context, workerTimeout int) error {
// stopcontainer & removecontainer
//log.Printf("Time: %d - %d", currenttime-container.Created, int64(workerTimeout))
if container.State == "running" && currenttime-container.Created > int64(workerTimeout) {
log.Printf("Should remove above container because RUNNING and old")
stopContainers = append(stopContainers, container.ID)
containerNames[container.ID] = name
}
@@ -701,7 +709,7 @@ func zombiecheck(ctx context.Context, workerTimeout int) error {
log.Printf("[INFO] Should REMOVE %d containers.", len(removeContainers))
for _, containername := range removeContainers {
go dockercli.ContainerRemove(ctx, containername, removeOptions)
dockercli.ContainerRemove(ctx, containername, removeOptions)
}
return nil
functions/onprem/worker/build.sh
+3 -3
View File
@@ -1,12 +1,12 @@
NAME=shuffle-worker
VERSION=0.8.3
VERSION=0.8.41
echo "Running docker build with $NAME:$VERSION"
#CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin .
docker build . -t frikky/shuffle:$NAME -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
docker build . -t frikky/shuffle:$NAME -t frikky/shuffle:$NAME_$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
# Push both for now..
#docker push frikky/$NAME:$VERSION
docker push frikky/shuffle:$NAME
#docker push frikky/shuffle:$NAME_$VERSION
#docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION
docker push ghcr.io/frikky/$NAME:$VERSION