FEATURE: Added sub-workflow trigger

backend/app_sdk/app_base.py

@@ -25,6 +25,7 @@ class AppBase:
         self.authorization = os.getenv("AUTHORIZATION", "")
         self.current_execution_id = os.getenv("EXECUTIONID", "")
         self.full_execution = os.getenv("FULL_EXECUTION", "") 
+        self.result_wrapper_count = 0
 
         if isinstance(self.action, str):
             self.action = json.loads(self.action)
@@ -111,22 +112,52 @@ class AppBase:
         #check_value = "$Filter_list_testing.wrapper.#.tmp"
         #self.action = action
 
-
+        loopnames = []
         for key, value in baseparams.items():
             check_value = ""
             for param in self.action["parameters"]:
                 if param["name"] == key:
+                    #print("PARAM: %s" % param)
                     check_value = param["value"]
-                    break
+                    # self.result_wrapper_count = 0
+
+                octothorpe_count = param["value"].count(".#")
+                if octothorpe_count > self.result_wrapper_count:
+                    self.result_wrapper_count = octothorpe_count
+                    print("NEW OCTOTHORPE WRAPPER: %d" % octothorpe_count)
+
+            # This whole thing is hard.
+            # item = [{"data": "1.2.3.4", "dataType": "ip"}] 
+            # $item         = DONT loop items. 
+            # $item.#       = Loop items
+            # $item.#.data  = Loop items
+            # With a single item, this is fine.
+
+            # item = [{"list": [{"data": "1.2.3.4", "dataType": "ip"}]}] 
+            # $item                 = DONT loop items
+            # $item.#               = Loop items
+            # $item.#.list          = DONT loop items
+            # $item.#.list.#        = Loop items
+            # $item.#.list.#.data   = Loop items
+            # If the item itself is a list.. hmm
+            
+            # FIXME: Check the above, and fix so that nested looped items can be 
+            # Skipped if wanted
 
             print("\nCHECK: %s" % check_value)
+            should_merge = False
+            if "#" in check_value:
+                should_merge = True
+
             if isinstance(value, list):
                 if len(value) <= 1:
                     if len(value) == 1:
                         baseparams[key] = value[0]
 
+                    #if "#" in check_value:
+                    #    should_merge = True
                 else:
-                    if not check_value.endswith("#"):
+                    if not should_merge: 
                         print("Adding WITHOUT looping list")
                     else:
                         if len(value) not in listlengths:
@@ -239,36 +270,48 @@ class AppBase:
         
         results = []
         if has_loop:
-            print("Should run inner loop: %s" % newparams)
+            print("[WARNING] Should run inner loop: %s" % newparams)
             ret = await self.run_recursed_items(func, newparams, loop_wrapper)
-
         else:
-            print("Should run with params (inner): %s" % newparams)
+            print("[INFO] Should run multiplier check with params (inner): %s" % newparams)
-
             # 1. Find the loops that are required and create new multipliers
             # If here: check for multipliers within this scope.
             ret = []
             param_multiplier = await self.get_param_multipliers(newparams)
 
-            print("Multiplier length: %d" % len(param_multiplier))
+            print("[INFO] Multiplier length: %d" % len(param_multiplier))
             for subparams in param_multiplier:
-                tmp = await func(**subparams)
+                try:
+                    tmp = await func(**subparams)
+                except:
+                    tmp = "An error occured for value %s" % subparams
 
-                print("Return from execution: %s" % ret)
+                print("RET from execution: %s" % ret)
+                new_value = tmp
                 if tmp == None:
-                    ret.append("")
+                    new_value = ""
                 elif isinstance(tmp, dict):
-                    ret.append(tmp)
+                    new_value = json.dumps(tmp)
                 elif isinstance(tmp, list):
-                    ret.append(tmp)
+                    new_value = json.dumps(tmp)
-                else:
+                #else:
-                    #tmp = tmp.replace("\"", "\\\"", -1)
+                #tmp = tmp.replace("\"", "\\\"", -1)
 
-                    try:
+                try:
-                        ret.append(json.loads(tmp))
+                    new_value = json.loads(new_value)
-                    except json.decoder.JSONDecodeError as e:
+                except json.decoder.JSONDecodeError as e:
+                    pass
+                except TypeError as e:
+                    pass
+                except:
+                    pass
                         #print("Json: %s" % e)
-                        ret.append(tmp)
+                        #ret.append(tmp)
+                
+                #if self.result_wrapper_count > 0:
+                #    ret.append("["*(self.result_wrapper_count-1)+new_value+"]"*(self.result_wrapper_count-1))
+                #else:
+                ret.append(new_value)
 
             print("Ret length: %d" % len(ret))
             if len(ret) == 1:
@@ -293,6 +336,10 @@ class AppBase:
             except json.decoder.JSONDecodeError as e:
                 #print("Json: %s" % e)
                 results.append(ret)
+            except TypeError as e:
+                results.append(ret)
+            except:
+                results.append(ret)
 
         if len(results) == 1: 
             results = results[0]
@@ -424,7 +471,6 @@ class AppBase:
             upload_path = "/api/v1/files/%s/upload?execution_id=%s" % (cur_id, full_execution["execution_id"])
             print("Create path: %s" % create_path)
 
-            # FIXME: Typical failure here if data is returned badly formatted
             files={"shuffle_file": (filename, curfile["data"])}
             #open(filename,'rb')}
 
@@ -436,8 +482,6 @@ class AppBase:
         return file_ids
     
     async def execute_action(self, action):
-        # FIXME - add request for the function STARTING here. Use "results stream" or something
-        # PAUSED, AWAITING_DATA, PENDING, COMPLETED, ABORTED, EXECUTING, SUCCESS, FAILURE
 
         # !!! Let this line stay - its used for some horrible codegeneration / stitching !!! # 
         #STARTCOPY
@@ -451,7 +495,7 @@ class AppBase:
             "status": "EXECUTING"
         }
 
-        self.action = action
+        self.action = copy.deepcopy(action)
         self.logger.info("ACTION RESULT (start): %s", action_result)
 
         if len(self.action) == 0:
@@ -809,8 +853,7 @@ class AppBase:
                         return newvalue, True
 
                     elif len(actualitem) > 0:
-                        # FIXME: This is absolutely not perfect. 
+                        print("[INFO] In recursion v2: ", actualitem)
-                        print("In recursion v2: ", actualitem)
 
                         is_loop = True
                         newvalue = []
@@ -819,13 +862,14 @@ class AppBase:
 
                         # Means it's a single item -> continue
                         if seconditem == "":
-                            print("In first - handling %s" % seconditem)
+                            print("[INFO] In first - handling %s" % firstitem)
                             tmpitem = basejson[int(firstitem)]
                             try:
                                 newvalue, is_loop = recurse_json(tmpitem, parsersplit[outercnt+1:])
                             except IndexError:
                                 newvalue, is_loop = (tmpitem, parsersplit[outercnt+1:])
                         else:
+                            print("[INFO] In ELSE - handling %s and %s" % (firstitem, seconditem))
                             if seconditem == "max": 
                                 seconditem = len(basejson)
                             if seconditem == "min": 
@@ -850,7 +894,6 @@ class AppBase:
 
                         return newvalue, is_loop 
 
-                    # FIXME: Add specific loop for other indexes
                     else:
                         #print("BEFORE NORMAL VALUE: ", basejson, value)
                         if len(value) == 0:
@@ -1603,7 +1646,7 @@ class AppBase:
                             #})
 
                             print("[INFO] APP_SDK DONE: Starting NORMAL execution of function")
-                            #print("[INFO] Running with params (0): %s" % params) 
+                            print("[INFO] Running with params (0): %s" % params) 
                             newres = await func(**params)
                             print("[INFO] Returned from execution.")
                             if isinstance(newres, tuple):

backend/app_sdk/build.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 NAME=shuffle-app_sdk
-VERSION=0.8.43
+VERSION=0.8.45
 
 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force
 docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION

backend/go-app/docker.go

@@ -788,3 +788,108 @@ func hookTest() {
 		log.Printf("Success! - %s", returnHook.Id)
 	}
 }
+
+//https://stackoverflow.com/questions/23935141/how-to-copy-docker-images-from-one-host-to-another-without-using-a-repository
+func getDockerImage(resp http.ResponseWriter, request *http.Request) {
+	cors := handleCors(resp, request)
+	if cors {
+		return
+	}
+
+	// Just here to verify that the user is logged in
+	_, err := handleApiAuthentication(resp, request)
+	if err != nil {
+		log.Printf("Api authentication failed in validate swagger: %s", err)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false}`))
+		return
+	}
+
+	body, err := ioutil.ReadAll(request.Body)
+	if err != nil {
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`))
+		return
+	}
+
+	type requestCheck struct {
+		Name string `datastore:"name" json:"name" yaml:"name"`
+	}
+
+	//body = []byte(`swagger: "2.0"`)
+	//body = []byte(`swagger: '1.0'`)
+	//newbody := string(body)
+	//newbody = strings.TrimSpace(newbody)
+	//body = []byte(newbody)
+	//log.Println(string(body))
+	//tmpbody, err := yaml.YAMLToJSON(body)
+	//log.Println(err)
+	//log.Println(string(tmpbody))
+
+	// This has to be done in a weird way because Datastore doesn't
+	// support map[string]interface and similar (openapi3.Swagger)
+	var version requestCheck
+
+	err = json.Unmarshal(body, &version)
+	if err != nil {
+		resp.WriteHeader(422)
+		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed JSON marshalling: %s"}`, err)))
+		return
+	}
+
+	log.Printf("Image to load: %s", version.Name)
+	//cli, err := client.NewEnvClient()
+	//if err != nil {
+	//	log.Println("Unable to create docker client")
+	//	return err
+	//}
+
+	dockercli, err := client.NewEnvClient()
+	if err != nil {
+		log.Printf("Unable to create docker client: %s", err)
+		resp.WriteHeader(422)
+		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed JSON marshalling: %s"}`, err)))
+		return
+	}
+
+	ctx := context.Background()
+	images, err := dockercli.ImageList(ctx, types.ImageListOptions{
+		All: true,
+	})
+
+	img := types.ImageSummary{}
+	tagFound := ""
+	for _, image := range images {
+		for _, tag := range image.RepoTags {
+			log.Printf("Image: %s", tag)
+
+			if strings.ToLower(tag) == strings.ToLower(version.Name) {
+				img = image
+				tagFound = tag
+				break
+			}
+		}
+	}
+
+	if len(img.ID) == 0 {
+		resp.WriteHeader(401)
+		resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "Couldn't find image %s"}`, version.Name)))
+		return
+	}
+	_ = tagFound
+
+	/*
+		log.Printf("IMg: %#v", img)
+		pullOptions := types.ImagePullOptions{}
+		log.Printf("[INFO] Pulling image %s", image)
+		reader, err := dockercli.ImagePull(ctx, tag, pullOptions)
+		if err != nil {
+			log.Printf("[ERROR] Failed getting image %s: %s", image, err)
+		}
+
+		io.Copy(os.Stdout, r)
+	*/
+
+	resp.WriteHeader(200)
+	resp.Write([]byte(fmt.Sprintf(`{"success": true, "message": "Downloading image %s"}`, version.Name)))
+}

backend/go-app/main.go

@@ -8214,6 +8214,9 @@ func initHandlers() {
 	r.HandleFunc("/api/v1/orgs/{orgId}", handleEditOrg).Methods("POST", "OPTIONS")
 	//r.HandleFunc("/api/v1/orgs/{orgId}", handleEditOrg).Methods("POST", "OPTIONS")
 
+	// Docker orborus specific
+	//r.HandleFunc("/api/v1/get_docker_image", getDockerImage).Methods("POST", "OPTIONS")
+
 	// Important for email, IDS etc. Create this by:
 	// PS: For cloud, this has to use cloud storage.
 	// https://developer.box.com/reference/get-files-id-content/

backend/go-app/walkoff.go

@@ -1244,12 +1244,16 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl
 		if result.Action.Name == "User Input" && result.Action.AppName == "User Input" {
 			log.Printf("Found User Input node - prepare cloud?")
 			extraInputs += 1
+		} else if result.Action.Name == "run_subflow" && result.Action.AppName == "shuffle-subflow" {
+			log.Printf("[INFO] Found Shuffle Workflow node")
+			extraInputs += 1
 		}
 	}
 
 	//log.Printf("LENGTH: %d - %d", len(workflowExecution.Results), len(workflowExecution.Workflow.Actions))
 
 	if len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions)+extraInputs {
+		log.Printf("\nIN HERE WITH RESULTS %d vs %d\n", len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)+extraInputs)
 		finished := true
 		lastResult := ""
 
@@ -2065,6 +2069,8 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
 			} else if schedule.Id == "" {
 				trigger.Status = "stopped"
 			}
+		} else if trigger.TriggerType == "SUBFLOW" {
+			//log.Printf("Found subflow: %#v", trigger.Parameters)
 		} else if trigger.TriggerType == "WEBHOOK" && trigger.Status != "uninitialized" {
 			hook, err := getHook(ctx, trigger.ID)
 			if err != nil {
@@ -2718,7 +2724,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf
 		var execution ExecutionRequest
 		err = json.Unmarshal(body, &execution)
 		if err != nil {
-			log.Printf("Failed execution POST unmarshaling - continuing anyway: %s", err)
+			log.Printf("[WARNING] Failed execution POST unmarshaling - continuing anyway: %s", err)
 			//return WorkflowExecution{}, "", err
 		}
 
@@ -4689,9 +4695,10 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) {
 	//log.Printf("Length: %d", len(workflowapps))
 
 	// FIXME - this is really garbage, but is here to protect again null values etc.
+
+	skipApps := []string{"Shuffle Subflow"}
 	newapps := []WorkflowApp{}
 	baseApps := []WorkflowApp{}
-
 	for _, workflowapp := range workflowapps {
 		if !workflowapp.Activated && workflowapp.Generated {
 			continue
@@ -4701,6 +4708,18 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) {
 			continue
 		}
 
+		continueOuter := false
+		for _, skip := range skipApps {
+			if workflowapp.Name == skip {
+				continueOuter = true
+				break
+			}
+		}
+
+		if continueOuter {
+			continue
+		}
+
 		//workflowapp.Environment = "cloud"
 		newactions := []WorkflowAppAction{}
 		for _, action := range workflowapp.Actions {

docker-compose.yml

@@ -45,7 +45,7 @@ services:
       - database
   orborus:
     #build: ./functions/onprem/orborus
-    image: ghcr.io/frikky/shuffle-orborus:0.8.40
+    image: ghcr.io/frikky/shuffle-orborus:0.8.42
     container_name: shuffle-orborus
     hostname: shuffle-orborus
     networks:
@@ -53,8 +53,8 @@ services:
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
     environment:
-      - SHUFFLE_APP_SDK_VERSION=0.8.3
+      - SHUFFLE_APP_SDK_VERSION=0.8.45
-      - SHUFFLE_WORKER_VERSION=0.8.3
+      - SHUFFLE_WORKER_VERSION=0.8.4
       - ORG_ID=${ORG_ID}
       - ENVIRONMENT_NAME=${ENVIRONMENT_NAME}
       - BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT}

frontend/src/defaultCytoscapeStyle.js

@@ -74,6 +74,8 @@ const data = [{
 				'shape': 'octagon',
 				'border-color': 'orange',
 				'background-color': '#213243',
+				'background-width': '100%',
+				'background-height': '100%',
 			},
 		},
 		{

frontend/src/views/SettingsPage.jsx

@@ -85,7 +85,7 @@ const Settings = (props) => {
 
 	const generateApikey = () => {
 		fetch(globalUrl+"/api/v1/generateapikey", {
-    	  	method: 'GET',
+    	  method: 'GET',
 				headers: {
 					'Content-Type': 'application/json',
 					'Accept': 'application/json',
@@ -99,7 +99,7 @@ const Settings = (props) => {
 
 			return response.json()
 		})
-    	.then((responseJson) => {
+    .then((responseJson) => {
 			setUserSettings(responseJson)
     	})
 		.catch(error => {

functions/onprem/orborus/build.sh

@@ -1,5 +1,5 @@
 NAME=shuffle-orborus
-VERSION=0.8.41
+VERSION=0.8.42
 
 echo "Running docker build with $NAME:$VERSION"
 #docker rmi frikky/shuffle:$NAME --force

functions/onprem/orborus/orborus.go

@@ -244,11 +244,11 @@ func initializeImages() {
 	ctx := context.Background()
 
 	if appSdkVersion == "" {
-		appSdkVersion = "0.8.3"
+		appSdkVersion = "0.8.45"
 		log.Printf("[WARNING] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %s", appSdkVersion)
 	}
 	if workerVersion == "" {
-		workerVersion = "0.8.3"
+		workerVersion = "0.8.41"
 		log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %s", workerVersion)
 	}
 
@@ -633,6 +633,8 @@ func zombiecheck(ctx context.Context, workerTimeout int) error {
 		All: true,
 	})
 
+	log.Printf("Len: %d", len(containers))
+
 	if err != nil {
 		log.Printf("[ERROR] Failed creating Containerlist: %s", err)
 		return err
@@ -642,7 +644,7 @@ func zombiecheck(ctx context.Context, workerTimeout int) error {
 
 	stopContainers := []string{}
 	removeContainers := []string{}
-	log.Printf("Workertimeout: %d", int64(workerTimeout))
+	log.Printf("[INFO] Baseimage: %s, Workertimeout: %d", baseimagename, int64(workerTimeout))
 	for _, container := range containers {
 		// Skip random containers. Only handle things related to Shuffle.
 		if !strings.Contains(container.Image, baseimagename) {
@@ -656,10 +658,14 @@ func zombiecheck(ctx context.Context, workerTimeout int) error {
 
 			// Check image name
 			if !shuffleFound {
+				log.Printf("Skipping: %s, %s", container.Labels, container.Image)
 				continue
 			}
 			//} else {
 			//	log.Printf("NAME: %s", container.Image)
+		} else {
+			//log.Printf("Img: %s", container.Image)
+			//log.Printf("Names: %s", container.Names)
 		}
 
 		for _, name := range container.Names {
@@ -673,6 +679,7 @@ func zombiecheck(ctx context.Context, workerTimeout int) error {
 
 			// Need to check time here too because a container can be removed the same instant as its created
 			if container.State != "running" && currenttime-container.Created > int64(workerTimeout) {
+				log.Printf("Should remove above container because not running and old")
 				removeContainers = append(removeContainers, container.ID)
 				containerNames[container.ID] = name
 			}
@@ -680,6 +687,7 @@ func zombiecheck(ctx context.Context, workerTimeout int) error {
 			// stopcontainer & removecontainer
 			//log.Printf("Time: %d - %d", currenttime-container.Created, int64(workerTimeout))
 			if container.State == "running" && currenttime-container.Created > int64(workerTimeout) {
+				log.Printf("Should remove above container because RUNNING and old")
 				stopContainers = append(stopContainers, container.ID)
 				containerNames[container.ID] = name
 			}
@@ -701,7 +709,7 @@ func zombiecheck(ctx context.Context, workerTimeout int) error {
 
 	log.Printf("[INFO] Should REMOVE %d containers.", len(removeContainers))
 	for _, containername := range removeContainers {
-		go dockercli.ContainerRemove(ctx, containername, removeOptions)
+		dockercli.ContainerRemove(ctx, containername, removeOptions)
 	}
 
 	return nil

functions/onprem/worker/build.sh

@@ -1,12 +1,12 @@
 NAME=shuffle-worker
-VERSION=0.8.3
+VERSION=0.8.41
 
 echo "Running docker build with $NAME:$VERSION"
 #CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin .
-docker build . -t frikky/shuffle:$NAME -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
+docker build . -t frikky/shuffle:$NAME -t frikky/shuffle:$NAME_$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
 
 # Push both for now..
 #docker push frikky/$NAME:$VERSION 
-docker push frikky/shuffle:$NAME
+#docker push frikky/shuffle:$NAME_$VERSION
 #docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION
 docker push ghcr.io/frikky/$NAME:$VERSION