Merge branch 'launch' into master

.env

@@ -30,6 +30,7 @@ SHUFFLE_FILE_LOCATION=./shuffle-files
 SHUFFLE_ENCRYPTION_MODIFIER=
 
 # Other configs
+BASE_URL=http://shuffle-backend:5001 
 BACKEND_HOSTNAME=shuffle-backend
 BACKEND_PORT=5001
 FRONTEND_PORT=3001

backend/app_sdk/app_base.py

@@ -4,16 +4,18 @@ import sys
 import re
 import time 
 import json
+import liquid
 import logging
-import requests
-import urllib.parse
-import http.client
 import urllib3
 import hashlib
-from liquid import Liquid
-import liquid
 import zipfile
+import requests
+import http.client
+import urllib.parse
 from io import BytesIO
+from liquid import Liquid
+
+runtime = os.getenv("SHUFFLE_SWARM_CONFIG", "")
 
 class AppBase:
     __version__ = None
@@ -27,7 +29,7 @@ class AppBase:
         # apikey is for the user / org
         # authorization is for the specific workflow
 
-        self.url = os.getenv("CALLBACK_URL", "https://shuffler.io")
+        self.url = os.getenv("CALLBACK_URL",  "https://shuffler.io")
         self.base_url = os.getenv("BASE_URL", "https://shuffler.io")
         self.action = os.getenv("ACTION", "")
         self.original_action = os.getenv("ACTION", "")
@@ -51,8 +53,10 @@ class AppBase:
             try:
                 self.action = json.loads(self.action)
                 self.original_action = json.loads(self.action)
-            except:
+            except Exception as e:
-                self.logger.info("[WARNING] Failed parsing action as JSON")
+                self.logger.info(f"[WARNING] Failed parsing action as JSON (init): {e}. NOT important if running apps with webserver")
+
+        #print(f"ACTION: {self.action}")
 
         if len(self.base_url) == 0:
             self.base_url = self.url
@@ -80,14 +84,14 @@ class AppBase:
         # I wonder if this actually works 
         self.logger.info(f"[DEBUG] Before last stream result")
         url = "%s%s" % (self.base_url, stream_path)
-        #self.logger.info("[INFO] URL (URL): %s" % url)
+        self.logger.info("[INFO] URL FOR RESULT (URL): %s" % url)
         try:
             ret = requests.post(url, headers=headers, json=action_result)
             #self.logger.info(f"[DEBUG] Result: {ret.status_code}")
             #if ret.status_code != 200:
             #    self.logger.info(f"[DEBUG] Shuffle Response: {ret.text}")
         
-            self.logger.info(f"[DEBUG] Successful request: Status= {ret.status_code} & Response= {ret.text}")
+            self.logger.info(f"""[DEBUG] Successful request result request: Status= {ret.status_code} & Response= {ret.text}. Action status: {action_result["status"]}""")
         except requests.exceptions.ConnectionError as e:
             self.logger.info(f"[DEBUG] Unexpected ConnectionError happened: {e}")
             return
@@ -520,8 +524,10 @@ class AppBase:
                 }
 
                 self.send_result(self.action_result, {"Content-Type": "application/json", "Authorization": "Bearer %s" % self.authorization}, "/api/v1/streams")
-                exit()
+                if runtime != "run":
-                #return
+                    exit()
+                else:
+                    return
             else:
                 #subparams = new_params
                 #self.logger.info(f"NEW PARAMS: {new_params}")
@@ -849,8 +855,8 @@ class AppBase:
         self.logger.info("IDS TO RETURN: %s" % file_ids)
         return file_ids
     
-    async def execute_action(self, action):
+    #async def execute_action(self, action):
-
+    def execute_action(self, action):
         # !!! Let this line stay - its used for some horrible codegeneration / stitching !!! # 
         #STARTCOPY
         stream_path = "/api/v1/streams"
@@ -880,19 +886,19 @@ class AppBase:
         }
 
         if len(self.action) == 0:
-            self.logger.info("ACTION env not defined")
+            self.logger.info("[WARNING] ACTION env not defined")
             self.action_result["result"] = "Error in setup ENV: ACTION not defined"
             self.send_result(self.action_result, headers, stream_path) 
             return
 
         if len(self.authorization) == 0:
-            self.logger.info("AUTHORIZATION env not defined")
+            self.logger.info("[WARING] AUTHORIZATION env not defined")
             self.action_result["result"] = "Error in setup ENV: AUTHORIZATION not defined"
             self.send_result(self.action_result, headers, stream_path) 
             return
 
         if len(self.current_execution_id) == 0:
-            self.logger.info("EXECUTIONID env not defined")
+            self.logger.info("[WARNING] EXECUTIONID env not defined")
             self.action_result["result"] = "Error in setup ENV: EXECUTIONID not defined"
             self.send_result(self.action_result, headers, stream_path) 
             return
@@ -918,7 +924,7 @@ class AppBase:
         # Verify whether there are any parameters with ACTION_RESULT required
         # If found, we get the full results list from backend
         fullexecution = {}
-        if len(self.full_execution) == 0:
+        if isinstance(self.full_execution, str) and len(self.full_execution) == 0:
             self.logger.info("[DEBUG] NO EXECUTION - LOADING!")
             try:
                 tmpdata = {
@@ -937,8 +943,8 @@ class AppBase:
                     fullexecution = ret.json()
                 else:
                     try:
-                        self.logger.info("Error: Data: ", ret.json())
+                        self.logger.info("[DEBUG] Error: Data: ", ret.json())
-                        self.logger.info("Error with status code for results. Crashing because ACTION_RESULTS or WORKFLOW_VARIABLE can't be handled. Status: %d" % ret.status_code)
+                        self.logger.info("[DEBUG] Error with status code for results. Crashing because ACTION_RESULTS or WORKFLOW_VARIABLE can't be handled. Status: %d" % ret.status_code)
                     except json.decoder.JSONDecodeError:
                         pass
 
@@ -946,11 +952,12 @@ class AppBase:
                     self.send_result(self.action_result, headers, stream_path) 
                     return
             except requests.exceptions.ConnectionError as e:
-                self.logger.info("Connectionerror: %s" %  e)
+                self.logger.info("[DEBUG] FullExec Connectionerror: %s" %  e)
                 self.action_result["result"] = "Connection error during startup: %s" % e
                 self.send_result(self.action_result, headers, stream_path) 
                 return
         else:
+            self.logger.info(f"[DEBUG] Setting execution to default value with type {type(self.full_execution)}")
             try:
                 fullexecution = json.loads(self.full_execution)
             except json.decoder.JSONDecodeError as e:
@@ -1147,6 +1154,11 @@ class AppBase:
             except TypeError:
                 return data, False
 
+            # Because liquid can handle ALL of this now.
+            # Implemented for >0.9.25
+            self.logger.info("[DEBUG] Skipping parser because use of its been deprecated >0.9.25 due to Liquid implementation")
+            return data, False
+
             wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length", "lenght", "join", "replace"]
 
             if not any(wrapper in data for wrapper in wrappers):
@@ -1219,7 +1231,7 @@ class AppBase:
 
             if isinstance(data, str) and len(data) > 4:
                 if (data[0] == "{" or data[0] == "[") and (data[len(data)-1] == "]" or data[len(data)-1] == "}"):
-                    self.logger.info("Skipping parser because use of {[ and ]}")
+                    self.logger.info("[DEBUG] Skipping parser because use of {[ and ]}")
                     return data
 
             newdata = []
@@ -1588,9 +1600,10 @@ class AppBase:
         # Sending self as it's not a normal function
         def parse_liquid(template, self):
             
-            #self.logger.info("Inside liquid with glob: %s" % globals())
+            errors = False
+            error_msg = ""
             try:
-                if len(template) > 5000000:
+                if len(template) > 10000000:
                     self.logger.info("[DEBUG] Skipping liquid - size too big (%d)" % len(template))
                     return template
 
@@ -1609,21 +1622,41 @@ class AppBase:
                 # Can't handle self yet (?)
                 ret = run.render(**globals())
                 return ret
-                #try:
-                    #run = Liquid(template)
-                    #return ret
-                #except liquid.exceptions.LiquidSyntaxError as  e:
-                #    run = Liquid(template, {'mode': 'python'})
-                #    ret = run.render(**globals())
-                #    return ret
-                #except liquid.exceptions.LiquidRenderError as e:
-                #    self.logger.info("Render error: %s" % e)
             except jinja2.exceptions.TemplateNotFound as e:
-                self.logger.info("[ERROR] Template error: %s" % e)
+                self.logger.info(f"[ERROR] Liquid Template error: {e}")
+                error = True
+                error_msg = e
             except jinja2.exceptions.TemplateSyntaxError as e:
-                self.logger.info("[ERROR] Syntax error: %s" % e)
+                self.logger.info(f"[ERROR] Liquid Syntax error: {e}")
-            except:
+                error = True
-                self.logger.info("[ERROR] General exception for liquid")
+                error_msg = e
+            except Exception as e:
+                self.logger.info(f"[ERROR] General exception for liquid: {e}")
+                error = True
+                error_msg = e
+
+            if error == True:
+                self.action_result["status"] = "FAILURE" 
+                data = {
+                    "success": False,
+                    "input": template,
+                    "reason": f"Failed to parse LiquidPy: {error_msg}",
+                }
+                try:
+                    self.action_result["result"] = json.dumps(data)
+                except Exception as e:
+                    self.action_result["result"] = f"Failed to parse LiquidPy: {error_msg}"
+                    print("[WARNING] Failed to set LiquidPy result")
+
+                self.action_result["completed_at"] = int(time.time())
+                self.send_result(self.action_result, headers, stream_path)
+
+                self.logger.info(f"[ERROR] Sent FAILURE response to backend due to : {e}")
+        
+                if runtime == "run":
+                    return template
+                else:
+                    os.exit()
 
             return template
 
@@ -1983,11 +2016,11 @@ class AppBase:
                 except KeyError:
                     continue
 
-                self.logger.info("Relevant conditions: %s" % branch["conditions"])
+                self.logger.info("[DEBUG] Relevant conditions: %s" % branch["conditions"])
                 successful_conditions = []
                 failed_conditions = []
                 for condition in branch["conditions"]:
-                    self.logger.info("Getting condition value of %s" % condition)
+                    self.logger.info("[DEBUG] Getting condition value of %s" % condition)
 
                     # Parse all values first here
                     sourcevalue = condition["source"]["value"]
@@ -2083,7 +2116,7 @@ class AppBase:
         if " " in actionname:
             actionname.replace(" ", "_", -1) 
 
-
+        #print(action)
         #if action.generated:
         #    actionname = actionname.lower()
 
@@ -2091,13 +2124,14 @@ class AppBase:
         try:
             func = getattr(self, actionname, None)
             if func == None:
-                self.logger.debug(f"Failed executing {actionname} because func is None.")
+                self.logger.debug(f"[DEBUG] Failed executing {actionname} because func is None.")
                 self.action_result["status"] = "FAILURE" 
                 self.action_result["result"] = "Function %s doesn't exist." % actionname
             elif callable(func):
                 try:
                     if len(action["parameters"]) < 1:
-                        result = await func()
+                        #result = await func()
+                        result = func()
                     else:
                         # Potentially parse JSON here
                         # FIXME - add potential authentication as first parameter(s) here
@@ -2438,7 +2472,7 @@ class AppBase:
                                 # This part has fucked over so many random JSON usages because of weird paranthesis parsing
 
                                 value = parse_wrapper_start(value, self)
-                                self.logger.info("[DEBUG] Post return: %s" % value)
+                                #self.logger.info("[DEBUG] Post return: %s" % value)
 
                                 #self.logger.info("POST data value: %s" % value)
                                 params[parameter["name"]] = value
@@ -2529,11 +2563,12 @@ class AppBase:
                             #newres = ""
                             while True:
                                 try:
-                                    newres = await func(**params)
+                                    #newres = await func(**params)
+                                    newres = func(**params)
                                     break
                                 except TypeError as e:
                                     newres = ""
-                                    self.logger.info(f"[DEBUG] Got exec error: {errorstring}")
+                                    self.logger.info(f"[DEBUG] Got exec error: {e}")
                                     errorstring = f"{e}"
                                     if "got an unexpected keyword argument" in errorstring:
                                         fieldsplit = errorstring.split("'")
@@ -2604,7 +2639,8 @@ class AppBase:
 
                             self.logger.info("[INFO] Running WITHOUT outer loop (looping)")
                             json_object = False
-                            results = await self.run_recursed_items(func, multi_parameters, {})
+                            #results = await self.run_recursed_items(func, multi_parameters, {})
+                            results = self.run_recursed_items(func, multi_parameters, {})
                             if isinstance(results, dict) or isinstance(results, list):
                                 json_object = True
 
@@ -2799,34 +2835,141 @@ class AppBase:
         logger = logging.getLogger(f"{cls.__name__}")
         logger.setLevel(logging.DEBUG)
 
-        #self.logger.info("Started execution: %s!!" % cls)
-        #self.logger.info("Action: %s" % action)
-        #if isinstance(cls, object):
-        #    self.action = cls
 
-        app = cls(redis=None, logger=logger, console_logger=logger)
+        ##############################################
-        if isinstance(action, str):
-            print("[DEBUG] Normal execution. Action is a string.")
-        elif isinstance(action, object):
-            print("[DEBUG] OBJECT execution. Action is NOT a string.")
-            app.action = action
 
-            try:
+        exposed_port = os.getenv("SHUFFLE_APP_EXPOSED_PORT", "")
-                app.authorization = action["authorization"]
+        logger.info(f"[DEBUG] \"{runtime}\" - run indicates microservices. Port: \"{exposed_port}\"")
-                app.current_execution_id = action["execution_id"]
+        if runtime == "run" and exposed_port != "":
-            except:
+            # Base port is 33334. Exposed port may differ based on discovery from Worker
-                pass
+            port = int(exposed_port)
+            logger.info(f"[DEBUG] Starting webserver on port {port} (same as exposed port)")
+            from flask import Flask, request
+            from waitress import serve
+            import asyncio
         
-            try:
+            flask_app = Flask(__name__)
-                app.url = action["url"]
-            except:
-                pass
         
-            try:
+            @flask_app.route("/api/v1/run", methods=["POST"])
-                app.base_url = action["base_url"]
+            #async def execute():
-            except:
+            def execute():
-                pass
+                if request.method == "POST":
+                    #print(request.get_json(force=True))
+                    #print("DATA: ", request.data)
+                    requestdata = {}
+                    try:
+                        requestdata = json.loads(request.data)
+                    except Exception as e:
+                        return {
+                            "success": False,
+                            "reason": f"Invalid Action data {e}",
+                        }
+        
+                    #logger.info(f"[DEBUG] Datatype: {type(requestdata)}: {requestdata}")
+
+                    # Remaking class for each request
+                    #print(f"APP: {app}")
+        
+                    app = cls(redis=None, logger=logger, console_logger=logger)
+                    try:
+                        #asyncio.run(AppBase.run(action=requestdata), debug=True)
+                        #value = json.dumps(value)
+                        try:
+                            app.full_execution = json.dumps(requestdata["workflow_execution"])
+                        except Exception as e:
+                            logger.info(f"Failed parsing full execution from workflow_execution: {e}")
+                        try:
+                            app.action = requestdata["action"] 
+                        except:
+                            logger.info("Failed parsing action")
+
+                        try:
+                            app.authorization = requestdata["authorization"]
+                            app.current_execution_id = requestdata["execution_id"]
+                        except:
+                            logger.info("Failed parsing auth and exec id")
+
+                        # BASE URL (backend)
+                        try:
+                            app.url = requestdata["url"]
+                            logger.info(f"BACKEND URL: {app.url}")
+                        except:
+                            logger.info("Failed parsing url")
+
+                        # URL (worker)
+                        try:
+                            app.base_url = requestdata["base_url"]
+                            logger.info(f"WORKER URL: {app.base_url}")
+                        except:
+                            logger.info("Failed parsing base url")
+                        
+                        #await 
+                        app.execute_action(app.action)
+                        logger.info("\n\n[DEBUG] Done awaiting app action running\n\n")
+                    except Exception as e:
+                        return {
+                            "success": False,
+                            "reason": f"Problem in execution {e}",
+                        }
+        
+                    return {
+                        "success": True,
+                        "reason": "App successfully finished",
+                    }
+                else:
+                    return {
+                        "success": False,
+                        "reason": f"HTTP method {request.method} not allowed",
+                    }
+        
+            logger.info(f"[DEBUG] Serving on port {port}")
+            serve(
+                flask_app, 
+                host="0.0.0.0", 
+                port=port, 
+                threads=8,
+                channel_timeout=30,
+                expose_tracebacks=True,
+                asyncore_use_poll=True,
+            )
+            #######################
         else:
-            self.logger.info("ACTION TYPE (unhandled): %s" % type(action))
+            # Has to start like this due to imports in other apps
+            # Move it outside everything?
+            app = cls(redis=None, logger=logger, console_logger=logger)
+            logger.info(f"[DEBUG] Action: {action}")
             
-        await app.execute_action(app.action)
+            if isinstance(action, str):
+                logger.info("[DEBUG] Normal execution (env var). Action is a string.")
+            elif isinstance(action, object):
+                logger.info("[DEBUG] OBJECT execution (cloud). Action is NOT a string.")
+                app.action = action
+
+                try:
+                    app.authorization = action["authorization"]
+                    app.current_execution_id = action["execution_id"]
+                except:
+                    pass
+
+                # BASE URL (worker)
+                try:
+                    app.url = action["url"]
+                except:
+                    pass
+
+                # Callback URL (backend)
+                try:
+                    app.base_url = action["base_url"]
+                except:
+                    pass
+            else:
+                self.logger.info("ACTION TYPE (unhandled): %s" % type(action))
+
+            #await app.execute_action(app.action)
+            app.execute_action(app.action)
+
+    #app.run(host="0.0.0.0", port=33334)
+
+if __name__ == "__main__":
+    import asyncio
+    asyncio.run(AppBase.run(), debug=True)

backend/app_sdk/build.sh

@@ -3,7 +3,7 @@
 
 ### DEFAULT 
 NAME=shuffle-app_sdk
-VERSION=0.9.25
+VERSION=0.9.30
 
 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force
 docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly

backend/app_sdk/requirements.txt

@@ -1,4 +1,7 @@
 urllib3==1.26.5
 requests==2.25.1
 MarkupSafe==2.0.1
-liquidpy==0.7.1
+liquidpy==0.7.2
+flask[async]==2.0.2
+waitress==2.0.0
+#flask==1.1.2

backend/go-app/go.mod

@@ -1,50 +1,32 @@
-module shuffle
+module main
 
-go 1.13
+go 1.15
-
-replace github.com/shuffle/shuffle-shared => ../../../../git/shuffle-shared
 
+//replace github.com/shuffle/shuffle-shared => ../../../../git/shuffle-shared
 //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi
+//replace github.com/frikky/go-elasticsearch => ../../../../git/go-elasticsearch
 
 require (
-	cloud.google.com/go v0.75.0
+	cloud.google.com/go/datastore v1.6.0
-	cloud.google.com/go/datastore v1.4.0
+	cloud.google.com/go/pubsub v1.17.0
-	cloud.google.com/go/pubsub v1.3.1
+	cloud.google.com/go/storage v1.18.1
-	cloud.google.com/go/storage v1.12.0
-	github.com/Masterminds/semver v1.5.0 // indirect
-	github.com/RobotsAndPencils/go-saml v0.0.0-20170520135329-fb13cb52a46b // indirect
-	github.com/algolia/algoliasearch-client-go/v3 v3.18.1 // indirect
 	github.com/basgys/goxml2json v1.1.0
-	github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 // indirect
 	github.com/carlescere/scheduler v0.0.0-20170109141437-ee74d2f83d82
-	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/docker/docker v20.10.9+incompatible
-	github.com/docker/docker v20.10.3-0.20210216175712-646072ed6524+incompatible
+	github.com/frikky/kin-openapi v0.40.0
-	github.com/docker/go-connections v0.4.0
+	github.com/fsouza/go-dockerclient v1.7.4
-	github.com/docker/go-units v0.4.0 // indirect
-	github.com/elastic/go-elasticsearch/v7 v7.13.1 // indirect
-	github.com/frikky/kin-openapi v0.39.0
-	github.com/frikky/shuffle-shared v0.1.15
-	github.com/fsouza/go-dockerclient v1.7.2
 	github.com/ghodss/yaml v1.0.0
-	github.com/go-git/go-billy/v5 v5.0.0
+	github.com/go-git/go-billy/v5 v5.3.1
-	github.com/go-git/go-git/v5 v5.0.0
+	github.com/go-git/go-git/v5 v5.4.2
-	github.com/google/go-github/v28 v28.1.1
-	github.com/gorilla/handlers v1.4.2 // indirect
 	github.com/gorilla/mux v1.8.0
-	github.com/h2non/filetype v1.0.12
+	github.com/h2non/filetype v1.1.1
-	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
-	github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d // indirect
-	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/satori/go.uuid v1.2.0
-	github.com/shuffle/shuffle-shared v0.1.15
+	github.com/shuffle/shuffle-shared v0.1.27
 	go4.org v0.0.0-20201209231011-d4a079459e60 // indirect
-	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
+	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
-	golang.org/x/oauth2 v0.0.0-20210113160501-8b1d76fa0423
+	google.golang.org/api v0.58.0
-	google.golang.org/api v0.36.0
 	google.golang.org/appengine v1.6.7
-	google.golang.org/genproto v0.0.0-20210113195801-ae06605f4595
+	google.golang.org/grpc v1.41.0
-	google.golang.org/grpc v1.34.1
 	gopkg.in/src-d/go-git.v4 v4.13.1
-	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )

backend/go-app/main.go

@@ -1955,9 +1955,18 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) {
 	// 1. Get callback data
 	// 2. Load the configuration
 	// 3. Execute the workflow
-	cors := shuffle.HandleCors(resp, request)
+	//cors := shuffle.HandleCors(resp, request)
-	if cors {
+	//if cors {
-		return
+	//	return
+	//}
+
+	if request.Method != "POST" {
+		request.Method = "POST"
+	}
+
+	if request.Body == nil {
+		stringReader := strings.NewReader("")
+		request.Body = ioutil.NopCloser(stringReader)
 	}
 
 	path := strings.Split(request.URL.String(), "/")
@@ -3837,7 +3846,7 @@ func runInitEs(ctx context.Context) {
 			//}
 
 		} else {
-			log.Printf("[DEBUG] There are %d org(s).", len(activeOrgs))
+			log.Printf("[DEBUG] Found %d org(s) in total.", len(activeOrgs))
 
 			if len(activeOrgs) == 1 {
 				if len(activeOrgs[0].Users) == 0 {
@@ -5639,6 +5648,7 @@ func initHandlers() {
 	log.Printf("[DEBUG] Initialized Shuffle database connection. Setting up environment.")
 
 	if elasticConfig == "elasticsearch" {
+		time.Sleep(5 * time.Second)
 		go runInitEs(ctx)
 	} else {
 		go runInit(ctx)
@@ -5732,7 +5742,7 @@ func initHandlers() {
 
 	// Triggers
 	r.HandleFunc("/api/v1/hooks/new", shuffle.HandleNewHook).Methods("POST", "OPTIONS")
-	r.HandleFunc("/api/v1/hooks/{key}", handleWebhookCallback).Methods("POST", "OPTIONS")
+	r.HandleFunc("/api/v1/hooks/{key}", handleWebhookCallback).Methods("POST", "GET", "PATCH", "PUT", "DELETE", "OPTIONS")
 	r.HandleFunc("/api/v1/hooks/{key}/delete", shuffle.HandleDeleteHook).Methods("DELETE", "OPTIONS")
 
 	// OpenAPI configuration

backend/go-app/walkoff.go

@@ -569,7 +569,7 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque
 	// FIXME: Add authentication?
 	id := request.Header.Get("Org-Id")
 	if len(id) == 0 {
-		log.Printf("No Org-Id header set - confirm")
+		log.Printf("[ERROR] No Org-Id header set - confirm")
 		resp.WriteHeader(401)
 		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Specify the org-id header."}`)))
 		return
@@ -579,7 +579,7 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque
 	ctx := context.Background()
 	executionRequests, err := shuffle.GetWorkflowQueue(ctx, id)
 	if err != nil {
-		log.Printf("(1) Failed reading body for workflowqueue: %s", err)
+		log.Printf("[WARNING] (1) Failed reading body for workflowqueue: %s", err)
 		resp.WriteHeader(401)
 		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Entity parsing error - confirm"}`)))
 		return
@@ -676,6 +676,19 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
 	}
 
 	ctx := context.Background()
+	env, err := shuffle.GetEnvironment(ctx, id, "")
+	timeNow := time.Now().Unix()
+	if err == nil && len(env.Id) > 0 && len(env.Name) > 0 {
+		if time.Now().Unix() > env.Edited+60 {
+			env.RunningIp = request.RemoteAddr
+			env.Checkin = timeNow
+			err = shuffle.SetEnvironment(ctx, env)
+			if err != nil {
+				log.Printf("[WARNING] Failed updating environment: %s", err)
+			}
+		}
+	}
+
 	executionRequests, err := shuffle.GetWorkflowQueue(ctx, id)
 	if err != nil {
 		// Skipping as this comes up over and over
@@ -685,11 +698,48 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
 		return
 	}
 
+	// Checking and updating the environment related to the first execution
 	if len(executionRequests.Data) == 0 {
 		executionRequests.Data = []shuffle.ExecutionRequest{}
 	} else {
-		//log.Printf("[INFO] Executionrequests (%s): %d", id, len(executionRequests.Data))
+		//log.Printf("In workflowqueue with %d", len(executionRequests.Data))
-		//log.Printf("IDS: %#v", executionRequests.Data[0].ExecutionId)
+
+		// Try again :)
+		if len(env.Id) == 0 && len(env.Name) == 0 {
+			orgId := ""
+			for _, requestData := range executionRequests.Data {
+				execution, err := shuffle.GetWorkflowExecution(ctx, requestData.ExecutionId)
+				if err == nil {
+					if len(execution.ExecutionOrg) > 0 {
+						orgId = execution.ExecutionOrg
+						break
+					}
+				}
+			}
+
+			if len(orgId) > 0 {
+				env, err := shuffle.GetEnvironment(ctx, id, orgId)
+				if err != nil {
+					log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", id, err)
+					//resp.WriteHeader(401)
+					//resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No env found matching %s"}`, id)))
+					//return
+				} else {
+					if timeNow > env.Edited+60 {
+						env.RunningIp = request.RemoteAddr
+						env.Checkin = timeNow
+						err = shuffle.SetEnvironment(ctx, env)
+						if err != nil {
+							log.Printf("[WARNING] Failed updating environment: %s", err)
+						}
+					}
+				}
+			}
+		}
+
+		if len(executionRequests.Data) > 10 {
+			executionRequests.Data = executionRequests.Data[0:9]
+		}
 	}
 
 	newjson, err := json.Marshal(executionRequests)
@@ -849,7 +899,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
 		var trigger shuffle.Trigger
 		err = json.Unmarshal([]byte(actionResult.Result), &trigger)
 		if err != nil {
-			log.Printf("Failed unmarshaling actionresult for user input: %s", err)
+			log.Printf("[WARNING] Failed unmarshaling actionresult for user input: %s", err)
 			resp.WriteHeader(401)
 			resp.Write([]byte(`{"success": false}`))
 			return
@@ -862,21 +912,21 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
 
 		err := handleUserInput(trigger, orgId, workflowExecution.Workflow.ID, workflowExecution.ExecutionId)
 		if err != nil {
-			log.Printf("Failed userinput handler: %s", err)
+			log.Printf("[WARNING] Failed userinput handler: %s", err)
 			actionResult.Result = fmt.Sprintf("Cloud error: %s", err)
 			workflowExecution.Results = append(workflowExecution.Results, actionResult)
 			workflowExecution.Status = "ABORTED"
 			err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, true)
 			if err != nil {
-				log.Printf("Failed to set execution during wait")
+				log.Printf("[WARNING] Failed to set execution during wait: %s", err)
 			} else {
-				log.Printf("Successfully set the execution to waiting.")
+				log.Printf("[INFO] Successfully set the execution %s to waiting.", workflowExecution.ExecutionId)
 			}
 
 			resp.WriteHeader(401)
 			resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error: %s"}`, err)))
 		} else {
-			log.Printf("Successful userinput handler")
+			log.Printf("[INFO] Successful userinput handler")
 			resp.WriteHeader(200)
 			resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "CLOUD IS DONE"}`)))
 
@@ -886,7 +936,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
 			workflowExecution.Status = actionResult.Status
 			err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, true)
 			if err != nil {
-				log.Printf("Failed ")
+				log.Printf("[WARNING] Failed setting userinput: %s", err)
 			} else {
 				log.Printf("Successfully set the execution to waiting.")
 			}
@@ -1168,8 +1218,6 @@ func getWorkflowLocal(fileId string, request *http.Request) ([]byte, error) {
 	return body, nil
 }
 
-//// New execution with firestore
-
 func handleExecution(id string, workflow shuffle.Workflow, request *http.Request) (shuffle.WorkflowExecution, string, error) {
 	//go func() {
 	//	log.Printf("\n\nPRE TIME: %s\n\n", time.Now().Format("2006-01-02 15:04:05"))
@@ -1233,725 +1281,38 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request
 		return shuffle.WorkflowExecution{}, fmt.Sprintf(`workflow %s is invalid`, workflow.ID), errors.New("Failed getting workflow")
 	}
 
-	workflowBytes, err := json.Marshal(workflow)
+	workflowExecution, execInfo, _, err := shuffle.PrepareWorkflowExecution(ctx, workflow, request)
 	if err != nil {
-		log.Printf("Failed workflow unmarshal in execution: %s", err)
+		log.Printf("[WARNING] Failed in prepareExecution: %s", err)
 		return shuffle.WorkflowExecution{}, "", err
 	}
 
-	//log.Println(workflow)
+	err = imageCheckBuilder(execInfo.ImageNames)
-	var workflowExecution shuffle.WorkflowExecution
-	err = json.Unmarshal(workflowBytes, &workflowExecution.Workflow)
 	if err != nil {
-		log.Printf("Failed execution unmarshaling: %s", err)
+		log.Printf("[ERROR] Failed building the required images from %#v: %s", execInfo.ImageNames, err)
-		return shuffle.WorkflowExecution{}, "Failed unmarshal during execution", err
-	}
-
-	makeNew := true
-	start, startok := request.URL.Query()["start"]
-	if request.Method == "POST" {
-		body, err := ioutil.ReadAll(request.Body)
-		if err != nil {
-			log.Printf("[ERROR] Failed request POST read: %s", err)
-			return shuffle.WorkflowExecution{}, "Failed getting body", err
-		}
-
-		// This one doesn't really matter.
-		log.Printf("[INFO] Running POST execution with body of length %d for workflow %s", len(string(body)), workflowExecution.Workflow.ID)
-
-		if len(body) >= 4 {
-			if body[0] == 34 && body[len(body)-1] == 34 {
-				body = body[1 : len(body)-1]
-			}
-			if body[0] == 34 && body[len(body)-1] == 34 {
-				body = body[1 : len(body)-1]
-			}
-		}
-
-		sourceAuth, sourceAuthOk := request.URL.Query()["source_auth"]
-		if sourceAuthOk {
-			//log.Printf("\n\n\nSETTING SOURCE WORKFLOW AUTH TO %s!!!\n\n\n", sourceAuth[0])
-			workflowExecution.ExecutionSourceAuth = sourceAuth[0]
-		} else {
-			//log.Printf("Did NOT get source workflow")
-		}
-
-		sourceNode, sourceNodeOk := request.URL.Query()["source_node"]
-		if sourceNodeOk {
-			//log.Printf("\n\n\nSETTING SOURCE WORKFLOW NODE TO %s!!!\n\n\n", sourceNode[0])
-			workflowExecution.ExecutionSourceNode = sourceNode[0]
-		} else {
-			//log.Printf("Did NOT get source workflow")
-		}
-
-		//workflowExecution.ExecutionSource = "default"
-		sourceWorkflow, sourceWorkflowOk := request.URL.Query()["source_workflow"]
-		if sourceWorkflowOk {
-			//log.Printf("Got source workflow %s", sourceWorkflow)
-			workflowExecution.ExecutionSource = sourceWorkflow[0]
-		} else {
-			//log.Printf("Did NOT get source workflow")
-		}
-
-		sourceExecution, sourceExecutionOk := request.URL.Query()["source_execution"]
-		if sourceExecutionOk {
-			//log.Printf("[INFO] Got source execution%s", sourceExecution)
-			workflowExecution.ExecutionParent = sourceExecution[0]
-		} else {
-			//log.Printf("Did NOT get source execution")
-		}
-
-		if len(string(body)) < 50 {
-			//log.Println(body)
-			// String in string
-			//log.Println(body)
-
-			//if string(body)[0] == "\"" && string(body)[string(body)
-			log.Printf("[DEBUG] Body: %s", string(body))
-		}
-
-		var execution shuffle.ExecutionRequest
-		err = json.Unmarshal(body, &execution)
-		if err != nil {
-			log.Printf("[WARNING] Failed execution POST unmarshaling - continuing anyway: %s", err)
-			//return shuffle.WorkflowExecution{}, "", err
-		}
-
-		if execution.Start == "" && len(body) > 0 {
-			execution.ExecutionArgument = string(body)
-		}
-
-		// FIXME - this should have "execution_argument" from executeWorkflow frontend
-		//log.Printf("EXEC: %#v", execution)
-		if len(execution.ExecutionArgument) > 0 {
-			workflowExecution.ExecutionArgument = execution.ExecutionArgument
-		}
-
-		if len(execution.ExecutionSource) > 0 {
-			workflowExecution.ExecutionSource = execution.ExecutionSource
-		}
-
-		//log.Printf("Execution data: %#v", execution)
-		if len(execution.Start) == 36 && len(workflow.Actions) > 0 {
-			log.Printf("[INFO] Should start execution on node %s", execution.Start)
-			workflowExecution.Start = execution.Start
-
-			found := false
-			for _, action := range workflow.Actions {
-				if action.ID == execution.Start {
-					found = true
-					break
-				}
-			}
-
-			if !found {
-				log.Printf("[ERROR] Action %s was NOT found! Exiting execution.", execution.Start)
-				return shuffle.WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", workflow.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", workflow.Start))
-			}
-		} else if len(execution.Start) > 0 {
-			//log.Printf("[INFO] !")
-			//log.Printf("[ERROR] START ACTION %s IS WRONG ID LENGTH %d!", execution.Start, len(execution.Start))
-			//return shuffle.WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", execution.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", execution.Start))
-		}
-
-		if len(execution.ExecutionId) == 36 {
-			workflowExecution.ExecutionId = execution.ExecutionId
-		} else {
-			sessionToken := uuid.NewV4()
-			workflowExecution.ExecutionId = sessionToken.String()
-		}
-	} else {
-		// Check for parameters of start and ExecutionId
-		// This is mostly used for user input trigger
-
-		answer, answerok := request.URL.Query()["answer"]
-		referenceId, referenceok := request.URL.Query()["reference_execution"]
-		if answerok && referenceok {
-			// If answer is false, reference execution with result
-			log.Printf("[INFO] Answer is OK AND reference is OK!")
-			if answer[0] == "false" {
-				log.Printf("Should update reference and return, no need for further execution!")
-
-				// Get the reference execution
-				oldExecution, err := shuffle.GetWorkflowExecution(ctx, referenceId[0])
-				if err != nil {
-					log.Printf("Failed getting execution (execution) %s: %s", referenceId[0], err)
-					return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed getting execution ID %s because it doesn't exist.", referenceId[0]), err
-				}
-
-				if oldExecution.Workflow.ID != id {
-					log.Println("Wrong workflowid!")
-					return shuffle.WorkflowExecution{}, fmt.Sprintf("Bad ID %s", referenceId), errors.New("Bad ID")
-				}
-
-				newResults := []shuffle.ActionResult{}
-				//log.Printf("%#v", oldExecution.Results)
-				for _, result := range oldExecution.Results {
-					log.Printf("%s - %s", result.Action.ID, start[0])
-					if result.Action.ID == start[0] {
-						note, noteok := request.URL.Query()["note"]
-						if noteok {
-							result.Result = fmt.Sprintf("User note: %s", note[0])
-						} else {
-							result.Result = fmt.Sprintf("User clicked %s", answer[0])
-						}
-
-						// Stopping the whole thing
-						result.CompletedAt = int64(time.Now().Unix())
-						result.Status = "ABORTED"
-						oldExecution.Status = result.Status
-						oldExecution.Result = result.Result
-						oldExecution.LastNode = result.Action.ID
-					}
-
-					newResults = append(newResults, result)
-				}
-
-				oldExecution.Results = newResults
-				err = shuffle.SetWorkflowExecution(ctx, *oldExecution, true)
-				if err != nil {
-					log.Printf("Error saving workflow execution actionresult setting: %s", err)
-					return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed setting workflowexecution actionresult in execution: %s", err), err
-				}
-
-				return shuffle.WorkflowExecution{}, "", nil
-			}
-		}
-
-		if referenceok {
-			log.Printf("Handling an old execution continuation!")
-			// Will use the old name, but still continue with NEW ID
-			oldExecution, err := shuffle.GetWorkflowExecution(ctx, referenceId[0])
-			if err != nil {
-				log.Printf("Failed getting execution (execution) %s: %s", referenceId[0], err)
-				return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed getting execution ID %s because it doesn't exist.", referenceId[0]), err
-			}
-
-			workflowExecution = *oldExecution
-		}
-
-		if len(workflowExecution.ExecutionId) == 0 {
-			sessionToken := uuid.NewV4()
-			workflowExecution.ExecutionId = sessionToken.String()
-		} else {
-			log.Printf("Using the same executionId as before: %s", workflowExecution.ExecutionId)
-			makeNew = false
-		}
-
-		// Don't override workflow defaults
-	}
-
-	if startok {
-		//log.Printf("\n\n[INFO] Setting start to %s based on query!\n\n", start[0])
-		//workflowExecution.Workflow.Start = start[0]
-		workflowExecution.Start = start[0]
-	}
-
-	// FIXME - regex uuid, and check if already exists?
-	if len(workflowExecution.ExecutionId) != 36 {
-		log.Printf("Invalid uuid: %s", workflowExecution.ExecutionId)
-		return shuffle.WorkflowExecution{}, "Invalid uuid", err
-	}
-
-	// FIXME - find owner of workflow
-	// FIXME - get the actual workflow itself and build the request
-	// MAYBE: Don't send the workflow within the pubsub, as this requires more data to be sent
-	// Check if a worker already exists for company, else run one with:
-	// locations, project IDs and subscription names
-
-	// When app is executed:
-	// Should update with status execution (somewhere), which will trigger the next node
-	// IF action.type == internal, we need the internal watcher to be running and executing
-	// This essentially means the WORKER has to be the responsible party for new actions in the INTERNAL landscape
-	// Results are ALWAYS posted back to cloud@execution_id?
-	if makeNew {
-		workflowExecution.Type = "workflow"
-		//workflowExecution.Stream = "tmp"
-		//workflowExecution.WorkflowQueue = "tmp"
-		//workflowExecution.SubscriptionNameNodestream = "testcompany-nodestream"
-		//workflowExecution.Locations = []string{"europe-west2"}
-		workflowExecution.ProjectId = gceProject
-		workflowExecution.WorkflowId = workflow.ID
-		workflowExecution.StartedAt = int64(time.Now().Unix())
-		workflowExecution.CompletedAt = 0
-		workflowExecution.Authorization = uuid.NewV4().String()
-
-		// Status for the entire workflow.
-		workflowExecution.Status = "EXECUTING"
-	}
-
-	if len(workflowExecution.ExecutionSource) == 0 {
-		log.Printf("[INFO] No execution source (trigger) specified. Setting to default")
-		workflowExecution.ExecutionSource = "default"
-	} else {
-		log.Printf("[INFO] Execution source is %s for execution ID %s in workflow %s", workflowExecution.ExecutionSource, workflowExecution.ExecutionId, workflowExecution.Workflow.ID)
-	}
-
-	workflowExecution.ExecutionVariables = workflow.ExecutionVariables
-	if len(workflowExecution.Start) == 0 && len(workflowExecution.Workflow.Start) > 0 {
-		workflowExecution.Start = workflowExecution.Workflow.Start
-	}
-
-	startnodeFound := false
-	newStartnode := ""
-	for _, item := range workflowExecution.Workflow.Actions {
-		if item.ID == workflowExecution.Start {
-			startnodeFound = true
-		}
-
-		if item.IsStartNode {
-			newStartnode = item.ID
-		}
-	}
-
-	if !startnodeFound {
-		log.Printf("[INFO] Couldn't find startnode %s. Remapping to %#v", workflowExecution.Start, newStartnode)
-
-		if len(newStartnode) > 0 {
-			workflowExecution.Start = newStartnode
-		} else {
-			return shuffle.WorkflowExecution{}, fmt.Sprintf("Startnode couldn't be found"), errors.New("Startnode isn't defined in this workflow..")
-		}
-	}
-
-	childNodes := shuffle.FindChildNodes(workflowExecution, workflowExecution.Start)
-
-	topic := "workflows"
-	startFound := false
-	// FIXME - remove this?
-	newActions := []shuffle.Action{}
-	defaultResults := []shuffle.ActionResult{}
-
-	allAuths := []shuffle.AppAuthenticationStorage{}
-	for _, action := range workflowExecution.Workflow.Actions {
-		//action.LargeImage = ""
-		if action.ID == workflowExecution.Start {
-			startFound = true
-		}
-		//log.Println(action.Environment)
-
-		if action.Environment == "" {
-			return shuffle.WorkflowExecution{}, fmt.Sprintf("Environment is not defined for %s", action.Name), errors.New("Environment not defined!")
-		}
-
-		// FIXME: Authentication parameters
-		if len(action.AuthenticationId) > 0 {
-			if len(allAuths) == 0 {
-				allAuths, err = shuffle.GetAllWorkflowAppAuth(ctx, workflow.ExecutingOrg.Id)
-				if err != nil {
-					log.Printf("Api authentication failed in get all app auth: %s", err)
-					return shuffle.WorkflowExecution{}, fmt.Sprintf("Api authentication failed in get all app auth: %s", err), err
-				}
-			}
-
-			curAuth := shuffle.AppAuthenticationStorage{Id: ""}
-			for _, auth := range allAuths {
-				if auth.Id == action.AuthenticationId {
-					curAuth = auth
-					break
-				}
-			}
-
-			if len(curAuth.Id) == 0 {
-				return shuffle.WorkflowExecution{}, fmt.Sprintf("Auth ID %s doesn't exist", action.AuthenticationId), errors.New(fmt.Sprintf("Auth ID %s doesn't exist", action.AuthenticationId))
-			}
-
-			if curAuth.Encrypted {
-				setField := true
-				newFields := []shuffle.AuthenticationStore{}
-				for _, field := range curAuth.Fields {
-					parsedKey := fmt.Sprintf("%s_%d_%s_%s", curAuth.OrgId, curAuth.Created, curAuth.Label, field.Key)
-					newValue, err := shuffle.HandleKeyDecryption(field.Value, parsedKey)
-					if err != nil {
-						log.Printf("[WARNING] Failed decryption for %s: %s", field.Key, err)
-						setField = false
-						break
-					}
-
-					field.Value = newValue
-					newFields = append(newFields, field)
-				}
-
-				if setField {
-					curAuth.Fields = newFields
-				}
-			} else {
-				log.Printf("[INFO] AUTH IS NOT ENCRYPTED - attempting encrypting!")
-				err = shuffle.SetWorkflowAppAuthDatastore(ctx, curAuth, curAuth.Id)
-				if err != nil {
-					log.Printf("[WARNING] Failed running encryption during execution: %s", err)
-				}
-			}
-
-			newParams := []shuffle.WorkflowAppActionParameter{}
-			if strings.ToLower(curAuth.Type) == "oauth2" {
-				log.Printf("[DEBUG] Should replace auth parameters (Oauth2)")
-
-				for _, param := range curAuth.Fields {
-					if param.Key == "expiration" {
-						continue
-					}
-
-					newParams = append(newParams, shuffle.WorkflowAppActionParameter{
-						Name:  param.Key,
-						Value: param.Value,
-					})
-				}
-
-				for _, param := range action.Parameters {
-					//log.Printf("Param: %#v", param)
-					if param.Configuration {
-						continue
-					}
-
-					newParams = append(newParams, param)
-				}
-			} else {
-				// Rebuild params with the right data. This is to prevent issues on the frontend
-				for _, param := range action.Parameters {
-
-					for _, authparam := range curAuth.Fields {
-						if param.Name == authparam.Key {
-							param.Value = authparam.Value
-							//log.Printf("Name: %s - value: %s", param.Name, param.Value)
-							//log.Printf("Name: %s - value: %s\n", param.Name, param.Value)
-							break
-						}
-					}
-
-					newParams = append(newParams, param)
-				}
-			}
-
-			action.Parameters = newParams
-		}
-
-		action.LargeImage = ""
-		if len(action.Label) == 0 {
-			action.Label = action.ID
-		}
-		//log.Printf("LABEL: %s", action.Label)
-		newActions = append(newActions, action)
-
-		// If the node is NOT found, it's supposed to be set to SKIPPED,
-		// as it's not a childnode of the startnode
-		// This is a configuration item for the workflow itself.
-		if len(workflowExecution.Results) > 0 {
-			defaultResults = []shuffle.ActionResult{}
-			for _, result := range workflowExecution.Results {
-				if result.Status == "WAITING" {
-					result.Status = "FINISHED"
-					result.Result = "Continuing"
-				}
-
-				defaultResults = append(defaultResults, result)
-			}
-		} else if len(workflowExecution.Results) == 0 && !workflowExecution.Workflow.Configuration.StartFromTop {
-			found := false
-			for _, nodeId := range childNodes {
-				if nodeId == action.ID {
-					//log.Printf("Found %s", action.ID)
-					found = true
-				}
-			}
-
-			if !found {
-				if action.ID == workflowExecution.Start {
-					continue
-				}
-
-				//log.Printf("[WARNING] Set %s to SKIPPED as it's NOT a childnode of the startnode.", action.ID)
-				curaction := shuffle.Action{
-					AppName:    action.AppName,
-					AppVersion: action.AppVersion,
-					Label:      action.Label,
-					Name:       action.Name,
-					ID:         action.ID,
-				}
-				//action
-				//curaction.Parameters = []
-				defaultResults = append(defaultResults, shuffle.ActionResult{
-					Action:        curaction,
-					ExecutionId:   workflowExecution.ExecutionId,
-					Authorization: workflowExecution.Authorization,
-					Result:        "Skipped because it's not under the startnode",
-					StartedAt:     0,
-					CompletedAt:   0,
-					Status:        "SKIPPED",
-				})
-			}
-		}
-	}
-
-	removeTriggers := []string{}
-	for triggerIndex, trigger := range workflowExecution.Workflow.Triggers {
-		//log.Printf("[INFO] ID: %s vs %s", trigger.ID, workflowExecution.Start)
-		if trigger.ID == workflowExecution.Start {
-			if trigger.AppName == "User Input" {
-				startFound = true
-				break
-			}
-		}
-
-		if trigger.AppName == "User Input" || trigger.AppName == "Shuffle Workflow" {
-			found := false
-			for _, node := range childNodes {
-				if node == trigger.ID {
-					found = true
-					break
-				}
-			}
-
-			if !found {
-				//log.Printf("SHOULD SET TRIGGER %s TO BE SKIPPED", trigger.ID)
-
-				curaction := shuffle.Action{
-					AppName:    "shuffle-subflow",
-					AppVersion: trigger.AppVersion,
-					Label:      trigger.Label,
-					Name:       trigger.Name,
-					ID:         trigger.ID,
-				}
-
-				defaultResults = append(defaultResults, shuffle.ActionResult{
-					Action:        curaction,
-					ExecutionId:   workflowExecution.ExecutionId,
-					Authorization: workflowExecution.Authorization,
-					Result:        "Skipped because it's not under the startnode",
-					StartedAt:     0,
-					CompletedAt:   0,
-					Status:        "SKIPPED",
-				})
-			} else {
-				// Replaces trigger with the subflow
-				//if trigger.AppName == "Shuffle Workflow" {
-				//	replaceActions := false
-				//	workflowAction := ""
-				//	for _, param := range trigger.Parameters {
-				//		if param.Name == "argument" && !strings.Contains(param.Value, ".#") {
-				//			replaceActions = true
-				//		}
-
-				//		if param.Name == "startnode" {
-				//			workflowAction = param.Value
-				//		}
-				//	}
-
-				//	if replaceActions {
-				//		replacementNodes, newBranches, lastnode := shuffle.GetReplacementNodes(ctx, workflowExecution, trigger, trigger.Label)
-				//		log.Printf("REPLACEMENTS: %d, %d", len(replacementNodes), len(newBranches))
-				//		if len(replacementNodes) > 0 {
-				//			for _, action := range replacementNodes {
-				//				found := false
-
-				//				for subActionIndex, subaction := range newActions {
-				//					if subaction.ID == action.ID {
-				//						found = true
-				//						//newActions[subActionIndex].Name = action.Name
-				//						newActions[subActionIndex].Label = action.Label
-				//						break
-				//					}
-				//				}
-
-				//				if !found {
-				//					action.SubAction = true
-				//					newActions = append(newActions, action)
-				//				}
-
-				//				// Check if it's already set to have a value
-				//				for resultIndex, result := range defaultResults {
-				//					if result.Action.ID == action.ID {
-				//						defaultResults = append(defaultResults[:resultIndex], defaultResults[resultIndex+1:]...)
-				//						break
-				//					}
-				//				}
-				//			}
-
-				//			for _, branch := range newBranches {
-				//				workflowExecution.Workflow.Branches = append(workflowExecution.Workflow.Branches, branch)
-				//			}
-
-				//			// Append branches:
-				//			// parent -> new inner node (FIRST one)
-				//			for branchIndex, branch := range workflowExecution.Workflow.Branches {
-				//				if branch.DestinationID == trigger.ID {
-				//					log.Printf("REPLACE DESTINATION WITH %s!!", workflowAction)
-				//					workflowExecution.Workflow.Branches[branchIndex].DestinationID = workflowAction
-				//				}
-
-				//				if branch.SourceID == trigger.ID {
-				//					log.Printf("REPLACE SOURCE WITH LASTNODE %s!!", lastnode)
-				//					workflowExecution.Workflow.Branches[branchIndex].SourceID = lastnode
-				//				}
-				//			}
-
-				//			// Remove the trigger
-				//			removeTriggers = append(removeTriggers, workflowExecution.Workflow.Triggers[triggerIndex].ID)
-				//		}
-
-				//		log.Printf("NEW ACTION LENGTH %d, RESULT: %d, Triggers: %d, BRANCHES: %d", len(newActions), len(defaultResults), len(workflowExecution.Workflow.Triggers), len(workflowExecution.Workflow.Branches))
-				//	}
-				//}
-				_ = triggerIndex
-			}
-		}
-	}
-
-	//newTriggers := []shuffle.Trigger{}
-	//for _, trigger := range workflowExecution.Workflow.Triggers {
-	//	found := false
-	//	for _, triggerId := range removeTriggers {
-	//		if trigger.ID == triggerId {
-	//			found = true
-	//			break
-	//		}
-	//	}
-
-	//	if found {
-	//		log.Printf("[WARNING] Removed trigger %s during execution", trigger.ID)
-	//		continue
-	//	}
-
-	//	newTriggers = append(newTriggers, trigger)
-	//}
-	//workflowExecution.Workflow.Triggers = newTriggers
-	_ = removeTriggers
-
-	if !startFound {
-		if len(workflowExecution.Start) == 0 && len(workflowExecution.Workflow.Start) > 0 {
-			workflowExecution.Start = workflow.Start
-		} else if len(workflowExecution.Workflow.Actions) > 0 {
-			workflowExecution.Start = workflowExecution.Workflow.Actions[0].ID
-		} else {
-			log.Printf("[ERROR] Startnode %s doesn't exist!!", workflowExecution.Start)
-			return shuffle.WorkflowExecution{}, fmt.Sprintf("Workflow action %s doesn't exist in workflow", workflowExecution.Start), errors.New(fmt.Sprintf(`Workflow start node "%s" doesn't exist. Exiting!`, workflowExecution.Start))
-		}
-	}
-
-	//log.Printf("EXECUTION START: %s", workflowExecution.Start)
-
-	// Verification for execution environments
-	workflowExecution.Results = defaultResults
-	workflowExecution.Workflow.Actions = newActions
-	onpremExecution := true
-	environments := []string{}
-
-	if len(workflowExecution.ExecutionOrg) == 0 && len(workflow.ExecutingOrg.Id) > 0 {
-		workflowExecution.ExecutionOrg = workflow.ExecutingOrg.Id
-	}
-
-	var allEnvs []shuffle.Environment
-	if len(workflowExecution.ExecutionOrg) > 0 {
-		//log.Printf("[INFO] Executing ORG: %s", workflowExecution.ExecutionOrg)
-
-		allEnvironments, err := shuffle.GetEnvironments(ctx, workflowExecution.ExecutionOrg)
-		if err != nil {
-			log.Printf("Failed finding environments: %s", err)
-			return shuffle.WorkflowExecution{}, fmt.Sprintf("Workflow environments not found for this org"), errors.New(fmt.Sprintf("Workflow environments not found for this org"))
-		}
-
-		for _, curenv := range allEnvironments {
-			if curenv.Archived {
-				continue
-			}
-
-			allEnvs = append(allEnvs, curenv)
-		}
-	} else {
-		log.Printf("[ERROR] No org identified for execution of %s. Returning", workflowExecution.Workflow.ID)
-		return shuffle.WorkflowExecution{}, "No org identified for execution", errors.New("No org identified for execution")
-	}
-
-	if len(allEnvs) == 0 {
-		log.Printf("[ERROR] No active environments found for org: %s", workflowExecution.ExecutionOrg)
-		return shuffle.WorkflowExecution{}, "No active environments found", errors.New(fmt.Sprintf("No active env found for org %s", workflowExecution.ExecutionOrg))
-	}
-
-	// Check if the actions are children of the startnode?
-	imageNames := []string{}
-	cloudExec := false
-	for _, action := range workflowExecution.Workflow.Actions {
-		// Verify if the action environment exists and append
-		found := false
-		for _, env := range allEnvs {
-			if env.Name == action.Environment {
-				found = true
-
-				if env.Type == "cloud" {
-					cloudExec = true
-				} else if env.Type == "onprem" {
-					onpremExecution = true
-				} else {
-					log.Printf("[ERROR] No handler for environment type %s", env.Type)
-					return shuffle.WorkflowExecution{}, "No active environments found", errors.New(fmt.Sprintf("No handler for environment type %s", env.Type))
-				}
-				break
-			}
-		}
-
-		if !found {
-			log.Printf("[ERROR] Couldn't find environment %s. Maybe it's inactive?", action.Environment)
-			return shuffle.WorkflowExecution{}, "Couldn't find the environment", errors.New(fmt.Sprintf("Couldn't find env %s in org %s", action.Environment, workflowExecution.ExecutionOrg))
-		}
-
-		found = false
-		for _, env := range environments {
-			if env == action.Environment {
-
-				found = true
-				break
-			}
-		}
-
-		// Check if the app exists?
-		newName := action.AppName
-		newName = strings.ReplaceAll(newName, " ", "-")
-		imageNames = append(imageNames, fmt.Sprintf("%s:%s_%s", baseDockerName, newName, action.AppVersion))
-
-		if !found {
-			environments = append(environments, action.Environment)
-		}
-	}
-
-	err = imageCheckBuilder(imageNames)
-	if err != nil {
-		log.Printf("[ERROR] Failed building the required images from %#v: %s", imageNames, err)
 		return shuffle.WorkflowExecution{}, "Failed building missing Docker images", err
 	}
 
-	//b, err := json.Marshal(workflowExecution)
-	//if err == nil {
-	//	log.Printf("LEN: %d", len(string(b)))
-	//	//workflowExecution.ExecutionOrg.SyncFeatures = Org{}
-	//}
-
-	workflowExecution.Workflow.ExecutingOrg = shuffle.OrgMini{
-		Id: workflowExecution.Workflow.ExecutingOrg.Id,
-	}
-	workflowExecution.Workflow.Org = []shuffle.OrgMini{
-		workflowExecution.Workflow.ExecutingOrg,
-	}
-
 	//Org               []Org    `json:"org,omitempty" datastore:"org"`
 	err = shuffle.SetWorkflowExecution(ctx, workflowExecution, true)
 	if err != nil {
-		log.Printf("[WARNING] Error saving workflow execution for updates %s: %s", topic, err)
+		log.Printf("[WARNING] Error saving workflow execution for updates %s", err)
 		return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed setting workflowexecution: %s", err), err
 	}
 
 	// Adds queue for onprem execution
 	// FIXME - add specifics to executionRequest, e.g. specific environment (can run multi onprem)
-	if onpremExecution {
+	if execInfo.OnpremExecution {
 		// FIXME - tmp name based on future companyname-companyId
 		// This leads to issues with overlaps. Should set limits and such instead
-		for _, environment := range environments {
+		for _, environment := range execInfo.Environments {
 			log.Printf("[INFO] Execution: %s should execute onprem with execution environment \"%s\". Workflow: %s", workflowExecution.ExecutionId, environment, workflowExecution.Workflow.ID)
 
 			executionRequest := shuffle.ExecutionRequest{
 				ExecutionId:   workflowExecution.ExecutionId,
 				WorkflowId:    workflowExecution.Workflow.ID,
 				Authorization: workflowExecution.Authorization,
-				Environments:  environments,
+				Environments:  execInfo.Environments,
 			}
 
 			//executionRequestWrapper, err := getWorkflowQueue(ctx, environment)
@@ -1972,7 +1333,7 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request
 	}
 
 	// Verifies and runs cloud executions
-	if cloudExec {
+	if execInfo.CloudExec {
 		featuresList, err := handleVerifyCloudsync(workflowExecution.ExecutionOrg)
 		if !featuresList.Workflows.Active || err != nil {
 			log.Printf("Error: %s", err)
@@ -2684,7 +2045,7 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) {
 	// Double unmarshal because of user apps
 	newbody, err := json.Marshal(newapps)
 	if err != nil {
-		log.Printf("Failed unmarshalling all newapps: %s", err)
+		log.Printf("[ERROR] Failed unmarshalling all newapps: %s", err)
 		resp.WriteHeader(401)
 		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking workflow apps"}`)))
 		return
@@ -4050,9 +3411,19 @@ func LoadSpecificApps(resp http.ResponseWriter, request *http.Request) {
 		if tmpBody.ForceUpdate {
 			dockercli, err := dockerclient.NewEnvClient()
 			if err == nil {
-				_, err := dockercli.ImagePull(ctx, "frikky/shuffle:app_sdk", types.ImagePullOptions{})
+
-				if err != nil {
+				appSdk := os.Getenv("SHUFFLE_APP_SDK_VERSION")
-					log.Printf("[WARNING] Failed to download apps with the new App SDK: %s", err)
+				if len(appSdk) == 0 {
+					_, err := dockercli.ImagePull(ctx, "frikky/shuffle:app_sdk", types.ImagePullOptions{})
+					if err != nil {
+						log.Printf("[WARNING] Failed to download new App SDK: %s", err)
+					}
+				} else {
+					_, err := dockercli.ImagePull(ctx, fmt.Sprintf("%s/%s/shuffle-app_sdk:%s", "ghcr.io", "frikky", appSdk), types.ImagePullOptions{})
+					if err != nil {
+						log.Printf("[WARNING] Failed to download new App SDK %s: %s", err)
+					}
+
 				}
 			} else {
 				log.Printf("[WARNING] Failed to download apps with the new App SDK because of docker cli: %s", err)

docker-compose.yml

@@ -2,7 +2,7 @@ version: '3'
 services:
   frontend:
     #build: ./frontend
-    image: ghcr.io/frikky/shuffle-frontend:latest
+    image: ghcr.io/frikky/shuffle-frontend:nightly
     container_name: shuffle-frontend
     hostname: shuffle-frontend
     ports:
@@ -17,7 +17,7 @@ services:
       - backend
   backend:
     #build: ./backend
-    image: ghcr.io/frikky/shuffle-backend:latest
+    image: ghcr.io/frikky/shuffle-backend:nightly
     container_name: shuffle-backend
     hostname: ${BACKEND_HOSTNAME}
     # Here for debugging:
@@ -39,8 +39,7 @@ services:
       #- opensearch #Not necessary because dependancy is handled within the backend itself instead
       #- database
   orborus:
-    #build: ./functions/onprem/orborus
+    image: ghcr.io/frikky/shuffle-orborus:nightly
-    image: ghcr.io/frikky/shuffle-orborus:latest
     container_name: shuffle-orborus
     hostname: shuffle-orborus
     networks:
@@ -48,8 +47,8 @@ services:
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
     environment:
-      - SHUFFLE_APP_SDK_VERSION=latest
+      - SHUFFLE_APP_SDK_VERSION=0.8.97
-      - SHUFFLE_WORKER_VERSION=latest
+      - SHUFFLE_WORKER_VERSION=0.9.30
       - ORG_ID=${ORG_ID}
       - ENVIRONMENT_NAME=${ENVIRONMENT_NAME}
       - BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT}

frontend/Dockerfile

@@ -17,6 +17,7 @@ COPY ./src /usr/src/app/src/
 COPY ./*.sh /usr/src/app/
 COPY ./*.json /usr/src/app/
 
+RUN rm -rf /usr/src/app/node_modules/webpack
 RUN yarn build
 
 # Production environment

frontend/package.json

@@ -1,9 +1,10 @@
 {
   "name": "shuffler",
   "homepage": "https://shuffler.io",
-  "version": "0.9.24",
+  "version": "0.9.25",
   "private": true,
   "dependencies": {
+    "@babel/core": "^7.15.8",
     "@material-ui/core": "^4.5.2",
     "@material-ui/data-grid": "^4.0.0-alpha.22",
     "@material-ui/icons": "^4.11.2",
@@ -13,8 +14,8 @@
     "@uiw/react-codemirror": "^3.2.1",
     "@use-it/interval": "^1.0.0",
     "babel-eslint": "^10.1.0",
-    "class-transformer": "^0.3.1",
+    "class-transformer": "^0.4.0",
-    "create-react-app": "^2.0.3",
+    "create-react-app": "^4.0.3",
     "cytoscape": "^3.11.0",
     "cytoscape-clipboard": "^2.2.1",
     "cytoscape-cxtmenu": "^3.1.1",
@@ -62,7 +63,7 @@
     "shellwords": "^0.1.1",
     "simplebar": "^4.2.3",
     "styled-components": "^4.4.0",
-    "webpack": "4.44.2",
+    "webpack": "^4.44.2",
     "websocket": "^1.0.30",
     "yaml": "^1.7.2",
     "yamljs": "^0.3.0",

frontend/public/images/btn_google_light_focus_ios.svg

@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg width="46px" height="46px" viewBox="0 0 46 46" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:sketch="http://www.bohemiancoding.com/sketch/ns">
+    <!-- Generator: Sketch 3.3.3 (12081) - http://www.bohemiancoding.com/sketch -->
+    <title>btn_google_light_focus_ios</title>
+    <desc>Created with Sketch.</desc>
+    <defs>
+        <filter x="-50%" y="-50%" width="200%" height="200%" filterUnits="objectBoundingBox" id="filter-1">
+            <feOffset dx="0" dy="1" in="SourceAlpha" result="shadowOffsetOuter1"></feOffset>
+            <feGaussianBlur stdDeviation="0.5" in="shadowOffsetOuter1" result="shadowBlurOuter1"></feGaussianBlur>
+            <feColorMatrix values="0 0 0 0 0   0 0 0 0 0   0 0 0 0 0  0 0 0 0.168 0" in="shadowBlurOuter1" type="matrix" result="shadowMatrixOuter1"></feColorMatrix>
+            <feOffset dx="0" dy="0" in="SourceAlpha" result="shadowOffsetOuter2"></feOffset>
+            <feGaussianBlur stdDeviation="0.5" in="shadowOffsetOuter2" result="shadowBlurOuter2"></feGaussianBlur>
+            <feColorMatrix values="0 0 0 0 0   0 0 0 0 0   0 0 0 0 0  0 0 0 0.084 0" in="shadowBlurOuter2" type="matrix" result="shadowMatrixOuter2"></feColorMatrix>
+            <feMerge>
+                <feMergeNode in="shadowMatrixOuter1"></feMergeNode>
+                <feMergeNode in="shadowMatrixOuter2"></feMergeNode>
+                <feMergeNode in="SourceGraphic"></feMergeNode>
+            </feMerge>
+        </filter>
+        <rect id="path-2" x="0" y="0" width="40" height="40" rx="2"></rect>
+    </defs>
+    <g id="Google-Button" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd" sketch:type="MSPage">
+        <g id="9-PATCH" sketch:type="MSArtboardGroup" transform="translate(-668.000000, -160.000000)"></g>
+        <g id="btn_google_light_focus" sketch:type="MSArtboardGroup" transform="translate(-1.000000, -1.000000)">
+            <rect id="Rectangle-14" fill-opacity="0.3" fill="#4285F4" sketch:type="MSShapeGroup" x="1" y="1" width="46" height="46"></rect>
+            <g id="button" sketch:type="MSLayerGroup" transform="translate(4.000000, 4.000000)" filter="url(#filter-1)">
+                <g id="button-bg">
+                    <use fill="#FFFFFF" fill-rule="evenodd" sketch:type="MSShapeGroup" xlink:href="#path-2"></use>
+                    <use fill="none" xlink:href="#path-2"></use>
+                    <use fill="none" xlink:href="#path-2"></use>
+                    <use fill="none" xlink:href="#path-2"></use>
+                </g>
+            </g>
+            <g id="logo_googleg_48dp" sketch:type="MSLayerGroup" transform="translate(15.000000, 15.000000)">
+                <path d="M17.64,9.20454545 C17.64,8.56636364 17.5827273,7.95272727 17.4763636,7.36363636 L9,7.36363636 L9,10.845 L13.8436364,10.845 C13.635,11.97 13.0009091,12.9231818 12.0477273,13.5613636 L12.0477273,15.8195455 L14.9563636,15.8195455 C16.6581818,14.2527273 17.64,11.9454545 17.64,9.20454545 L17.64,9.20454545 Z" id="Shape" fill="#4285F4" sketch:type="MSShapeGroup"></path>
+                <path d="M9,18 C11.43,18 13.4672727,17.1940909 14.9563636,15.8195455 L12.0477273,13.5613636 C11.2418182,14.1013636 10.2109091,14.4204545 9,14.4204545 C6.65590909,14.4204545 4.67181818,12.8372727 3.96409091,10.71 L0.957272727,10.71 L0.957272727,13.0418182 C2.43818182,15.9831818 5.48181818,18 9,18 L9,18 Z" id="Shape" fill="#34A853" sketch:type="MSShapeGroup"></path>
+                <path d="M3.96409091,10.71 C3.78409091,10.17 3.68181818,9.59318182 3.68181818,9 C3.68181818,8.40681818 3.78409091,7.83 3.96409091,7.29 L3.96409091,4.95818182 L0.957272727,4.95818182 C0.347727273,6.17318182 0,7.54772727 0,9 C0,10.4522727 0.347727273,11.8268182 0.957272727,13.0418182 L3.96409091,10.71 L3.96409091,10.71 Z" id="Shape" fill="#FBBC05" sketch:type="MSShapeGroup"></path>
+                <path d="M9,3.57954545 C10.3213636,3.57954545 11.5077273,4.03363636 12.4404545,4.92545455 L15.0218182,2.34409091 C13.4631818,0.891818182 11.4259091,0 9,0 C5.48181818,0 2.43818182,2.01681818 0.957272727,4.95818182 L3.96409091,7.29 C4.67181818,5.16272727 6.65590909,3.57954545 9,3.57954545 L9,3.57954545 Z" id="Shape" fill="#EA4335" sketch:type="MSShapeGroup"></path>
+                <path d="M0,0 L18,0 L18,18 L0,18 L0,0 Z" id="Shape" sketch:type="MSShapeGroup"></path>
+            </g>
+            <g id="handles_square" sketch:type="MSLayerGroup"></g>
+        </g>
+    </g>
+</svg>

frontend/src/components/Header.js

@@ -276,7 +276,7 @@ const Header = props => {
 				setAnchorEl(event.currentTarget);
 			}}>
 			  <Badge badgeContent={notifications.length} color="primary">
-					<NotificationsIcon color="secondary" style={{height: 35, width: 35,}} alt="Your username here" src="" />
+					<NotificationsIcon color="secondary" style={{height: 30, width: 30,}} alt="Your username here" src="" />
 				</Badge>
 			</IconButton>
 			<Menu
@@ -325,7 +325,7 @@ const Header = props => {
 			<IconButton color="primary" style={{zIndex: 10001, marginRight: 15, }} aria-controls="simple-menu" aria-haspopup="true" onClick={(event) => {
 				setAnchorElAvatar(event.currentTarget);
 			}}>
-				<Avatar style={{height: 35, width: 35,}} alt="Your username here" src="" />
+				<Avatar style={{height: 30, width: 30,}} alt="Your username here" src="" />
 			</IconButton>
 			<Menu
 				id="simple-menu"
@@ -439,7 +439,7 @@ const Header = props => {
 				{notificationMenu}
 				{userdata === undefined || userdata.admin === undefined || userdata.admin === null || !userdata.admin ? null : 
 					<Link to="/admin" style={hrefStyle}>
-						<Button color="primary" variant="contained" style={{marginRight: 15, marginTop: 12}}>
+						<Button color="primary" variant="outlined" style={{marginRight: 15, marginTop: 12}}>
 							Admin
 						</Button>
 					</Link>

frontend/src/components/Oauth2Auth.jsx

@@ -53,23 +53,31 @@ const AuthenticationOauth2 = (props) => {
 
 		var resources = ""
 		if (scopes !== undefined && scopes !== null & scopes.length > 0) {
+			//scopes.push("offline_access")
 			resources = scopes.join(",")
 		}
 
 		const authentication_url = authenticationType.token_uri
-
+		//console.log("AUTH: ", authenticationType)
-		console.log("SCOPES2: ", resources)
+		//console.log("SCOPES2: ", resources)
 		const redirectUri = `${window.location.protocol}//${window.location.host}/set_authentication`
 		var state = `workflow_id%3D${workflow.id}%26reference_action_id%3d${selectedAction.app_id}%26app_name%3d${selectedAction.app_name}%26app_id%3d${selectedAction.app_id}%26app_version%3d${selectedAction.app_version}%26authentication_url%3d${authentication_url}%26scope%3d${resources}%26client_id%3d${client_id}%26client_secret%3d${client_secret}`
 		if (oauth_url !== undefined && oauth_url !== null && oauth_url.length > 0) {
 			state += `%26oauth_url%3d${oauth_url}`
 			console.log("ADDING OAUTH2 URL: ", state)
 		}
-		const url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&scope=${resources}&prompt=consent&state=${state}`
+
+		if (authenticationType.refresh_uri !== undefined && authenticationType.refresh_uri !== null && authenticationType.refresh_uri.length > 0) {
+			state += `%26refresh_uri%3d${authenticationType.refresh_uri}`
+		} else {
+			state += `%26refresh_uri%3d${authentication_url}`
+		}
+
+		const url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&scope=${resources}&prompt=consent&state=${state}&access_type=offline`
 
 		//const url = `https://accounts.zoho.com/oauth/v2/auth?response_type=code&client_id=${client_id}&scope=AaaServer.profile.Read&redirect_uri=${redirectUri}&prompt=consent`
-		console.log("Full URI: ", url)
+		//console.log("Full URI: ", url)
-		console.log("Redirect Uri: ", redirectUri)
+		//console.log("Redirect Uri: ", redirectUri)
 		// &resource=https%3A%2F%2Fgraph.microsoft.com&
 		
 		// FIXME: Awful, but works for prototyping

frontend/src/components/OrgHeader.js

@@ -38,6 +38,7 @@ const OrgHeader = (props) => {
 	const [workflowDownloadBranch, setWorkflowDownloadBranch] = React.useState(selectedOrganization.defaults === undefined ? defaultBranch : selectedOrganization.defaults.workflow_download_branch === undefined || selectedOrganization.defaults.workflow_download_branch.length === 0 ? defaultBranch : selectedOrganization.defaults.workflow_download_branch)
 	const [ssoEntrypoint, setSsoEntrypoint] = React.useState(selectedOrganization.sso_config === undefined ? "" : selectedOrganization.sso_config.sso_entrypoint === undefined || selectedOrganization.sso_config.sso_entrypoint.length === 0 ? "" : selectedOrganization.sso_config.sso_entrypoint)
 	const [ssoCertificate, setSsoCertificate] = React.useState(selectedOrganization.sso_config === undefined ? "" : selectedOrganization.sso_config.sso_certificate === undefined || selectedOrganization.sso_config.sso_certificate.length === 0 ? "" : selectedOrganization.sso_config.sso_certificate)
+	const [notificationWorkflow, setNotificationWorkflow] = React.useState(selectedOrganization.defaults === undefined ? "" : selectedOrganization.defaults.notification_workflow === undefined || selectedOrganization.defaults.notification_workflow.length === 0 ? "" : selectedOrganization.defaults.notification_workflow)
 
 	const [file, setFile] = React.useState("")
 	const [fileBase64, setFileBase64] = React.useState(selectedOrganization.image)
@@ -113,7 +114,7 @@ const OrgHeader = (props) => {
 		setFile(fileObject)
 	}
 
-	console.log("USER: ", userdata)
+	//console.log("USER: ", userdata)
 	const orgSaveButton = 
 		<Tooltip title="Save any unsaved data" placement="bottom">
 			<Button
@@ -126,6 +127,7 @@ const OrgHeader = (props) => {
 					"app_download_branch": appDownloadBranch,
 					"workflow_download_repo": workflowDownloadUrl,
 					"workflow_download_branch": workflowDownloadBranch,
+					"notification_workflow": notificationWorkflow,
 				},
 				{
 					"sso_entrypoint": ssoEntrypoint,
@@ -232,6 +234,35 @@ const OrgHeader = (props) => {
 					</IconButton>
 					{expanded ? 
 						<Grid container spacing={3} style={{textAlign: "left"}}>
+							<Grid item xs={12} style={{}}>
+								<span>
+									<Typography>
+										Notification Workflow ID	
+									</Typography>
+									<TextField
+										required
+										style={{flex: "1", marginTop: "5px", marginRight: "15px", backgroundColor: theme.palette.inputColor}}
+										fullWidth={true}
+										type="name"
+										id="outlined-with-placeholder"
+										margin="normal"
+										variant="outlined"
+										placeholder="ID of the workflow to receive notifications"
+										value={notificationWorkflow}
+										onChange={e => {
+											setNotificationWorkflow(e.target.value)
+										}}
+										InputProps={{
+											classes: {
+												notchedOutline: classes.notchedOutline,
+											},
+											style:{
+												color: "white",
+											},
+										}}
+										/>
+								</span>
+							</Grid>
 							<Grid item xs={6} style={{}}>
 								<span>
 									<Typography>

frontend/src/components/ParsedAction.jsx

@@ -49,6 +49,7 @@ const useStyles = makeStyles({
 //		},
 //)
 
+const openApiFieldDesc = "Generated by OpenAPI body example"
 const ParsedAction = (props) => {
 	const {workflow, setWorkflow, setAction, setSelectedAction, setUpdate, appActionArguments, selectedApp, workflowExecutions, setSelectedResult, selectedAction, setSelectedApp, setSelectedTrigger, setSelectedEdge, setCurrentView, cy, setAuthenticationModalOpen,setVariablesModalOpen, setCodeModalOpen, selectedNameChange, rightsidebarStyle, showEnvironment, selectedActionEnvironment, environments, setNewSelectedAction, appApiViewStyle, globalUrl, setSelectedActionEnvironment, requiresAuthentication, hideExtraTypes, scrollConfig, setScrollConfig, authenticationType, appAuthentication, getAppAuthentication } = props
 
@@ -56,6 +57,8 @@ const ParsedAction = (props) => {
 	const classes = useStyles()
 
 	const [expansionModalOpen, setExpansionModalOpen] = React.useState(false);
+	const [hideBody, setHideBody] = React.useState(false)
+	const [activateHidingBody, setActivateHidingBody] = React.useState(false)
 
 	const keywords = ["len(", "lower(", "upper(", "trim(", "split(", "length(", "number(", "parse(", "join("]
 	const getParents = (action) => {
@@ -258,7 +261,8 @@ const ParsedAction = (props) => {
 
 			if (actionlist.length === 0) {
 				// FIXME: Have previous execution values in here
-				actionlist.push({"type": "Execution Argument", "name": "Execution Argument", "value": "$exec", "highlight": "exec", "autocomplete": "exec", "example": "hello"})
+				actionlist.push({"type": "Execution Argument", "name": "Execution Argument", "value": "$exec", "highlight": "exec", "autocomplete": "exec", "example": ""})
+				actionlist.push({"type": "Shuffle DB", "name": "Shuffle DB", "value": "$shuffle_cache", "highlight": "shuffle", "autocomplete": "shuffle", "example": ""})
 				if (workflow.workflow_variables !== null && workflow.workflow_variables !== undefined && workflow.workflow_variables.length > 0) {
 					for (var key in workflow.workflow_variables) {
 						const item = workflow.workflow_variables[key]
@@ -788,12 +792,52 @@ const ParsedAction = (props) => {
 							//setSelectedActionParameters(selectedActionParameters)
 						}
 
+						var hideBodyButton = ""
+						const hideBodyButtonValue = 
+							<div style={{marginTop: 25, border:"1px solid rgba(255,255,255,0.7)",borderTop: "1px solid rgba(255,255,255,0.7)", borderRadius: theme.palette.borderRadius, alignItems: "center", textAlign: "center",}}>
+								<Tooltip color="secondary" title={"Automatically change body"} placement="top">
+									<FormControlLabel
+										control={
+											<Checkbox
+												tabIndex="-1"
+												checked={hideBody}
+												style={{
+													color: theme.palette.primary.secondary,
+												}}
+												onChange={(event) => {
+													setHideBody(!hideBody)
+
+													for (var key in selectedActionParameters) {
+														var currentItem = selectedActionParameters[key]
+
+														if (currentItem.description === openApiFieldDesc) {
+															currentItem.field_active = !hideBody 
+															console.log("Changing", currentItem)
+														}
+													}
+												}}
+												name="requires_unique"
+											/>
+										}
+										label={"Automatically fix body"}
+									/>
+								</Tooltip>
+							</div>
+
 						if (selectedApp.generated && data.name === "body") {
 							const regex = /\${(\w+)}/g
 							const found = placeholder.match(regex)
-							if (found === null) {
+
+							hideBodyButton = hideBodyButtonValue
+							if (found === null || !hideBody) {
 								//setExtraBodyFields([])
+								//
+								if (found === null) {
+									setActivateHidingBody(true)
+								}
 							} else {
+								console.log("SHOW BUTTON")
+
 								rows = "1"
 								disabled = true
 								openApiHelperText = "OpenAPI spec: fill the following fields."
@@ -818,7 +862,7 @@ const ParsedAction = (props) => {
 									selectedActionParameters.push({
 										action_field: "",
 										configuration: false,
-										description: "Generated by OpenAPI body example",
+										description: openApiFieldDesc,
 										example: "",
 										id: "",
 										multiline: false,
@@ -830,6 +874,7 @@ const ParsedAction = (props) => {
 										tags: null,
 										value: "",
 										variant: "STATIC_VALUE",
+										field_active: true,
 									})
 								}
 
@@ -837,10 +882,14 @@ const ParsedAction = (props) => {
 									setSelectedActionParameters(selectedActionParameters)
 								}
 
-								return <Divider key={Math.random()} /> 
+								return hideBodyButton 
 							}
 						}
 
+						if (activateHidingBody === true) {
+							hideBodyButton = ""
+						}
+
 						const clickedFieldId = "rightside_field_"+count
 						//<TextareaAutosize
 						// <CodeMirror
@@ -1000,7 +1049,8 @@ const ParsedAction = (props) => {
 							}
 							*/
 						} else if (selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0) {
-							if (selectedActionParameters[count].value === "" && selectedActionParameters[count].required) {
+							if (selectedActionParameters[count].value === "") {
+								// && selectedActionParameters[count].required) {
 								// Rofl, dirty workaround :)
 								const e = {
 									target: {
@@ -1046,6 +1096,10 @@ const ParsedAction = (props) => {
 							staticcolor = "#f85a3e"	
 						}
 
+						if (data.field_active === false) {
+							return null
+						}
+
 						// Shows nested list of nodes > their JSON lists
 						const ActionlistWrapper = (props) => {
 
@@ -1296,6 +1350,7 @@ const ParsedAction = (props) => {
 						{/*<div style={{width: 17, height: 17, borderRadius: 17 / 2, backgroundColor: itemColor, marginRight: 10, marginTop: 2, marginTop: "auto", marginBottom: "auto",}}/>*/}
 						return (
 						<div key={data.name}>	
+							{hideBodyButton}
 							<div style={{marginTop: 20, marginBottom: 0, display: "flex"}}>
 								{data.configuration === true ? 
 									<Tooltip color="primary" title={`Authenticate ${selectedApp.name}`} placement="top">
@@ -1306,6 +1361,7 @@ const ParsedAction = (props) => {
 								:
 									null
 								}
+
 								<div style={{flex: "10", marginTop: "auto", marginBottom: "auto",}}> 
 									<Tooltip title={tooltipDescription} placement="top">
 										<b>{tmpitem} </b> 
@@ -1355,7 +1411,7 @@ const ParsedAction = (props) => {
           						  onChange={(event) => {
 													//console.log("CHECKED!: ", selectedActionParameters[count])
           						  	selectedActionParameters[count].unique_toggled = !selectedActionParameters[count].unique_toggled
-													selectedAction.parameters[count].unique_toggled = selectedActionParameters[count].unique_toggled
+												  selectedAction.parameters[count].unique_toggled = selectedActionParameters[count].unique_toggled
           						  	setSelectedActionParameters(selectedActionParameters)
 													setSelectedAction(selectedAction)
 													setUpdate(Math.random())
@@ -1610,7 +1666,9 @@ const ParsedAction = (props) => {
 			: null}
 			{selectedAction.authentication !== undefined && selectedAction.authentication !== null && selectedAction.authentication.length > 0 ? 
 				<div style={{marginTop: 15, }}>
-					Authentication
+					<Typography>
+						Authentication
+					</Typography>
 					<div style={{display: "flex"}}>
 						<Select
 							labelId="select-app-auth"

frontend/src/defaultCytoscapeStyle.js

@@ -14,6 +14,7 @@ const data = [{
 				'margin': '5px',
 				'border-width': '1px',
 				'text-margin-x': '10px',
+				'cursor': 'pointer',
 			}
 		},
 		{
@@ -26,6 +27,7 @@ const data = [{
 				'text-margin-y': '-15px',
 				'width': '5px',
 				"color": "white",
+				'cursor': 'pointer',
 				"line-fill": "linear-gradient",
 				"line-gradient-stop-positions": ["0.0", "100"],
 				"line-gradient-stop-colors": ["grey", "grey"],
@@ -34,7 +36,7 @@ const data = [{
 		{
 			selector: `node[type="ACTION"]`,
 			css: {
-				'shape': 'square',
+				'shape': 'roundrectangle',
 				'background-color': '#213243',
 				'border-color': '#81c784',
 				'background-width': '100%',
@@ -99,6 +101,7 @@ const data = [{
 			selector: `node[type="TRIGGER"]`,
 			css: {
 				'shape': 'octagon',
+				'border-radius': '5px',
 				'border-color': 'orange',
 				'background-color': '#213243',
 				'background-width': '100%',
@@ -201,6 +204,16 @@ const data = [{
 				'transition-duration': '0.5s',
 			},
 		},
+		{
+			selector: '.hover-highlight',
+			css: {
+				'background-color': '#5f9265',
+				'border-color': '#5f9265',
+				'border-width': '5px',
+				'transition-property': 'background-color',
+				'transition-duration': '0.5s',
+			},
+		},
 		{
 			selector: '.failure-highlight',
 			css: {

frontend/src/views/Admin.jsx

@@ -2717,7 +2717,7 @@ const Admin = (props) => {
 						style={{minWidth: 150, maxWidth: 150}}
 					/>
 					<ListItemText
-						primary="Orborus running (TBD)"
+						primary="Orborus running"
 						style={{minWidth: 200, maxWidth: 200}}
 					/>
 					<ListItemText
@@ -2736,6 +2736,10 @@ const Admin = (props) => {
 						primary="Archived"
 						style={{minWidth: 150, maxWidth: 150}}
 					/>
+					<ListItemText
+						primary="Last Changed"
+						style={{minWidth: 150, maxWidth: 150}}
+					/>
 				</ListItem>
 				{environments === undefined || environments === null ? null : environments.map((environment, index)=> {
 					if (!showArchived && environment.archived) {
@@ -2747,19 +2751,19 @@ const Admin = (props) => {
 						return null
 					}
 
-					//var bgColor = "#27292d"
+					var bgColor = "#27292d"
-					//if (index % 2 === 0) {
+					if (index % 2 === 0) {
-					//	bgColor = "#1f2023"
+						bgColor = "#1f2023"
-					//}
+					}
 
 					return (
-						<ListItem key={index}>
+						<ListItem key={index} style={{backgroundColor: bgColor}}>
 							<ListItemText
 								primary={environment.Name}
 								style={{minWidth: 150, maxWidth: 150, overflow: "hidden"}}
 							/>
 							<ListItemText
-								primary={environment.Type === "cloud" ? "N/A" : "TBD"}
+								primary={environment.Type !== "cloud" ? environment.running_ip === undefined || environment.running_ip === null || environment.running_ip.length === 0 ? "Not started" : environment.running_ip : "N/A"}
 								style={{minWidth: 200, maxWidth: 200, overflow: "hidden"}}
 							/>
 							<ListItemText
@@ -2786,6 +2790,10 @@ const Admin = (props) => {
 								style={{minWidth: 150, maxWidth: 150, overflow: "hidden"}}
 								primary={environment.archived.toString()}
 							/>
+							<ListItemText
+								style={{minWidth: 150, maxWidth: 150, overflow: "hidden"}}
+								primary={environment.edited !== undefined && environment.edited !== null && environment.edited !== 0 ? new Date(environment.edited*1000).toISOString() : 0}
+							/>
 						</ListItem>
 					)
 				})}
@@ -2907,7 +2915,7 @@ const Admin = (props) => {
 						style={{minWidth: 150, maxWidth: 150}}
 					/>
 					<ListItemText
-						primary="Orborus running (TBD)"
+						primary="Orborus running"
 						style={{minWidth: 200, maxWidth: 200}}
 					/>
 					<ListItemText

frontend/src/views/AngularWorkflow.jsx

@@ -11,7 +11,7 @@ import NestedMenuItem from "material-ui-nested-menu-item";
 import ReactMarkdown from 'react-markdown';
 
 
-import {TextField, Drawer, Button, Paper, Grid, Tabs, InputAdornment, Tab, ButtonBase, Tooltip, Select, MenuItem, Divider, Dialog, Modal, DialogActions, DialogTitle, InputLabel, DialogContent, FormControl, IconButton, Menu, Input, FormGroup, FormControlLabel, Typography, Checkbox, Breadcrumbs, CircularProgress, Switch, Fade} from '@material-ui/core';
+import {Slide, TextField, Drawer, Button, Paper, Grid, Tabs, InputAdornment, Tab, ButtonBase, Tooltip, Select, MenuItem, Divider, Dialog, Modal, DialogActions, DialogTitle, InputLabel, DialogContent, FormControl, IconButton, Menu, Input, FormGroup, FormControlLabel, Typography, Checkbox, Breadcrumbs, CircularProgress, Switch, Fade} from '@material-ui/core';
 import {OpenInNew as OpenInNewIcon,Undo as UndoIcon, FileCopy as FileCopyIcon, GetApp as GetAppIcon, Search as SearchIcon, ArrowUpward as ArrowUpwardIcon, Visibility as VisibilityIcon, Done as DoneIcon, Close as CloseIcon, Error as ErrorIcon, FindReplace as FindreplaceIcon, ArrowLeft as ArrowLeftIcon, Cached as CachedIcon, DirectionsRun as DirectionsRunIcon, Add as AddIcon, Polymer as PolymerIcon, FormatListNumbered as FormatListNumberedIcon, Create as CreateIcon, PlayArrow as PlayArrowIcon, AspectRatio as AspectRatioIcon, MoreVert as MoreVertIcon, Apps as AppsIcon, Schedule as ScheduleIcon, FavoriteBorder as FavoriteBorderIcon, Pause as PauseIcon, Delete as DeleteIcon, AddCircleOutline as AddCircleOutlineIcon, Save as SaveIcon, KeyboardArrowLeft as KeyboardArrowLeftIcon, KeyboardArrowRight as KeyboardArrowRightIcon, ArrowBack as ArrowBackIcon, Settings as SettingsIcon, LockOpen as LockOpenIcon, ExpandMore as ExpandMoreIcon, VpnKey as VpnKeyIcon} from '@material-ui/icons';
 
 import * as cytoscape from 'cytoscape';
@@ -887,6 +887,8 @@ const AngularWorkflow = (props) => {
 					//console.log(curworkflowTrigger)
 
 					newTriggers.push(curworkflowTrigger)
+				} else {
+					alert.info("No handler for type: "+type)
 				}
 			}
 		}
@@ -1258,9 +1260,9 @@ const AngularWorkflow = (props) => {
 									selectedAction.selectedAuthentication = item
 
 									for (var key in workflow.actions) {
-										console.log(workflow.actions[key].app_name)
+										//console.log(workflow.actions[key].app_name)
 										if (workflow.actions[key].app_name == selectedApp.name) {
-											console.log("Setting auth at: ", workflow.actions[key], item.id)
+											//console.log("Setting auth at: ", workflow.actions[key], item.id)
 											workflow.actions[key].selectedAuthentication = item
 											workflow.actions[key].authentication_id = item.id
 											appUpdates = true
@@ -1593,6 +1595,19 @@ const AngularWorkflow = (props) => {
 		if (event.target.data().decorator) {
 			alert.info("This edge can't be edited.")
 		} else {
+			//console.log("DATA: ", event.target.data())
+			const destinationId = event.target.data("target")
+			//console.log("DATA: ", event.target.data())
+			const curaction = workflow.actions.find(a => a.id === destinationId)
+			//console.log("ACTION: ", curaction)
+			if (curaction !== undefined && curaction !== null) {
+				if (curaction.app_name == "Shuffle Tools" && curaction.name === "router") {
+					alert.info("Router action can't have incoming conditions")
+					event.target.unselect()
+					return
+				}
+			}
+
 			setSelectedEdgeIndex(workflow.branches.findIndex(data => data.id === event.target.data()["id"]))
 			setSelectedEdge(event.target.data())
 		}
@@ -1941,6 +1956,7 @@ const AngularWorkflow = (props) => {
 					}
 
 					workflow.start = parentNode.data('id')
+					setLastSaved(true)
 					parentNode.data('isStartNode', true) 
 				}
 
@@ -2476,7 +2492,7 @@ const AngularWorkflow = (props) => {
 	// Checks for errors in edges when they're added 
 	const onEdgeAdded = (event) => {
 		const edge = event.target.data()
-		console.log("EDGE ADDED: ", edge)
+		//console.log("EDGE ADDED: ", edge)
 		//setLastSaved(false)
 		var targetnode = workflow.triggers.findIndex(data => data.id === edge.target)
 		if (targetnode !== -1) {
@@ -2488,7 +2504,7 @@ const AngularWorkflow = (props) => {
 			}
 		}
 
-		console.log("TARGET: ", event.target.target().data())
+		//console.log("TARGET: ", event.target.target().data())
 		if (event.target.target().data("isButton") === true || event.target.target().data("isDescriptor") === true) {
 			event.target.remove()
 			return
@@ -2496,7 +2512,7 @@ const AngularWorkflow = (props) => {
 
 		targetnode = -1
 		var sourcenode = workflow.triggers.findIndex(data => data.id === edge.source)
-		console.log("SOURCENODE: ", sourcenode)
+		//console.log("SOURCENODE: ", sourcenode)
 		if (sourcenode !== -1) {
 			if (workflow.triggers[sourcenode].app_name === "User Input" || workflow.triggers[sourcenode].app_name === "Shuffle Workflow") {
 				//console.log("NORMAL TRIGGER")
@@ -3304,6 +3320,16 @@ const AngularWorkflow = (props) => {
 		}, {
 			duration: animationDuration,	
 		})
+
+		const outgoingEdges = event.target.outgoers('edge')
+		const incomingEdges = event.target.incomers('edge')
+		if (outgoingEdges.length > 0) {
+			outgoingEdges.removeClass('hover-highlight')
+		}
+
+		if (incomingEdges.length > 0) {
+			outgoingEdges.removeClass('hover-highlight')
+		}
 	}
 
 	const buttonColor = "rgba(255,255,255,0.9)"
@@ -3479,6 +3505,16 @@ const AngularWorkflow = (props) => {
 		})
 
 		previousnodecolor = event.target.style("border-color")
+
+		const outgoingEdges = event.target.outgoers('edge')
+		const incomingEdges = event.target.incomers('edge')
+		if (outgoingEdges.length > 0) {
+			outgoingEdges.addClass('hover-highlight')
+		}
+
+		if (incomingEdges.length > 0) {
+			outgoingEdges.addClass('hover-highlight')
+		}
 	}
 
 	const onEdgeHoverOut = (event) => {
@@ -4666,9 +4702,9 @@ const AngularWorkflow = (props) => {
 							</div>
 						:
 							<div style={{textAlign: "center", width: leftBarSize}}>
-								<CircularProgress style={{marginTop: 25, height: 35, width: 35, marginLeft: "auto", marginRight: "auto", }} /> 
+								<CircularProgress style={{marginTop: "27vh", height: 35, width: 35, marginLeft: "auto", marginRight: "auto", }} /> 
 								<Typography variant="body1" color="textSecondary">
-									Loading apps
+									Loading Apps
 								</Typography>
 							</div>
 					}
@@ -4942,7 +4978,7 @@ const AngularWorkflow = (props) => {
 		top: appBarSize+25,
 		right: 25, 
 		height: "80vh",
-		width: 350, 
+		width: 365, 
 		minWidth: 200, 
 		maxWidth: 600,
 		maxHeight: "100vh",
@@ -5768,7 +5804,7 @@ const AngularWorkflow = (props) => {
 			<Button 
 				fullWidth
 				variant="contained" 
-				style={{flex: 1, marginTop: 10, }} 
+				style={{flex: 1, textTransform: "none", textAlign: "left", justifyContent: "flex-start", marginTop: 10, padding: 0, backgroundColor: "#4285f4", color: "white", }} 
 				color="primary"
 				onClick={() => {
 					//const redirectUri = isCloud ? "https%3A%2F%2Fshuffler.io%2Fapi%2Fv1%2Ftriggers%2Foutlook%2Fregister" : "http%3A%2F%2Flocalhost:5001%2Fapi%2Fv1%2Ftriggers%2Fgmail%2Fregister"
@@ -5826,8 +5862,9 @@ const AngularWorkflow = (props) => {
 					}, 2500)
 
 					saveWorkflow(workflow)
-				}} >
+				}}>
-				Sign in with Google	
+				<img style={{margin: 0, }} src="/images/btn_google_light_focus_ios.svg" />
+				<Typography style={{margin: 0, marginLeft: 10, }} variant="body1">Sign in with Google</Typography>
 			</Button>
 
 		const outlookButton = selectedTrigger.name !== "Office365" ? null :
@@ -6055,12 +6092,12 @@ const AngularWorkflow = (props) => {
 					<div>
 						<Divider style={{marginTop: "20px", height: "1px", width: "100%", backgroundColor: "rgb(91, 96, 100)"}}/>
 						<div style={{marginTop: "20px", marginBottom: "7px", display: "flex"}}>
-							<Button style={{flex: "1",}} disabled={selectedTrigger.status === "running"} onClick={() => {
+							<Button variant="contained" style={{flex: "1",}} disabled={selectedTrigger.status === "running" || triggerFolders === undefined || triggerFolders === null || triggerFolders.length === 0} onClick={() => {
 								startMailSub(selectedTrigger, selectedTriggerIndex)
 							}} color="primary">
 								Start	
 							</Button>
-							<Button style={{flex: "1",}} disabled={selectedTrigger.status !== "running" } onClick={() => {
+							<Button variant="outlined" style={{flex: "1",}} disabled={selectedTrigger.status !== "running" } onClick={() => {
 								stopMailSub(selectedTrigger, selectedTriggerIndex)
 							}} color="primary">
 								Stop	
@@ -7798,46 +7835,46 @@ const AngularWorkflow = (props) => {
 			} 
 
 			return (
-				<div id="rightside_actions" style={rightsidebarStyle}>
+					<div id="rightside_actions" style={rightsidebarStyle}>
-					<ParsedAction 
+						<ParsedAction 
-						id="rightside_subactions"
+							id="rightside_subactions"
-						getAppAuthentication={getAppAuthentication}
+							getAppAuthentication={getAppAuthentication}
-						appAuthentication={appAuthentication}
+							appAuthentication={appAuthentication}
-						authenticationType={authenticationType}
+							authenticationType={authenticationType}
-						scrollConfig={scrollConfig}
+							scrollConfig={scrollConfig}
-						setScrollConfig={setScrollConfig}
+							setScrollConfig={setScrollConfig}
-						selectedAction={selectedAction}
+							selectedAction={selectedAction}
-						workflow={workflow} 
+							workflow={workflow} 
-						setWorkflow={setWorkflow} 
+							setWorkflow={setWorkflow} 
-						setSelectedAction={setSelectedAction}
+							setSelectedAction={setSelectedAction}
-						setUpdate={setUpdate}
+							setUpdate={setUpdate}
-						selectedApp={selectedApp}
+							selectedApp={selectedApp}
-						workflowExecutions={workflowExecutions}
+							workflowExecutions={workflowExecutions}
-						setSelectedResult={setSelectedResult}
+							setSelectedResult={setSelectedResult}
-						setSelectedApp={setSelectedApp}
+							setSelectedApp={setSelectedApp}
-						setSelectedTrigger={setSelectedTrigger}
+							setSelectedTrigger={setSelectedTrigger}
-						setSelectedEdge={setSelectedEdge}
+							setSelectedEdge={setSelectedEdge}
-						setCurrentView={setCurrentView}
+							setCurrentView={setCurrentView}
-						cy={cy}
+							cy={cy}
-						setAuthenticationModalOpen={setAuthenticationModalOpen}
+							setAuthenticationModalOpen={setAuthenticationModalOpen}
 
-						setVariablesModalOpen={setVariablesModalOpen}
+							setVariablesModalOpen={setVariablesModalOpen}
-						setLastSaved={setLastSaved}
+							setLastSaved={setLastSaved}
-						setCodeModalOpen={setCodeModalOpen}
+							setCodeModalOpen={setCodeModalOpen}
-						selectedNameChange={selectedNameChange}
+							selectedNameChange={selectedNameChange}
-						rightsidebarStyle={rightsidebarStyle}
+							rightsidebarStyle={rightsidebarStyle}
-						showEnvironment={showEnvironment}
+							showEnvironment={showEnvironment}
-						selectedActionEnvironment={selectedActionEnvironment}
+							selectedActionEnvironment={selectedActionEnvironment}
-						environments={environments}
+							environments={environments}
-						setNewSelectedAction={setNewSelectedAction}
+							setNewSelectedAction={setNewSelectedAction}
-						sortByKey={sortByKey}
+							sortByKey={sortByKey}
 							
-						appApiViewStyle={appApiViewStyle}
+							appApiViewStyle={appApiViewStyle}
-						globalUrl={globalUrl}
+							globalUrl={globalUrl}
-						setSelectedActionEnvironment={setSelectedActionEnvironment}
+							setSelectedActionEnvironment={setSelectedActionEnvironment}
-						requiresAuthentication={requiresAuthentication}
+							requiresAuthentication={requiresAuthentication}
-					/>
+						/>
-				</div>
+					</div>
 			)
 
 		} else if (Object.getOwnPropertyNames(selectedTrigger).length > 0) {
@@ -8097,7 +8134,7 @@ const AngularWorkflow = (props) => {
 	}
 	
 	const executionModal = 
-		<Drawer anchor={"right"} open={executionModalOpen} onClose={() => setExecutionModalOpen(false)} style={{resize: "both", overflow: "auto", zIndex: 10005}} PaperProps={{style: {resize: "both", overflow: "auto", minWidth: 400, maxWidth: 400, backgroundColor: "#1F2023", color: "white", fontSize: 18, zIndex: 10005}}}>
+		<Drawer anchor={"right"} open={executionModalOpen} onClose={() => setExecutionModalOpen(false)} style={{resize: "both", overflow: "auto", zIndex: 10005}} PaperProps={{style: {resize: "both", overflow: "auto", minWidth: 420, maxWidth: 420, backgroundColor: "#1F2023", color: "white", fontSize: 18, zIndex: 10005}}}>
 			{executionModalView === 0 ?
 			<div style={{padding: 25, }}>
 				<Breadcrumbs aria-label="breadcrumb" separator="›" style={{color: "white", fontSize: 16}}>
@@ -8109,6 +8146,7 @@ const AngularWorkflow = (props) => {
 				<Button 
 					style={{borderRadius: "0px"}}
 					variant="outlined"
+					fullWidth
 					onClick={() => {
 						getWorkflowExecution(props.match.params.key, "")
 					}} color="primary">
@@ -8568,6 +8606,7 @@ const AngularWorkflow = (props) => {
 						maxHeight: 700,
 						overflowY: "auto",
 						overflowX: "hidden",
+						zIndex: 10012,
 						//boxShadow: "none",
 					},
 				}}
@@ -8730,60 +8769,70 @@ const AngularWorkflow = (props) => {
 		<div style={{color: "white"}}>
 			<div style={{display: "flex", borderTop: "1px solid rgba(91, 96, 100, 1)"}}>
 				{leftView}
-				<CytoscapeComponent 
+				{workflow.id === undefined || workflow.id === null || apps.length === 0 ? 
-					elements={elements} 
+					<div style={{width: bodyWidth-leftBarSize-15, height: 150, textAlign: "center"}}>
-					minZoom={0.35}
+						<CircularProgress style={{marginTop: "30vh", height: 35, width: 35, marginLeft: "auto", marginRight: "auto", }} /> 
-					maxZoom={2.00}
+						<Typography variant="body1" color="textSecondary">
-					wheelSensitivity={0.25}
+							Loading Workflow 
-					style={{width: bodyWidth-leftBarSize-15, height: bodyHeight-appBarSize-5, backgroundColor: surfaceColor}} 
+						</Typography>
-					stylesheet={cystyle}
+					</div>	
-					boxSelectionEnabled={true}
+				: 
-					autounselectify={false}
+					<CytoscapeComponent 
-					showGrid={true}
+						elements={elements} 
-					id="cytoscape_view"
+						minZoom={0.35}
-					cy={(incy) => {
+						maxZoom={2.00}
-						// FIXME: There's something specific loading when
+						wheelSensitivity={0.25}
-						// you do the first hover of a node. Why is this different?
+						style={{width: bodyWidth-leftBarSize-15, height: bodyHeight-appBarSize-5, backgroundColor: surfaceColor}} 
-						//console.log("CY: ", incy)
+						stylesheet={cystyle}
-						setCy(incy)
+						boxSelectionEnabled={true}
-					}}
+						autounselectify={false}
-				/>
+						showGrid={true}
+						id="cytoscape_view"
+						cy={(incy) => {
+							// FIXME: There's something specific loading when
+							// you do the first hover of a node. Why is this different?
+							//console.log("CY: ", incy)
+							setCy(incy)
+						}}
+					/>
+				}
 			</div>
 			{executionModal}
-			<RightSideBar 
+			{/*<Slide appear={true} direction="right" timeout={5000} in={true}>*/}
-				scrollConfig={scrollConfig}
+				<RightSideBar 
-				setScrollConfig={setScrollConfig}
+					scrollConfig={scrollConfig}
-				selectedAction={selectedAction}
+					setScrollConfig={setScrollConfig}
-				workflow={workflow} 
+					selectedAction={selectedAction}
-				setWorkflow={setWorkflow} 
+					workflow={workflow} 
-				setSelectedAction={setSelectedAction}
+					setWorkflow={setWorkflow} 
-				setUpdate={setUpdate}
+					setSelectedAction={setSelectedAction}
-				selectedApp={selectedApp}
+					setUpdate={setUpdate}
-				workflowExecutions={workflowExecutions}
+					selectedApp={selectedApp}
-				setSelectedResult={setSelectedResult}
+					workflowExecutions={workflowExecutions}
-				setSelectedApp={setSelectedApp}
+					setSelectedResult={setSelectedResult}
-				setSelectedTrigger={setSelectedTrigger}
+					setSelectedApp={setSelectedApp}
-				setSelectedEdge={setSelectedEdge}
+					setSelectedTrigger={setSelectedTrigger}
-				setCurrentView={setCurrentView}
+					setSelectedEdge={setSelectedEdge}
-				cy={cy}
+					setCurrentView={setCurrentView}
-				setAuthenticationModalOpen={setAuthenticationModalOpen}
+					cy={cy}
+					setAuthenticationModalOpen={setAuthenticationModalOpen}
 
-				setVariablesModalOpen={setVariablesModalOpen}
+					setVariablesModalOpen={setVariablesModalOpen}
-				setLastSaved={setLastSaved}
+					setLastSaved={setLastSaved}
-				setCodeModalOpen={setCodeModalOpen}
+					setCodeModalOpen={setCodeModalOpen}
-				selectedNameChange={selectedNameChange}
+					selectedNameChange={selectedNameChange}
-				rightsidebarStyle={rightsidebarStyle}
+					rightsidebarStyle={rightsidebarStyle}
-				showEnvironment={showEnvironment}
+					showEnvironment={showEnvironment}
-				selectedActionEnvironment={selectedActionEnvironment}
+					selectedActionEnvironment={selectedActionEnvironment}
-				environments={environments}
+					environments={environments}
-				setNewSelectedAction={setNewSelectedAction}
+					setNewSelectedAction={setNewSelectedAction}
-				sortByKey={sortByKey}
+					sortByKey={sortByKey}
 					
-				appApiViewStyle={appApiViewStyle}
+					appApiViewStyle={appApiViewStyle}
-				globalUrl={globalUrl}
+					globalUrl={globalUrl}
-				setSelectedActionEnvironment={setSelectedActionEnvironment}
+					setSelectedActionEnvironment={setSelectedActionEnvironment}
-				requiresAuthentication={requiresAuthentication}
+					requiresAuthentication={requiresAuthentication}
-			/>
+				/>
 			<BottomCytoscapeBar />
 			<TopCytoscapeBar />
 		</div> 
@@ -9299,6 +9348,7 @@ const AngularWorkflow = (props) => {
 					maxHeight: 700,
 					padding: 15, 
 					overflow: "hidden",
+					zIndex: 10012,
 				},
 			}}
 		>

frontend/src/views/Apps.jsx

@@ -370,8 +370,13 @@ const Apps = (props) => {
 			}} />
 
 		var newAppname = data.name
-		newAppname = newAppname.charAt(0).toUpperCase()+newAppname.substring(1)
+		if (newAppname === undefined) {
-		newAppname = newAppname.replaceAll("_", " ")
+			newAppname = "Undefined"
+		} else {
+			newAppname = newAppname.charAt(0).toUpperCase()+newAppname.substring(1)
+			newAppname = newAppname.replaceAll("_", " ")
+		}
+
 		var sharing = "public"
 		if (!data.sharing) {
 			sharing = "private"
@@ -1073,11 +1078,11 @@ const Apps = (props) => {
 							<CircularProgress style={{width: 40, height: 40, margin: "auto"}}/>
 							:
 							<Paper square style={uploadViewPaperStyle}>
-								<Typography variant="body1" style={{margin: 10}}>
+								<Typography variant="body2" color="textSecondary" style={{margin: 10}}>
 									No apps have been created, uploaded or downloaded yet. Click "Load existing apps" above to get the baseline. This may take a while as its building docker images.
 								</Typography>
-								<Typography variant="body1" style={{margin: 10}}>
+								<Typography variant="body2" color="textSecondary" style={{margin: 10}}>
-									If you're still not able to see any apps, please follow our <a rel="noopener noreferrer" href={"https://shuffler.io/docs/troubleshooting#load_all_apps_locally"} style={{textDecoration: "none", color: "#f85a3e"}} target="_blank">troubleshooting guide for loading apps!</a>
+									If you're still not able to see any apps, please follow our <a href={"https://shuffler.io/docs/troubleshooting#load_all_apps_locally"} style={{textDecoration: "none", color: "#f85a3e"}} target="_blank">troubleshooting guide for loading apps!</a>
 								</Typography>
 							</Paper>
 						}

frontend/src/views/SetAuthentication.jsx

@@ -95,6 +95,14 @@ const SetAuthentication = (props) => {
 				if (query[0] === "oauth_url") {
 					appAuthData.fields.push({"key": "oauth_url", "value": query[1]})
 				}
+
+				if (query[0] === "refresh_uri") {
+					appAuthData.fields.push({"key": "refresh_uri", "value": query[1]})
+				}
+
+				if (query[0] === "refresh_url") {
+					appAuthData.fields.push({"key": "refresh_url", "value": query[1]})
+				}
 			}
 		}
 

frontend/src/views/SettingsPage.jsx

@@ -20,6 +20,8 @@ const Settings = (props) => {
 	const [currentPassword, setCurrentPassword] = useState("");
 	const [newPassword, setNewPassword] = useState("");
 	const [newPassword2, setNewPassword2] = useState("");
+	const [file, setFile] = React.useState("")
+	const [fileBase64, setFileBase64] = React.useState(userdata.image === undefined || userdata.image === null ? theme.palette.defaultImage : userdata.image)
 
 	// Used for error messages etc
 	const [formMessage, ] = useState("");
@@ -40,6 +42,7 @@ const Settings = (props) => {
 	const boxStyle = {
 		flex: "1",
 		color: "white",
+		position: "relative",
 		marginLeft: "10px",
 		marginRight: "10px",
 		paddingLeft: "30px",
@@ -168,9 +171,81 @@ const Settings = (props) => {
 	})
 
 	// Random names for type & autoComplete. Didn't research :^)
+	var imageData = file.length > 0 ? file : fileBase64 
+	imageData = imageData === undefined || imageData.length === 0 ? theme.palette.defaultImage : imageData
+	const imageInfo = <img src={imageData} alt="Click to upload an image (174x174)" id="logo" style={{maxWidth: 100, maxHeight: 100, minWidth: 100, minHeight: 100, position: "absolute", top: -80, left: 1020/2-25, borderRadius: 50, objectFit: "contain", border: "2px solid rgba(255,255,255,0.7)"}} />
 	const landingpageData = 
-			<div style={{display: "flex", marginTop: "80px"}}>
+			<div style={{display: "flex", marginTop: 120}}>
 				<Paper style={boxStyle}>
+					{imageInfo}
+					<h2>Settings</h2>
+					<div style={{flex: "1", display: "flex", flexDirection: "row"}}>
+						<TextField
+							style={{backgroundColor: theme.palette.inputColor, flex: "1"}}
+							InputProps={{
+								style:{
+									height: "50px", 
+									color: "white",
+								},
+							}}
+							color="primary"
+							required
+							disabled
+							fullWidth={true}
+							value={username}
+							placeholder="Username"
+							type="username"
+						  id="standard-required"
+							autoComplete="username"
+							margin="normal"
+							variant="outlined"
+      	 			//onChange={e => setUsername(e.target.value)}
+						/>
+					</div>
+					<div style={{flex: "1", display: "flex", flexDirection: "row"}}>
+						<TextField
+							style={{backgroundColor: theme.palette.inputColor, flex: "1", marginRight: "15px"}}
+							InputProps={{
+								style:{
+									height: "50px", 
+									color: "white",
+								},
+							}}
+							color="primary"
+							required
+							fullWidth={true}
+							value={firstname}
+							placeholder="First Name"
+							type="firstname"
+							disabled
+						  id="standard-required"
+							autoComplete="firstname"
+							margin="normal"
+							variant="outlined"
+      	 					onChange={e => setFirstname(e.target.value)}
+						/>
+						<TextField
+							style={{backgroundColor: theme.palette.inputColor, flex: "1", marginLeft: "15px"}}
+							InputProps={{
+								style:{
+									height: "50px", 
+									color: "white",
+								},
+							}}
+							color="primary"
+							required
+							value={lastname}
+							fullWidth={true}
+							placeholder="Last Name"
+							disabled
+							type="lastname"
+						  id="standard-required"
+							autoComplete="lastname"
+							margin="normal"
+							variant="outlined"
+      	 					onChange={e => setLastname(e.target.value)}
+						/>
+					</div>
 					<h2>APIKEY</h2>
 					<a target="_blank" href="/docs/API#authentication" style={{textDecoration: "none", color: "#f85a3e"}}>What is the API key used for?</a>
 					<TextField
@@ -193,78 +268,12 @@ const Settings = (props) => {
 					/>
 					<Button
 						style={{width: "100%", height: "40px", marginTop: "10px"}}
-						variant="contained"
+						variant="outlined"
 						color="primary"
 						onClick={() => generateApikey()}
 					>Re-Generate APIKEY</Button>
 					<Divider style={{marginTop: "40px"}}/>
 					{/*
-					<h2>Settings</h2>
-					<div style={{flex: "1", display: "flex", flexDirection: "row"}}>
-						<TextField
-							style={{backgroundColor: theme.palette.inputColor, flex: "1"}}
-							InputProps={{
-								style:{
-									height: "50px", 
-									color: "white",
-								},
-							}}
-							color="primary"
-							required
-							fullWidth={true}
-							value={username}
-							placeholder="Username"
-							type="username"
-						  	id="standard-required"
-							autoComplete="username"
-							margin="normal"
-							variant="outlined"
-      	 					onChange={e => setUsername(e.target.value)}
-						/>
-					</div>
-					<div style={{flex: "1", display: "flex", flexDirection: "row"}}>
-						<TextField
-							style={{backgroundColor: theme.palette.inputColor, flex: "1", marginRight: "15px"}}
-							InputProps={{
-								style:{
-									height: "50px", 
-									color: "white",
-								},
-							}}
-							color="primary"
-							required
-							fullWidth={true}
-							value={firstname}
-							placeholder="First Name"
-							type="firstname"
-						  	id="standard-required"
-							autoComplete="firstname"
-							margin="normal"
-							variant="outlined"
-      	 					onChange={e => setFirstname(e.target.value)}
-						/>
-						<TextField
-							style={{backgroundColor: theme.palette.inputColor, flex: "1", marginLeft: "15px"}}
-							InputProps={{
-								style:{
-									height: "50px", 
-									color: "white",
-								},
-							}}
-							color="primary"
-							required
-							value={lastname}
-							fullWidth={true}
-							placeholder="Last Name"
-							type="lastname"
-						  	id="standard-required"
-							autoComplete="lastname"
-							margin="normal"
-							variant="outlined"
-      	 					onChange={e => setLastname(e.target.value)}
-						/>
-					</div>
-					<div style={{flex: "1", display: "flex", flexDirection: "row"}}>
 						<TextField
 							style={{backgroundColor: theme.palette.inputColor, flex: "1", marginRight: "15px",}}
 							InputProps={{

frontend/src/views/Workflows.jsx

@@ -2,8 +2,9 @@ import React, { useEffect} from 'react';
 import { makeStyles } from '@material-ui/core/styles';
 import { useTheme } from '@material-ui/core/styles';
 
-import { Grid, Paper, Tooltip,  Button, TextField, FormControl, IconButton, Menu, MenuItem,  Chip,  Typography,  CircularProgress, Dialog, DialogTitle, DialogActions, DialogContent} from '@material-ui/core';
+import {Badge, Avatar, Grid, Paper, Tooltip, Divider, Button, TextField, FormControl, IconButton, Menu, MenuItem, FormControlLabel, Chip, Switch, Typography, Zoom, CircularProgress, Dialog, DialogTitle, DialogActions, DialogContent} from '@material-ui/core';
-import {GridOn as GridOnIcon, List as ListIcon, Close as CloseIcon, Compare as CompareIcon, Maximize as MaximizeIcon, Minimize as MinimizeIcon, AddCircle as AddCircleIcon, Toc as TocIcon, Send as SendIcon, Search as SearchIcon, FileCopy as FileCopyIcon, Delete as DeleteIcon, BubbleChart as BubbleChartIcon, Restore as RestoreIcon, Cached as CachedIcon, GetApp as GetAppIcon,  Edit as EditIcon, MoreVert as MoreVertIcon, PlayArrow as PlayArrowIcon, Add as AddIcon, Publish as PublishIcon, CloudUpload as CloudUploadIcon, CloudDownload as CloudDownloadIcon} from '@material-ui/icons';
+import {GridOn as GridOnIcon, List as ListIcon, Close as CloseIcon, Compare as CompareIcon, Maximize as MaximizeIcon, Minimize as MinimizeIcon, AddCircle as AddCircleIcon, Toc as TocIcon, Send as SendIcon, Search as SearchIcon, FileCopy as FileCopyIcon, Delete as DeleteIcon, BubbleChart as BubbleChartIcon, Restore as RestoreIcon, Cached as CachedIcon, GetApp as GetAppIcon, Apps as AppsIcon, Edit as EditIcon, MoreVert as MoreVertIcon, PlayArrow as PlayArrowIcon, Add as AddIcon, Publish as PublishIcon, CloudUpload as CloudUploadIcon, CloudDownload as CloudDownloadIcon} from '@material-ui/icons';
+
 import NestedMenuItem from "material-ui-nested-menu-item";
 //import {Search as SearchIcon, ArrowUpward as ArrowUpwardIcon, Visibility as VisibilityIcon, Done as DoneIcon, Close as CloseIcon, Error as ErrorIcon, FindReplace as FindreplaceIcon, ArrowLeft as ArrowLeftIcon, Cached as CachedIcon, DirectionsRun as DirectionsRunIcon, Add as AddIcon, Polymer as PolymerIcon, FormatListNumbered as FormatListNumberedIcon, Create as CreateIcon, PlayArrow as PlayArrowIcon, AspectRatio as AspectRatioIcon, MoreVert as MoreVertIcon, Apps as AppsIcon, Schedule as ScheduleIcon, FavoriteBorder as FavoriteBorderIcon, Pause as PauseIcon, Delete as DeleteIcon, AddCircleOutline as AddCircleOutlineIcon, Save as SaveIcon, KeyboardArrowLeft as KeyboardArrowLeftIcon, KeyboardArrowRight as KeyboardArrowRightIcon, ArrowBack as ArrowBackIcon, Settings as SettingsIcon, LockOpen as LockOpenIcon, ExpandMore as ExpandMoreIcon, VpnKey as VpnKeyIcon} from '@material-ui/icons';
 
@@ -291,6 +292,7 @@ const Workflows = (props) => {
 	const theme = useTheme();
 	const alert = useAlert()
 	const classes = useStyles(theme);
+	const imgSize = 60
 
 	const referenceUrl = globalUrl+"/api/v1/hooks/"
 
@@ -326,6 +328,7 @@ const Workflows = (props) => {
 	const [view, setView] = React.useState("grid")
 	const [filters, setFilters] = React.useState([])
 	const [submitLoading, setSubmitLoading] = React.useState(false)
+	const [actionImageList, setActionImageList] = React.useState([])
 
 	const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" 
 
@@ -344,6 +347,7 @@ const Workflows = (props) => {
 				found = filters.map(filter => curWorkflow.name.toLowerCase().includes(filter))
 			} else {
 				found = filters.map(filter => {
+					const newfilter = filter.toLowerCase()
 					if (filter === undefined) {
 						return false
 					}
@@ -357,10 +361,9 @@ const Workflows = (props) => {
 					} else if (curWorkflow.org_id === filter) {
 						return true
 					} else if (curWorkflow.actions !== null && curWorkflow.actions !== undefined) {
-						const newfilter = filter.toLowerCase()
 						for (var key in curWorkflow.actions) {
 							const action = curWorkflow.actions[key]
-							if (action.app_name.toLowerCase().includes(newfilter)) {
+							if (action.app_name.toLowerCase() === newfilter || action.app_name.toLowerCase().includes(newfilter)) {
 								return true
 							}
 						}
@@ -390,7 +393,7 @@ const Workflows = (props) => {
 			return
 		}
 
-		if (filters.includes(data)) {
+		if (filters.includes(data) || filters.includes(data.toLowerCase())) {
 			return
 		}
 
@@ -629,6 +632,27 @@ const Workflows = (props) => {
 
 			if (responseJson !== undefined) {
 				setWorkflows(responseJson)
+
+				if (responseJson !== undefined) {
+					var actionnamelist = []
+					var parsedactionlist = []
+					for (var key in responseJson) {
+						for (var actionkey in responseJson[key].actions) {
+							const action = responseJson[key].actions[actionkey]
+							console.log("Action: ", action)
+							if (actionnamelist.includes(action.app_name)) {
+								continue
+							}
+
+							actionnamelist.push(action.app_name)
+							parsedactionlist.push(action)
+						}
+					}
+
+					console.log(parsedactionlist)
+					setActionImageList(parsedactionlist)
+				}
+
 				setFilteredWorkflows(responseJson)
 				setWorkflowDone(true)
 			} else {
@@ -689,6 +713,7 @@ const Workflows = (props) => {
 		display: "flex",
 		flexWrap: 'wrap',
 		alignContent: "space-between",
+		marginTop: 5, 
 	}
 
 	const paperAppStyle = {
@@ -845,6 +870,16 @@ const Workflows = (props) => {
 
 		if (sanitize === true) {
 			data = sanitizeWorkflow(data)	
+
+			if (data.subflows !== null && data.subflows !== undefined) {
+				alert.info("Not exporting with subflows when sanitizing. Please manually export them.")
+				data.subflows = []
+			}
+
+			//	for (var key in data.subflows) {
+			//		if (data.sublof
+			//	}
+			//}
 		}
 
 		// Add correct ID's for triggers
@@ -1065,14 +1100,45 @@ const Workflows = (props) => {
 					<FileCopyIcon style={{marginLeft: 0, marginRight: 8}}/>
 					{"Duplicate Workflow"}
 				</MenuItem>
-				<NestedMenuItem disabled={userdata.orgs === undefined || userdata.orgs === null || userdata.orgs.length === 1 || userdata.orgs.length >= 0} style={{backgroundColor: inputColor, color: "white"}} onClick={() => {
+				{/*<NestedMenuItem disabled={userdata.orgs === undefined || userdata.orgs === null || userdata.orgs.length === 1 || userdata.orgs.length >= 0} style={{backgroundColor: inputColor, color: "white"}} onClick={() => {
-
+					//copyWorkflow(data)		
+					//setOpen(false)
 				}} key={"duplicate"}>
 					<FileCopyIcon style={{marginLeft: 0, marginRight: 8}}/>
 					{"Copy to Child Org"}
-				</NestedMenuItem>
+				</NestedMenuItem>*/}
 				<MenuItem style={{backgroundColor: inputColor, color: "white"}} onClick={() => {
 					setExportModalOpen(true)
+
+					if (data.triggers !== null && data.triggers !== undefined) { 
+						var newSubflows = []
+						for (var key in data.triggers) {
+							const trigger = data.triggers[key]
+
+							if (trigger.parameters !== null && trigger.parameters !== undefined) {
+								for (var subkey in trigger.parameters) {
+									const param = trigger.parameters[subkey]
+									if (param.name === "workflow" && param.value !== data.id && !newSubflows.includes(param.value)) {
+										newSubflows.push(param.value)
+									}
+								}
+							}
+						}
+
+						var parsedworkflows = []
+						for (var key in newSubflows) {
+							const foundWorkflow = workflows.find(workflow => workflow.id === newSubflows[key])
+							if (foundWorkflow !== undefined && foundWorkflow !== null) {
+								parsedworkflows.push(foundWorkflow)
+							}
+						}
+						
+						if (parsedworkflows.length > 0) {
+							console.log("Appending subflows during export: ", parsedworkflows.length)
+							data.subflows = parsedworkflows
+						}
+					}
+
 					setExportData(data)
 					setOpen(false)
 				}} key={"export"}>
@@ -1389,7 +1455,7 @@ const Workflows = (props) => {
 			let workflowData = "";
 			if (workflows.length > 0) {
 				const columns = [
-					{ field: 'image', headerName: 'Logo', width: 42, renderCell: (params) => {
+					{ field: 'image', headerName: 'Logo', width: 50, sortable: false, renderCell: (params) => {
 						const data = params.row.record
 
 						var boxColor = "#FECC00"
@@ -1430,7 +1496,7 @@ const Workflows = (props) => {
 
 						return (
 							<Grid item>
-								<Link to={`/workflows/{data.id}`} style={{textDecoration: "none", color: "inherit",}}>
+								<Link to={"/workflows/"+data.id} style={{textDecoration: "none", color: "inherit",}}>
 									<Typography>
 										{data.name}
 									</Typography>
@@ -1539,7 +1605,7 @@ const Workflows = (props) => {
 					}
 				];
 				let rows = [];
-				rows = workflows.map((data, index) => {
+				rows = filteredWorkflows.map((data, index) => {
 					let obj = {
 						"id":index+1, 
 						"title":data.name, 
@@ -1547,7 +1613,7 @@ const Workflows = (props) => {
 					}
 
 					return obj;
-				});
+				})
 				workflowData = 
 					<DataGrid 
 						color="primary" 
@@ -1691,7 +1757,7 @@ const Workflows = (props) => {
 
 					}} color="primary">
 						{submitLoading ? 
-							<CircularProgress />
+							<CircularProgress color="secondary" />
 							:
 	        		"Submit"
 						}
@@ -1805,6 +1871,53 @@ const Workflows = (props) => {
 							<h2>Workflows</h2>
 						</div>
 					</div>
+					{/*
+					<div style={flexContainerStyle}>
+						<div style={{...flexBoxStyle, ...activeWorkflowStyle}}>
+							<div style={flexContentStyle}>
+								<div><img src={mobileImage} style={iconStyle} /></div>
+								<div style={ blockRightStyle }>
+									<div style={counterStyle}>{workflows.length}</div>
+									<div style={fontSize_16}>ACTIVE WORKFLOWS</div>
+								</div>
+							</div>
+						</div>
+						<div style={{...flexBoxStyle, ...availableWorkflowStyle}}>
+							<div style={flexContentStyle}>
+								<div><img src={bookImage} style={iconStyle} /></div>
+								<div style={ blockRightStyle }>
+									<div style={counterStyle}>{workflows.length}</div>
+									<div style={fontSize_16}>AVAILABE WORKFLOWS</div>
+								</div>
+							</div>
+						</div>
+						<div style={{...flexBoxStyle, ...notificationStyle}}>
+							<div style={flexContentStyle}>
+								<div><img src={bagImage} style={iconStyle} /></div>
+								<div style={ blockRightStyle }>
+									<div style={counterStyle}>{workflows.length}</div>
+									<div style={fontSize_16}>NOTIFICATIONS</div>
+								</div>
+							</div>
+						</div>
+					</div>
+					*/}
+
+					{/*
+					chipRenderer={({ value, isFocused, isDisabled, handleClick, handleRequestDelete }, key) => {
+						console.log("VALUE: ", value)
+
+						return (
+							<Chip
+								key={key}
+								style={chipStyle}
+
+							>
+								{value}
+							</Chip>
+						)
+					}}
+					*/}
 					<div style={{display: "flex", margin: "0px 0px 20px 0px"}}>
 						<div style={{flex: 1}}>
 							<Typography style={{marginTop: 7, marginBottom: "auto"}}>
@@ -1836,6 +1949,36 @@ const Workflows = (props) => {
 						</div>
 					</div>
 					<div style={{marginTop: 15,}} />
+					{actionImageList !== undefined && actionImageList !== null && actionImageList.length > 0 ? 
+						<div style={{display: "flex", maxWidth: 1024, zIndex: 11, border: "1px solid rgba(255,255,255,0.1)", borderRadius: theme.palette.borderRadius, textAlign: "center", overflow: "auto",}}>
+							{actionImageList.map((data, index) => {
+								if (data.large_image === undefined || data.large_image === null || data.large_image.length === 0) {
+									return null
+								}
+
+								if (data.app_name.toLowerCase() === "shuffle tools") {
+									data.large_image = theme.palette.defaultImage
+								}
+
+								return (
+									<span style={{zIndex: 10}}>
+										<IconButton style={{backgroundColor: "transparent", margin: 0, padding: 12, }}  onClick={() => {
+												console.log("FILTER: ", data)	
+												addFilter(data.app_name) 
+										}}>
+											<Tooltip title={`Filter by ${data.app_name}`} placement="top">
+												<Badge badgeContent={0} color="secondary" style={{fontSize: 10}}>
+													<div style={{height: imgSize, width: imgSize, position: "relative", filter: "brightness(0.6)", backgroundColor: "#000", borderRadius: imgSize/2, zIndex: 100, overflow: "hidden", display: "flex", justifyContent: "center", }}>
+														<img style={{height: imgSize, width: imgSize, position: "absolute", top: -2, left: -2, cursor: "pointer", zIndex: 99, border: "2px solid rgba(255,255,255,0.7)", }} alt={data.app_name} src={data.large_image}/>
+													</div>
+												</Badge>
+											</Tooltip> 
+										</IconButton>
+									</span>
+								)
+							})}
+						</div>
+					: null}
 					{view === "grid" ?
 						<Grid container spacing={4} style={paperAppContainer}>
 							<NewWorkflowPaper />

functions/onprem/orborus/Dockerfile

@@ -11,7 +11,7 @@ RUN go get github.com/docker/docker/api/types && \
     go get github.com/mackerelio/go-osstat/cpu && \
     go get github.com/mackerelio/go-osstat/memory && \
     go get github.com/satori/go.uuid && \
-    go get github.com/frikky/shuffle-shared
+    go get github.com/shuffle/shuffle-shared
 RUN go build
 RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o orborus .
 

functions/onprem/orborus/build.sh

@@ -1,5 +1,5 @@
 NAME=shuffle-orborus
-VERSION=0.9.23
+VERSION=0.9.30
 
 echo "Running docker build with $NAME:$VERSION"
 #docker rmi frikky/shuffle:$NAME --force

functions/onprem/orborus/go.mod

@@ -1,46 +1,12 @@
 module orborus
 
-go 1.13
+go 1.15
 
 require (
-	cloud.google.com/go/datastore v1.6.0 // indirect
-	cloud.google.com/go/storage v1.18.1 // indirect
-	github.com/Masterminds/semver v1.5.0 // indirect
-	github.com/Microsoft/go-winio v0.5.0 // indirect
-	github.com/algolia/algoliasearch-client-go/v3 v3.21.0 // indirect
-	github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 // indirect
 	github.com/containerd/containerd v1.5.7 // indirect
-	github.com/creack/pty v1.1.16 // indirect
+	github.com/docker/docker v20.10.10+incompatible
-	github.com/docker/distribution v2.7.1+incompatible // indirect
-	github.com/docker/docker v20.10.9+incompatible
 	github.com/docker/go-connections v0.4.0 // indirect
-	github.com/docker/go-units v0.4.0 // indirect
-	github.com/frikky/kin-openapi v0.40.0 // indirect
-	github.com/frikky/shuffle-shared v0.1.15
-	github.com/go-openapi/swag v0.19.15 // indirect
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/go-querystring v1.1.0 // indirect
-	github.com/gorilla/mux v1.8.0 // indirect
-	github.com/kr/pretty v0.3.0 // indirect
 	github.com/mackerelio/go-osstat v0.2.1
-	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
-	github.com/morikuni/aec v1.0.0 // indirect
-	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.0.1 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
-	github.com/rogpeppe/go-internal v1.8.0 // indirect
 	github.com/satori/go.uuid v1.2.0
-	go4.org v0.0.0-20201209231011-d4a079459e60 // indirect
+	github.com/shuffle/shuffle-shared v0.1.26
-	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
-	golang.org/x/mod v0.5.1 // indirect
-	golang.org/x/net v0.0.0-20211014172544-2b766c08f1c0 // indirect
-	golang.org/x/sys v0.0.0-20211013075003-97ac67df715c // indirect
-	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
-	golang.org/x/tools v0.1.7 // indirect
-	google.golang.org/genproto v0.0.0-20211013025323-ce878158c4d4 // indirect
-	google.golang.org/grpc v1.41.0 // indirect
-	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 )

functions/onprem/orborus/orborus.go

@@ -1,15 +1,18 @@
 package main
 
 /*
-	Orborus exists to listen for new workflow executions and deploy workers.
+	Orborus exists to listen for new workflow executions whcih are deployed as workers.
 */
 
+// frikky@debian:~/git/shuffle/functions/onprem/worker$ docker service create --replicas 5 --name shuffle-workers --env SHUFFLE_SWARM_CONFIG=run --publish published=33333,target=33333 ghcr.io/frikky/shuffle-worker:nightly
+
 import (
-	"github.com/frikky/shuffle-shared"
+	"github.com/shuffle/shuffle-shared"
 
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -23,6 +26,9 @@ import (
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/mount"
+	//"github.com/docker/docker/api/types/network"
+	"github.com/docker/docker/api/types/swarm"
 	//"github.com/docker/docker/api/types/filters"
 	dockerclient "github.com/docker/docker/client"
 	"github.com/satori/go.uuid"
@@ -57,6 +63,7 @@ var runningMode = strings.ToLower(os.Getenv("RUNNING_MODE"))
 var cleanupEnv = strings.ToLower(os.Getenv("CLEANUP"))
 var timezone = os.Getenv("TZ")
 var containerName = os.Getenv("ORBORUS_CONTAINER_NAME")
+var swarmConfig = os.Getenv("SHUFFLE_SWARM_CONFIG")
 var executionIds = []string{}
 
 var dockercli *dockerclient.Client
@@ -101,11 +108,13 @@ func getThisContainerId() {
 		out, err := exec.Command("bash", "-c", cmd).Output()
 		if err == nil {
 			containerId = strings.TrimSpace(string(out))
+			log.Printf("[DEBUG] Set containerId network to %s", containerId)
 
 			// cgroup error. Use fallback strategy below.
 			// https://github.com/moby/moby/issues/7015
 			//log.Printf("Checking if %s is in %s", ".scope", string(out))
 			if strings.Contains(string(out), ".scope") {
+				log.Printf("[DEBUG] ContainerId contains scope. setting to empty.")
 				containerId = ""
 				//docker-76c537e9a4b7c7233011f5d70e6b7f2d600b6413ac58a96519b8dca7a3f7117a.scope
 			}
@@ -127,9 +136,132 @@ func getThisContainerId() {
 	log.Printf(`[INFO] Started with containerId "%s"`, containerId)
 }
 
+func deployServiceWorkers(image string) {
+	log.Printf("[DEBUG] Validating deployment of workers as services IF swarmConfig = run (value: %#v)", swarmConfig)
+	if swarmConfig == "run" {
+		// frikky@debian:~/git/shuffle/functions/onprem/worker$ docker service create --replicas 5 --name shuffle-workers --env SHUFFLE_SWARM_CONFIG=run --publish published=33333,target=33333 ghcr.io/frikky/shuffle-worker:nightly
+		networkName := "shuffle-executions"
+		ctx := context.Background()
+
+		//docker network create --driver=overlay workers
+		networkCreateOptions := types.NetworkCreate{
+			Driver: "overlay",
+		}
+		_, err := dockercli.NetworkCreate(
+			ctx,
+			networkName,
+			networkCreateOptions,
+		)
+
+		if err != nil {
+			if strings.Contains(fmt.Sprintf("%s", err), "already exists") {
+
+			} else {
+				log.Printf("[DEBUG] Failed to create network %s for workers: %s. This is not critical, and containers will still be added", networkName, err)
+			}
+		}
+		//serviceOptions := types.ServiceCreateOptions{}
+		//service, err := dockercli.ServiceCreate(
+		//	context.Background(),
+		//	serviceSpec,
+		//	serviceOptions,
+		//)
+
+		//containerName := fmt.Sprintf("shuffle-worker-%s", parsedUuid)
+
+		replicas := uint64(2)
+		scaleReplicas := os.Getenv("SHUFFLE_SCALE_REPLICAS")
+		if len(scaleReplicas) > 0 {
+			log.Printf("[DEBUG] SHUFFLE_SCALE_REPLICAS set to value %#v. Trying to overwrite default (2/node)", scaleReplicas)
+			tmpInt, err := strconv.Atoi(scaleReplicas)
+			if err != nil {
+				log.Printf("[ERROR] %s is not a valid number for replication", scaleReplicas)
+			} else {
+				replicas = uint64(tmpInt)
+			}
+		}
+
+		innerContainerName := fmt.Sprintf("shuffle-workers")
+		log.Printf("[DEBUG] Deploying %d containers for worker with swarm to each node. Service name: %s. Image: %s", replicas, innerContainerName, image)
+
+		serviceSpec := swarm.ServiceSpec{
+			Annotations: swarm.Annotations{
+				Name:   innerContainerName,
+				Labels: map[string]string{},
+			},
+			Networks: []swarm.NetworkAttachmentConfig{
+				swarm.NetworkAttachmentConfig{
+					Target: networkName,
+				},
+			},
+			EndpointSpec: &swarm.EndpointSpec{
+				Ports: []swarm.PortConfig{
+					swarm.PortConfig{
+						Protocol:      swarm.PortConfigProtocolTCP,
+						PublishMode:   swarm.PortConfigPublishModeIngress,
+						Name:          "worker-port",
+						PublishedPort: 33333,
+						TargetPort:    33333,
+					},
+				},
+			},
+			TaskTemplate: swarm.TaskSpec{
+				Resources: &swarm.ResourceRequirements{
+					Reservations: &swarm.Resources{},
+				},
+				ContainerSpec: &swarm.ContainerSpec{
+					Image: image,
+					Env: []string{
+						fmt.Sprintf("SHUFFLE_SWARM_CONFIG=%s", os.Getenv("SHUFFLE_SWARM_CONFIG")),
+					},
+					Mounts: []mount.Mount{
+						mount.Mount{
+							Source: "/var/run/docker.sock",
+							Target: "/var/run/docker.sock",
+							Type:   mount.TypeBind,
+						},
+					},
+				},
+				RestartPolicy: &swarm.RestartPolicy{
+					Condition: swarm.RestartPolicyConditionNone,
+				},
+				Placement: &swarm.Placement{
+					MaxReplicas: replicas,
+				},
+			},
+		}
+
+		if dockerApiVersion != "" {
+			serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("DOCKER_API_VERSION=%s", dockerApiVersion))
+		}
+
+		if len(os.Getenv("SHUFFLE_SCALE_REPLICAS")) > 0 {
+			serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("DOCKER_API_VERSION=%s", os.Getenv("SHUFFLE_SCALE_REPLICAS")))
+		}
+
+		serviceOptions := types.ServiceCreateOptions{}
+		_, err = dockercli.ServiceCreate(
+			ctx,
+			serviceSpec,
+			serviceOptions,
+		)
+
+		if err == nil {
+			log.Printf("[DEBUG] Successfully deployed workers with %d replicas", replicas)
+			//time.Sleep(time.Duration(10) * time.Second)
+			//log.Printf("[DEBUG] Servicecreate request: %#v %#v", service, err)
+		} else {
+			if !strings.Contains(fmt.Sprintf("%s", err), "Already Exists") && !strings.Contains(fmt.Sprintf("%s", err), "is already in use by service") {
+				log.Printf("[ERROR] Failed making service: %s", err)
+			}
+		}
+
+	}
+}
+
 // Deploys the internal worker whenever something happens
 // https://docs.docker.com/engine/api/sdk/examples/
-func deployWorker(image string, identifier string, env []string) {
+func deployWorker(image string, identifier string, env []string, executionRequest shuffle.ExecutionRequest) error {
 	// Binds is the actual "-v" volume.
 	// Max 20% CPU every second
 
@@ -145,9 +277,10 @@ func deployWorker(image string, identifier string, env []string) {
 		Binds: []string{
 			"/var/run/docker.sock:/var/run/docker.sock:rw",
 		},
-		NetworkMode: container.NetworkMode(fmt.Sprintf("container:%s", containerId)),
 	}
 
+	hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId))
+
 	if cleanupEnv == "true" {
 		hostConfig.AutoRemove = true
 	}
@@ -157,6 +290,27 @@ func deployWorker(image string, identifier string, env []string) {
 		Env:   env,
 	}
 
+	//var swarmConfig = os.Getenv("SHUFFLE_SWARM_CONFIG")
+	parsedUuid := uuid.NewV4()
+	if swarmConfig == "run" {
+		go func() {
+			err := sendWorkerRequest(executionRequest)
+			if err != nil {
+				log.Printf("[ERROR] Failed worker request for %s: %s", executionRequest.ExecutionId, err)
+
+				if strings.Contains(fmt.Sprintf("%s", err), "connection refused") || strings.Contains(fmt.Sprintf("%s", err), "EOF") {
+					workerImage := fmt.Sprintf("%s/%s/shuffle-worker:%s", baseimageregistry, baseimagename, workerVersion)
+					deployServiceWorkers(workerImage)
+				}
+				//return err
+			} else {
+				log.Printf("[DEBUG] Started worker from request with name: %s", executionRequest.ExecutionId)
+			}
+		}()
+
+		return nil
+	}
+
 	//log.Printf("[INFO] Identifier: %s", identifier)
 	cont, err := dockercli.ContainerCreate(
 		context.Background(),
@@ -169,8 +323,7 @@ func deployWorker(image string, identifier string, env []string) {
 
 	if err != nil {
 		if strings.Contains(fmt.Sprintf("%s", err), "Conflict. The container name ") {
-			uuid := uuid.NewV4()
+			identifier = fmt.Sprintf("%s-%s", identifier, parsedUuid)
-			identifier = fmt.Sprintf("%s-%s", identifier, uuid)
 			log.Printf("[INFO] 2 - Identifier: %s", identifier)
 			cont, err = dockercli.ContainerCreate(
 				context.Background(),
@@ -183,19 +336,40 @@ func deployWorker(image string, identifier string, env []string) {
 
 			if err != nil {
 				log.Printf("[ERROR] Container create error(2): %s", err)
-				return
+				return err
 			}
 		} else {
 			log.Printf("[ERROR] Container create error: %s", err)
-			return
+			return err
 		}
 	}
 
 	containerStartOptions := types.ContainerStartOptions{}
 	err = dockercli.ContainerStart(context.Background(), cont.ID, containerStartOptions)
 	if err != nil {
-		log.Printf("[ERROR] Failed to start container in environment %s: %s", environment, err)
+		log.Printf("[DEBUG] Failed initial container start. Running WITHOUT custom network. Err: %s", err)
-		return
+		// Trying to recreate and start WITHOUT network if it's possible. No extended checks. Old execution system (<0.9.30)
+		if strings.Contains(fmt.Sprintf("%s", err), "cannot join network") || strings.Contains(fmt.Sprintf("%s", err), "No such container") {
+			hostConfig.NetworkMode = ""
+			//container.NetworkMode(fmt.Sprintf("container:%s", containerId))
+			cont, err = dockercli.ContainerCreate(
+				context.Background(),
+				config,
+				hostConfig,
+				nil,
+				nil,
+				identifier+"-2",
+			)
+
+			err = dockercli.ContainerStart(context.Background(), cont.ID, containerStartOptions)
+		}
+
+		if err != nil {
+			log.Printf("[ERROR] Failed to start container in environment %s: %s", environment, err)
+			return err
+		} else {
+			log.Printf("[INFO] Container %s was created under environment %s", cont.ID, environment)
+		}
 
 		//stats, err := cli.ContainerInspect(context.Background(), containerName)
 		//if err != nil {
@@ -212,7 +386,7 @@ func deployWorker(image string, identifier string, env []string) {
 		//		return
 		//	}
 
-		//	err = deployWorker(cli, workerImage, containerName, env)
+		//	err = deployWorke(cli, workerImage, containerName, env)
 		//	if err != nil {
 		//		log.Printf("Failed executing worker %s in state %s", execution.ExecutionId, containerStatus)
 		//		return
@@ -222,7 +396,7 @@ func deployWorker(image string, identifier string, env []string) {
 		log.Printf("[INFO] Container %s was created under environment %s", cont.ID, environment)
 	}
 
-	return
+	return nil
 }
 
 func stopWorker(containername string) error {
@@ -255,6 +429,7 @@ func initializeImages() {
 		appSdkVersion = "0.8.97"
 		log.Printf("[WARNING] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %s", appSdkVersion)
 	}
+
 	if workerVersion == "" {
 		workerVersion = "nightly"
 		log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %s", workerVersion)
@@ -263,14 +438,16 @@ func initializeImages() {
 	if baseimageregistry == "" {
 		baseimageregistry = "docker.io"
 		baseimageregistry = "ghcr.io"
-		log.Printf("Setting baseimageregistry")
+		log.Printf("[DEBUG] Setting baseimageregistry")
 	}
 	if baseimagename == "" {
 		baseimagename = "frikky/shuffle"
 		baseimagename = "frikky"
-		log.Printf("Setting baseimagename")
+		log.Printf("[DEBUG] Setting baseimagename")
 	}
 
+	log.Printf("[DEBUG] Setting swarm config to %#v. Default is empty.", swarmConfig)
+
 	// check whether they are the same first
 	images := []string{
 		fmt.Sprintf("frikky/shuffle:app_sdk"),
@@ -403,6 +580,8 @@ func main() {
 	//workerImage := fmt.Sprintf("%s/%s:worker%s", baseimageregistry, baseimagename, baseimagetagsuffix)
 	workerImage := fmt.Sprintf("%s/%s/shuffle-worker:%s", baseimageregistry, baseimagename, workerVersion)
 
+	go deployServiceWorkers(workerImage)
+
 	log.Printf("[INFO] Finished configuring docker environment")
 
 	// FIXME - time limit
@@ -555,7 +734,6 @@ func main() {
 				continue
 			} else {
 				//log.Printf("[INFO] Adding to be ran %s", execution.ExecutionId)
-				executionIds = append(executionIds, execution.ExecutionId)
 			}
 
 			// Now, how do I execute this one?
@@ -569,6 +747,7 @@ func main() {
 				fmt.Sprintf("CLEANUP=%s", cleanupEnv),
 				fmt.Sprintf("TZ=%s", timezone),
 				fmt.Sprintf("SHUFFLE_PASS_APP_PROXY=%s", os.Getenv("SHUFFLE_PASS_APP_PROXY")),
+				fmt.Sprintf("SHUFFLE_SWARM_CONFIG=%s", os.Getenv("SHUFFLE_SWARM_CONFIG")),
 			}
 
 			//log.Printf("Running worker with proxy? %s", os.Getenv("SHUFFLE_PASS_WORKER_PROXY"))
@@ -581,11 +760,15 @@ func main() {
 				env = append(env, fmt.Sprintf("DOCKER_API_VERSION=%s", dockerApiVersion))
 			}
 
-			go deployWorker(workerImage, containerName, env)
+			err = deployWorker(workerImage, containerName, env, execution)
-
-			log.Printf("[INFO] ExecutionID %s was deployed and to be removed from queue.", execution.ExecutionId)
 			zombiecounter += 1
-			toBeRemoved.Data = append(toBeRemoved.Data, execution)
+			if err == nil {
+				log.Printf("[INFO] ExecutionID %s was deployed and to be removed from queue.", execution.ExecutionId)
+				toBeRemoved.Data = append(toBeRemoved.Data, execution)
+				executionIds = append(executionIds, execution.ExecutionId)
+			} else {
+				log.Printf("[WARNING] Execution ID %s failed to deploy: %s", execution.ExecutionId, err)
+			}
 		}
 
 		// Removes handled workflows (worker is made)
@@ -647,6 +830,7 @@ func main() {
 
 // Is this ok to do with Docker? idk :)
 func getRunningWorkers(ctx context.Context, workerTimeout int) int {
+	//log.Printf("[DEBUG] Getting running workers with API version %s", dockerApiVersion)
 	containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{
 		All: true,
 	})
@@ -657,8 +841,8 @@ func getRunningWorkers(ctx context.Context, workerTimeout int) int {
 
 		newVersionSplit := strings.Split(fmt.Sprintf("%s", err), "version is")
 		if len(newVersionSplit) > 1 {
-			dockerApiVersion = strings.TrimSpace(newVersionSplit[1])
+			//dockerApiVersion = strings.TrimSpace(newVersionSplit[1])
-			log.Printf("[INFO] Changed the API version to default to %s", dockerApiVersion)
+			log.Printf("[DEBUG] WANT to change the API version to default to %s?", strings.TrimSpace(newVersionSplit[1]))
 		}
 
 		return maxConcurrency
@@ -706,6 +890,11 @@ func getRunningWorkers(ctx context.Context, workerTimeout int) int {
 // Should it check what happened to the execution? idk
 func zombiecheck(ctx context.Context, workerTimeout int) error {
 	executionIds = []string{}
+	if swarmConfig == "run" {
+		//log.Printf("[DEBUG] Skipping Zombie check due to new execution model (swarm)")
+		return nil
+	}
+
 	log.Println("[INFO] Looking for old containers (zombies)")
 	containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{
 		All: true,
@@ -791,3 +980,67 @@ func zombiecheck(ctx context.Context, workerTimeout int) error {
 
 	return nil
 }
+
+func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest) error {
+	parsedRequest := shuffle.OrborusExecutionRequest{
+		ExecutionId:           workflowExecution.ExecutionId,
+		Authorization:         workflowExecution.Authorization,
+		BaseUrl:               os.Getenv("BASE_URL"),
+		EnvironmentName:       os.Getenv("ENVIRONMENT_NAME"),
+		Timezone:              os.Getenv("TZ"),
+		Cleanup:               os.Getenv("CLEANUP"),
+		HTTPProxy:             os.Getenv("HTTP_PROXY"),
+		HTTPSProxy:            os.Getenv("HTTPS_PROXY"),
+		ShufflePassProxyToApp: os.Getenv("SHUFFLE_PASS_APP_PROXY"),
+	}
+
+	parsedBaseurl := baseUrl
+	if strings.Contains(baseUrl, ":") {
+		baseUrlSplit := strings.Split(baseUrl, ":")
+		if len(baseUrlSplit) >= 3 {
+			parsedBaseurl = strings.Join(baseUrlSplit[0:2], ":")
+			//parsedRequest.BaseUrl = fmt.Sprintf("%s:33333", parsedBaseurl)
+		}
+	}
+
+	data, err := json.Marshal(parsedRequest)
+	if err != nil {
+		log.Printf("[ERROR] Failed marshalling worker request: %s", err)
+		return err
+	}
+
+	//log.Printf("[DEBUG] Data: %s", string(data))
+
+	streamUrl := fmt.Sprintf("%s:33333/api/v1/execute", parsedBaseurl)
+	req, err := http.NewRequest(
+		"POST",
+		streamUrl,
+		bytes.NewBuffer([]byte(data)),
+	)
+
+	client := &http.Client{}
+	if err != nil {
+		log.Printf("[ERROR] Failed creating worker request: %s", err)
+		return err
+	}
+
+	newresp, err := client.Do(req)
+	if err != nil {
+		log.Printf("[ERROR] Error running worker request: %s", err)
+		return err
+	}
+
+	if newresp.StatusCode != 200 {
+		log.Printf("[ERROR] Error running request - status code is %d, not 200", newresp.StatusCode)
+		return errors.New(fmt.Sprintf("Bad statuscode: %d - expecting 200", newresp.StatusCode))
+	}
+
+	body, err := ioutil.ReadAll(newresp.Body)
+	if err != nil {
+		log.Printf("[ERROR] Failed reading body in worker request: %s", err)
+		return err
+	}
+
+	log.Printf("[DEBUG] NEWRESP (from worker request %s): %s (Status: %d)", workflowExecution.ExecutionId, string(body), newresp.StatusCode)
+	return nil
+}

functions/onprem/worker/Dockerfile

@@ -6,7 +6,8 @@ WORKDIR /app
 #RUN go env -w GO111MODULE=auto 
 COPY worker.go /app/worker.go
 COPY go.mod /app/go.mod
-COPY go.sum /app/go.sum
+#RUN go
+#COPY go.sum /app/go.sum
 RUN go get
 
 #RUN go mod init worker

functions/onprem/worker/build.sh

@@ -1,5 +1,5 @@
 NAME=shuffle-worker
-VERSION=0.9.23
+VERSION=0.9.30
 
 echo "Running docker build with $NAME:$VERSION"
 #CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin .

functions/onprem/worker/go.mod

@@ -2,48 +2,14 @@ module worker
 
 go 1.15
 	
+//replace github.com/shuffle/shuffle-shared => ../../../../../git/shuffle-shared
+
 require (
-	cloud.google.com/go/datastore v1.6.0 // indirect
-	cloud.google.com/go/storage v1.18.1 // indirect
-	github.com/Masterminds/semver v1.5.0 // indirect
-	github.com/Microsoft/go-winio v0.5.0 // indirect
-	github.com/algolia/algoliasearch-client-go/v3 v3.21.0 // indirect
-	github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 // indirect
 	github.com/containerd/containerd v1.5.7 // indirect
-	github.com/creack/pty v1.1.16 // indirect
-	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/docker v20.10.9+incompatible
 	github.com/docker/go-connections v0.4.0 // indirect
-	github.com/docker/go-units v0.4.0 // indirect
-	github.com/frikky/kin-openapi v0.40.0 // indirect
-	github.com/frikky/shuffle-shared v0.1.15
-	github.com/fsouza/go-dockerclient v1.7.2
-	github.com/go-git/go-billy/v5 v5.3.1 // indirect
-	github.com/go-git/go-git/v5 v5.4.2 // indirect
-	github.com/go-openapi/swag v0.19.15 // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/go-github/v28 v28.1.1 // indirect
-	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/gorilla/mux v1.8.0
-	github.com/kr/pretty v0.3.0 // indirect
-	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
-	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.0.1 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible
-	github.com/pkg/errors v0.9.1 // indirect
+	github.com/shuffle/shuffle-shared v0.1.27
-	github.com/rogpeppe/go-internal v1.8.0 // indirect
 	go4.org v0.0.0-20201209231011-d4a079459e60 // indirect
-	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
-	golang.org/x/mod v0.5.1 // indirect
-	golang.org/x/net v0.0.0-20211014172544-2b766c08f1c0 // indirect
-	golang.org/x/sys v0.0.0-20211013075003-97ac67df715c // indirect
-	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
-	golang.org/x/tools v0.1.7 // indirect
-	google.golang.org/genproto v0.0.0-20211013025323-ce878158c4d4 // indirect
-	google.golang.org/grpc v1.41.0 // indirect
-	gopkg.in/src-d/go-git.v4 v4.13.1 // indirect
 )

functions/onprem/worker/run.sh

@@ -0,0 +1,2 @@
+#curl -XPOST "http://192.168.86.37:33335/api/v1/run" -H "Content-Type: application/json" -d '{"execution_id":"ec35241a-b713-45c5-bda2-823fa4ce21e6","authorization":"d044f927-ad19-4f0e-a01f-51365e46c4e9","http_proxy":"","https_proxy":"","base_url":"http:\/\/192.168.86.37:5001","url":"http:\/\/192.168.86.37:33333","environment_name":"Shuffle","timezone":"","cleanup":"false","shuffle_pass_proxy_to_app":"","action":{"app_name":"Shuffle Tools","app_version":"1.1.0","app_id":"080b539e-638e-4a2b-9202-07d6360c2ddf","errors":[],"id":"13d73f9c-82be-4104-8b1e-8651ef189412","is_valid":true,"isStartNode":true,"sharing":true,"label":"Shuffle Tools_1","public":true,"generated":false,"environment":"Shuffle","name":"repeat_back_to_me","parameters":[{"description":"The message to repeat","id":"572cfe8c-eddf-4b66-8e53-99aeecfbd838","name":"call","example":"REPEATING: Hello world","value":"hello world","multiline":true,"options":null,"action_field":"","variant":"STATIC_VALUE","required":true,"configuration":false,"tags":null,"schema":{"type":"string"},"skip_multicheck":false,"value_replace":null,"unique_toggled":false}],"execution_variable":{"description":"","id":"","name":"","value":""},"position":{"x":367.5,"y":398.25},"authentication_id":"","category":"Testing","reference_url":"","sub_action":false,"source_workflow":""},"workflow_execution":{"type":"workflow","status":"EXECUTING","start":"13d73f9c-82be-4104-8b1e-8651ef189412","execution_argument":"","execution_id":"ec35241a-b713-45c5-bda2-823fa4ce21e6","execution_org":"015c13d4-2e03-45fd-82a9-9b1f8095937a","started_at":1635630830,"completed_at":0,"workflow_id":"ca6d239b-6523-4b11-8c92-00ea4ab82b6d","last_node":"","authorization":"d044f927-ad19-4f0e-a01f-51365e46c4e9","result":"","project_id":"","locations":null,"workflow":{"actions":[{"app_name":"Shuffle Tools","app_version":"1.1.0","app_id":"080b539e-638e-4a2b-9202-07d6360c2ddf","errors":[],"id":"13d73f9c-82be-4104-8b1e-8651ef189412","is_valid":true,"isStartNode":true,"sharing":true,"label":"Shuffle Tools_1","public":true,"generated":false,"environm
+curl -XPOST http://192.168.86.37:33335/api/v1/run -H "Content-Type: application/json" -d '{"execution_id":"ec35241a-b713-45c5-bda2-823fa4ce21e6","authorization":"d044f927-ad19-4f0e-a01f-51365e46c4e9","http_proxy":"","https_proxy":"","base_url":"http:\/\/192.168.86.37:5001","url":"http:\/\/192.168.86.37:33333","environment_name":"Shuffle","timezone":"","cleanup":"false","shuffle_pass_proxy_to_app":"","action":{"app_name":"Shuffle Tools","app_version":"1.1.0","app_id":"080b539e-638e-4a2b-9202-07d6360c2ddf","errors":[],"id":"13d73f9c-82be-4104-8b1e-8651ef189412","is_valid":true,"isStartNode":true,"sharing":true,"label":"Shuffle Tools_1","public":true,"generated":false,"environment":"Shuffle","name":"repeat_back_to_me","parameters":[{"description":"The message to repeat","id":"572cfe8c-eddf-4b66-8e53-99aeecfbd838","name":"call","example":"REPEATING: Hello world","value":"hello world","multiline":true,"options":null,"action_field":"","variant":"STATIC_VALUE","required":true,"configuration":false,"tags":null,"schema":{"type":"string"},"skip_multicheck":false,"value_replace":null,"unique_toggled":false}],"execution_variable":{"description":"","id":"","name":"","value":""},"position":{"x":367.5,"y":398.25},"authentication_id":"","category":"Testing","reference_url":"","sub_action":false,"source_workflow":""},"workflow_execution":{"type":"workflow","status":"EXECUTING","start":"13d73f9c-82be-4104-8b1e-8651ef189412","execution_argument":"","execution_id":"ec35241a-b713-45c5-bda2-823fa4ce21e6","execution_org":"015c13d4-2e03-45fd-82a9-9b1f8095937a","started_at":1635630830,"completed_at":0,"workflow_id":"ca6d239b-6523-4b11-8c92-00ea4ab82b6d","last_node":"","authorization":"d044f927-ad19-4f0e-a01f-51365e46c4e9","result":"","project_id":"","locations":null,"workflow":{"actions":[{"app_name":"Shuffle Tools","app_version":"1.1.0","app_id":"080b539e-638e-4a2b-9202-07d6360c2ddf","errors":[],"id":"13d73f9c-82be-4104-8b1e-8651ef189412","is_valid":true,"isStartNode":true,"sharing":true,"label":"Shuffle Tools_1","public":true,"generated":false,"environment":"Shuffle","name":"repeat_back_to_me","parameters":[{"description":"The message to repeat","id":"572cfe8c-eddf-4b66-8e53-99aeecfbd838","name":"call","example":"REPEATING: Hello world","value":"hello world","multiline":true,"options":null,"action_field":"","variant":"STATIC_VALUE","required":true,"configuration":false,"tags":null,"schema":{"type":"string"},"skip_multicheck":false,"value_replace":null,"unique_toggled":false}],"execution_variable":{"description":"","id":"","name":"","value":""},"position":{"x":367.5,"y":398.25},"authentication_id":"","category":"Testing","reference_url":"","sub_action":false,"source_workflow":""}],"branches":[],"visual_branches":[],"triggers":[],"schedules":[],"configuration":{"exit_on_error":false,"start_from_top":false,"skip_notifications":false},"created":1635611089,"edited":1635611106,"last_runtime":0,"id":"ca6d239b-6523-4b11-8c92-00ea4ab82b6d","is_valid":true,"name":"Tools testing","description":"","start":"13d73f9c-82be-4104-8b1e-8651ef189412","owner":"2b3a37f3-fdd2-4965-b78d-f5d2fe9a2572","sharing":"private","org":[{"name":"","id":"015c13d4-2e03-45fd-82a9-9b1f8095937a","users":null,"role":"","creator_org":"","image":""}],"execution_org":{"name":"","id":"015c13d4-2e03-45fd-82a9-9b1f8095937a","users":null,"role":"","creator_org":"","image":""},"org_id":"015c13d4-2e03-45fd-82a9-9b1f8095937a","workflow_variables":null,"execution_environment":"","previously_saved":true,"categories":{"siem":{"name":"","description":"","count":0},"communication":{"name":"","description":"","count":0},"assets":{"name":"","description":"","count":0},"cases":{"name":"","description":"","count":0},"network":{"name":"","description":"","count":0},"intel":{"name":"","description":"","count":1},"edr":{"name":"","description":"","count":0},"other":{"name":"","description":"","count":0}},"example_argument":"","public":false,"default_return_value":"","contact_info":{"name":"","url":""},"published_id":""},"results":[],"org_id":"","sub_execution_count":0,"execution_source":"default","execution_parent":"","execution_source_node":"","execution_source_auth":""}}'