Merge pull request #1153 from Shuffle/master

Merging changes from 1.2.0 for cleanup

.env

@@ -2,6 +2,9 @@
 ORG_ID=Shuffle
 ENVIRONMENT_NAME=Shuffle
 
+# Sanitize liquid.py input
+LIQUID_SANITIZE_INPUT=true
+
 
 # Remote github config for first load
 SHUFFLE_DOWNLOAD_WORKFLOW_LOCATION=
@@ -55,14 +58,28 @@ SHUFFLE_ORBORUS_STARTUP_DELAY= 		# Used for setting up a startup delay for Orbor
 
 SHUFFLE_BASE_IMAGE_NAME=shuffle
 SHUFFLE_BASE_IMAGE_REGISTRY=ghcr.io
-SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-1.0.0"
+SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-1.1.0"
+
+## shuffle_memcached 			(for distributed caching)
+## shuffle_SWARM_CONFIG 	(run vs not run)
+## shuffle_Scale_Replicas (workers/node)
+## shuffle_App_Replicas 	(apps/node)
+
+SHUFFLE_SWARM_BRIDGE_DEFAULT_MTU=1500 # 1500 by default
+# The eth0 interface inside a container corresponds 
+# to the virtual Ethernet interface that connects 
+# the container to the docker0 
+SHUFFLE_SWARM_BRIDGE_DEFAULT_INTERFACE=eth0 
 
 # Used for auto-cleanup of containers. REALLY important at scale.
 SHUFFLE_CONTAINER_AUTO_CLEANUP=false
 SHUFFLE_ELASTIC=true
 SHUFFLE_LOGS_DISABLED=false
-SHUFFLE_CHAT_DISABLED=false # Controls support chat
+SHUFFLE_CHAT_DISABLED=false 	# Controls support chat
 SHUFFLE_RERUN_SCHEDULE=300
+SHUFFLE_DISABLE_RERUN_AND_ABORT=false
+SHUFFLE_WORKER_SERVER_URL= 		# Definition in case Worker & Orborus is talking to the wrong server
+SHUFFLE_ORBORUS_PULL_TIME= 		# Definition in case Orborus is pulling too often/not often enough
 
 # DATABASE CONFIGURATIONS
 DATASTORE_EMULATOR_HOST=shuffle-database:8000

.github/install-guide.md

@@ -91,7 +91,7 @@ npm start
 ## Backend - Golang
 http://localhost:5001 - REST API - requires [>=go1.13](https://golang.org/dl/)
 ```bash
-export SHUFFLE_OPENSEARCH_URL="http://localhost:9200"
+export SHUFFLE_OPENSEARCH_URL="https://localhost:9200"
 export SHUFFLE_ELASTIC=true
 export SHUFFLE_OPENSEARCH_USERNAME=admin
 export SHUFFLE_OPENSEARCH_PASSWORD=admin

.github/push_nightly.sh

@@ -2,40 +2,79 @@
 # Done manually for now since GHCR isn't being pushed to easily with the current Github action CI. Nightly = Latest IF we run hotfixes on latest
 
 ### Pull latest from ghcr CI/CD
-#docker pull ghcr.io/shuffle/shuffle-app_sdk:nightly
+docker pull ghcr.io/shuffle/shuffle-app_sdk:nightly
-#docker pull ghcr.io/shuffle/shuffle-worker:nightly
+docker pull ghcr.io/shuffle/shuffle-worker:nightly
-#docker pull ghcr.io/shuffle/shuffle-orborus:nightly
+docker pull ghcr.io/shuffle/shuffle-orborus:nightly
-#docker pull ghcr.io/shuffle/shuffle-frontend:nightly
+docker pull ghcr.io/shuffle/shuffle-frontend:nightly
 #docker pull ghcr.io/shuffle/shuffle-backend:nightly
+#
+### NIGHTLY releases
+docker tag ghcr.io/shuffle/shuffle-app_sdk:nightly ghcr.io/frikky/shuffle-app_sdk:nightly
+docker tag ghcr.io/shuffle/shuffle-worker:nightly  ghcr.io/frikky/shuffle-worker:nightly
+docker tag ghcr.io/shuffle/shuffle-orborus:nightly ghcr.io/frikky/shuffle-orborus:nightly
+docker tag ghcr.io/shuffle/shuffle-frontend:nightly ghcr.io/frikky/shuffle-frontend:nightly
+docker tag ghcr.io/shuffle/shuffle-backend:nightly ghcr.io/frikky/shuffle-backend:nightly
+
+docker push ghcr.io/frikky/shuffle-app_sdk:nightly
+docker push ghcr.io/frikky/shuffle-worker:nightly
+docker push ghcr.io/frikky/shuffle-orborus:nightly
+docker push ghcr.io/frikky/shuffle-frontend:nightly
+docker push ghcr.io/frikky/shuffle-backend:nightly
 
 ### LATEST releases:
-#docker tag ghcr.io/shuffle/shuffle-app_sdk:nightly ghcr.io/shuffle/shuffle-app_sdk:latest
+## shuffle/shuffle 
-#docker tag ghcr.io/shuffle/shuffle-worker:nightly  ghcr.io/shuffle/shuffle-worker:latest
+docker tag ghcr.io/shuffle/shuffle-app_sdk:nightly ghcr.io/shuffle/shuffle-app_sdk:latest
-#docker tag ghcr.io/shuffle/shuffle-orborus:nightly ghcr.io/shuffle/shuffle-orborus:latest
+docker tag ghcr.io/shuffle/shuffle-worker:nightly  ghcr.io/shuffle/shuffle-worker:latest
-#docker tag ghcr.io/shuffle/shuffle-frontend:nightly ghcr.io/shuffle/shuffle-frontend:latest
+docker tag ghcr.io/shuffle/shuffle-orborus:nightly ghcr.io/shuffle/shuffle-orborus:latest
-#docker tag ghcr.io/shuffle/shuffle-backend:nightly ghcr.io/shuffle/shuffle-backend:latest
+docker tag ghcr.io/shuffle/shuffle-frontend:nightly ghcr.io/shuffle/shuffle-frontend:latest
-#
+docker tag ghcr.io/shuffle/shuffle-backend:nightly ghcr.io/shuffle/shuffle-backend:latest
-#docker push ghcr.io/shuffle/shuffle-app_sdk:latest
+
-#docker push ghcr.io/shuffle/shuffle-worker:latest
+docker push ghcr.io/shuffle/shuffle-app_sdk:latest
-#docker push ghcr.io/shuffle/shuffle-orborus:latest
+docker push ghcr.io/shuffle/shuffle-worker:latest
-#docker push ghcr.io/shuffle/shuffle-frontend:latest
+docker push ghcr.io/shuffle/shuffle-orborus:latest
-#docker push ghcr.io/shuffle/shuffle-backend:latest
+docker push ghcr.io/shuffle/shuffle-frontend:latest
+docker push ghcr.io/shuffle/shuffle-backend:latest
+
+## frikky/shuffle
+docker tag ghcr.io/shuffle/shuffle-app_sdk:nightly ghcr.io/frikky/shuffle-app_sdk:latest
+docker tag ghcr.io/shuffle/shuffle-worker:nightly  ghcr.io/frikky/shuffle-worker:latest
+docker tag ghcr.io/shuffle/shuffle-orborus:nightly ghcr.io/frikky/shuffle-orborus:latest
+docker tag ghcr.io/shuffle/shuffle-frontend:nightly ghcr.io/frikky/shuffle-frontend:latest
+docker tag ghcr.io/shuffle/shuffle-backend:nightly ghcr.io/frikky/shuffle-backend:latest
+
+docker push ghcr.io/frikky/shuffle-app_sdk:latest
+docker push ghcr.io/frikky/shuffle-worker:latest
+docker push ghcr.io/frikky/shuffle-orborus:latest
+docker push ghcr.io/frikky/shuffle-frontend:latest
+docker push ghcr.io/frikky/shuffle-backend:latest
 
 
 ### 1.1.0 releases:
-#docker tag ghcr.io/shuffle/shuffle-app_sdk:nightly ghcr.io/shuffle/shuffle-app_sdk:1.1.0
+## shuffle/shuffle
-#docker tag ghcr.io/shuffle/shuffle-worker:nightly  ghcr.io/shuffle/shuffle-worker:1.1.0
+docker tag ghcr.io/shuffle/shuffle-app_sdk:nightly ghcr.io/shuffle/shuffle-app_sdk:1.1.0
-#docker tag ghcr.io/shuffle/shuffle-orborus:nightly ghcr.io/shuffle/shuffle-orborus:1.1.0
+docker tag ghcr.io/shuffle/shuffle-worker:nightly  ghcr.io/shuffle/shuffle-worker:1.1.0
-#docker tag ghcr.io/shuffle/shuffle-frontend:nightly ghcr.io/shuffle/shuffle-frontend:1.1.0
+docker tag ghcr.io/shuffle/shuffle-orborus:nightly ghcr.io/shuffle/shuffle-orborus:1.1.0
-#docker tag ghcr.io/shuffle/shuffle-backend:nightly ghcr.io/shuffle/shuffle-backend:1.1.0
+docker tag ghcr.io/shuffle/shuffle-frontend:nightly ghcr.io/shuffle/shuffle-frontend:1.1.0
-#
+docker tag ghcr.io/shuffle/shuffle-backend:nightly ghcr.io/shuffle/shuffle-backend:1.1.0
-#docker push ghcr.io/shuffle/shuffle-app_sdk:1.1.0
-#docker push ghcr.io/shuffle/shuffle-worker:1.1.0
-#docker push ghcr.io/shuffle/shuffle-orborus:1.1.0
-#docker push ghcr.io/shuffle/shuffle-frontend:1.1.0
-#docker push ghcr.io/shuffle/shuffle-backend:1.1.0
 
+docker push ghcr.io/shuffle/shuffle-app_sdk:1.1.0
+docker push ghcr.io/shuffle/shuffle-worker:1.1.0
+docker push ghcr.io/shuffle/shuffle-orborus:1.1.0
+docker push ghcr.io/shuffle/shuffle-frontend:1.1.0
+docker push ghcr.io/shuffle/shuffle-backend:1.1.0
 
+## frikky/shuffle
+docker tag ghcr.io/shuffle/shuffle-app_sdk:nightly 	ghcr.io/frikky/shuffle-app_sdk:1.1.0
+docker tag ghcr.io/shuffle/shuffle-worker:nightly  	ghcr.io/frikky/shuffle-worker:1.1.0
+docker tag ghcr.io/shuffle/shuffle-orborus:nightly 	ghcr.io/frikky/shuffle-orborus:1.1.0
+docker tag ghcr.io/shuffle/shuffle-frontend:nightly ghcr.io/frikky/shuffle-frontend:1.1.0
+docker tag ghcr.io/shuffle/shuffle-backend:nightly 	ghcr.io/frikky/shuffle-backend:1.1.0
+
+docker push ghcr.io/frikky/shuffle-app_sdk:1.1.0
+docker push ghcr.io/frikky/shuffle-worker:1.1.0
+docker push ghcr.io/frikky/shuffle-orborus:1.1.0
+docker push ghcr.io/frikky/shuffle-frontend:1.1.0
+docker push ghcr.io/frikky/shuffle-backend:1.1.0
 
 ### Manage worker-scale upload (Requires auth)
 # This is supposed to be unavailable, and only be downloadable by customers 

.github/workflows/docker-build.yaml

@@ -0,0 +1,66 @@
+name: docker-build
+
+on:
+  push:
+    branches: launch
+jobs:
+  main:
+    runs-on: ubuntu-latest
+    continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+         - app: frontend
+           path: frontend
+           version: 1.0.0
+           experimental: true
+         - app: backend
+           path: backend
+           version: 1.0.0
+           experimental: false
+         - app: orborus
+           path: functions/onprem/orborus
+           version: 1.0.0
+           experimental: false
+         - app: database
+           path: backend/database
+           version: 1.0.0
+           experimental: false
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Login to DockerHub
+        uses: docker/login-action@v1 
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+#        Use below configuration for ghcr.io
+#        with:
+#          registry: ghcr.io
+#          username: ${{ github.repository_owner }}
+#          password: ${{ secrets.CR_PAT }} 
+      -
+        name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        env:
+          BUILDX_NO_DEFAULT_LOAD: true
+        with:
+          context: ${{ matrix.path }}/
+          file: ${{ matrix.path }}/Dockerfile          
+          platforms: linux/amd64,linux/arm64
+          #,linux/386 - no node image I guess?
+          push: true
+          tags: ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.app }}:${{ matrix.version }}
+      -
+        name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}

backend/app_sdk/Dockerfile

@@ -1,5 +1,6 @@
 #FROM python:3.9.1-alpine as base
 FROM python:3.10.0-alpine as base
+#FROM python:3.11.3-alpine as base
 
 FROM base as builder
 RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev tzdata coreutils

backend/app_sdk/app_base.py

@@ -18,6 +18,10 @@ import urllib.parse
 import jinja2 
 import datetime
 import dateutil
+
+import threading
+import concurrent.futures
+
 from io import StringIO as StringBuffer
 from io import BytesIO
 from liquid import Liquid, defaults
@@ -138,13 +142,47 @@ def escape(a):
     a = str(a)
     return json_escape(a)
 
+@shuffle_filters.register
+def neat_json(a):
+    try:
+        a = json.loads(a)
+    except:
+        pass 
+
+    return json.dumps(a, indent=4, sort_keys=True)
+
 @shuffle_filters.register
 def flatten(a):
     a = list(a)
 
-    flat_list = [a for xs in xss for a in xs]
+    flat_list = [a for xs in a for a in xs]
     return flat_list
 
+@shuffle_filters.register
+def last(a):
+    try:
+        a = json.loads(a)
+    except:
+        pass
+
+    if len(a) == 0:
+        return ""
+
+    return a[-1]
+
+@shuffle_filters.register
+def first(a):
+    try:
+        a = json.loads(a)
+    except:
+        pass
+
+    if len(a) == 0:
+        return ""
+
+    return a[0]
+    
+
 @shuffle_filters.register
 def csv_parse(a):
     a = str(a)
@@ -186,9 +224,37 @@ def csv_parse(a):
         
         allitems.append(fullitem)
 
-    return allitems
+    try:
+        return json.dumps(allitems)
+    except:
+        print("[ERROR] Failed dumping from JSON in csv parse")
+        return allitems
+
+@shuffle_filters.register
+def parse_csv(a):
+    return csv_parse(a)
+
+@shuffle_filters.register
+def format_csv(a):
+    return csv_parse(a)
+
+@shuffle_filters.register
+def csv_format(a):
+    return csv_parse(a)@standard_filter_manager.register
+
+@shuffle_filters.register
+def split(base, sep):
+    if not sep:
+        try:
+            return json.dumps(list(base))
+        except:
+            return list(base)
+
+    try:
+        return json.dumps(base.split(sep))
+    except:
+        return base.split(sep)
 
-#print(standard_filter_manager.filters)
 #print(shuffle_filters.filters)
 #print(Liquid("{{ '10' | plus: 1}}", filters=shuffle_filters.filters).render())
 #print(Liquid("{{ '10' | minus: 1}}", filters=shuffle_filters.filters).render())
@@ -205,6 +271,7 @@ def csv_parse(a):
 ###
 ###
 
+
 class AppBase:
     __version__ = None
     app_name = None
@@ -327,13 +394,14 @@ class AppBase:
 
             new_input = fixed_return
         except Exception as e:
-            self.logger.info(f"[ERROR] Failed to run magic parser (2): {e}")
+            # Not used anymore
+            #self.logger.info(f"[ERROR] Failed to run magic parser (2): {e}")
             return input_data
 
         try:
             new_input = input_data.split()
         except Exception as e:
-            self.logger.info(f"[ERROR] Failed to run magic parser during split (1): {e}")
+            self.logger.info(f"[ERROR] Failed to run parser during split (1): {e}")
             return input_data
 
         # Won't ever touch this one?
@@ -389,9 +457,11 @@ class AppBase:
             else:
                 self.logger.warning(f"[WARNING] Magic output not defined.")
         except KeyError as e:
-            self.logger.warning(f"[DEBUG] Failed to run magic autoparser (send result) - keyerror: {e}")
+            #self.logger.warning(f"[DEBUG] Failed to run magic autoparser (send result) - keyerror: {e}")
+            pass 
         except Exception as e:
-            self.logger.warning(f"[DEBUG] Failed to run magic autoparser (send result): {e}")
+            #self.logger.warning(f"[DEBUG] Failed to run magic autoparser (send result): {e}")
+            pass
 
         # Try it with some magic
 
@@ -401,15 +471,6 @@ class AppBase:
 
         # FIXME: Add cleanup of parameters to not send to frontend here
         params = {}
-        #action = action_result["action"]
-        #try:
-        #    for item in action["authentication"]:
-        #        for action["parameters"]
-        #        self.logger.info("AUTH: ", key, value)
-        #        params[item["key"]] = item["value"]
-       #except KeyError:
-        #    self.logger.info("No authentication specified!")
-        #    pass
 
         # I wonder if this actually works 
         self.logger.info(f"[DEBUG] Before last stream result")
@@ -417,7 +478,7 @@ class AppBase:
         self.logger.info(f"[INFO] URL FOR RESULT (URL): {url}")
 
         try:
-            log_contents = "disabled: add env SHUFFLE_LOGS_DISABLED=true to Orborus to re-enable logs for apps"
+            log_contents = "disabled: add env SHUFFLE_LOGS_DISABLED=true to Orborus to re-enable logs for apps. Can not be enabled natively in Cloud except in Hybrid mode."
             if not os.getenv("SHUFFLE_LOGS_DISABLED") == "true":
                 log_contents = self.log_capture_string.getvalue()
 
@@ -446,9 +507,9 @@ class AppBase:
             finished = False
             for i in range (0, 10):
                 try:
-                    ret = requests.post(url, headers=headers, json=action_result, timeout=10)
+                    ret = requests.post(url, headers=headers, json=action_result, timeout=10, verify=False)
 
-                    self.logger.info(f"[DEBUG] Result: {ret.status_code} (break on 200)")
+                    self.logger.info(f"[DEBUG] Result: {ret.status_code} (break on 200 or 201)")
                     if ret.status_code == 200 or ret.status_code == 201:
                         finished = True
                         break
@@ -493,7 +554,7 @@ class AppBase:
                 action_result["status"] = "FAILURE"
                 action_result["result"] = json.dumps({"success": False, "reason": "POST error: Failed connecting to %s over 10 retries to the backend" % url})
                 self.logger.info(f"[DEBUG] Before typeerror stream result - NOT finished after 10 requests")
-                ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result)
+                ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result, verify=False)
         
             self.logger.info(f"""[DEBUG] Successful request result request: Status= {ret.status_code} & Response= {ret.text}. Action status: {action_result["status"]}""")
         except requests.exceptions.ConnectionError as e:
@@ -503,7 +564,7 @@ class AppBase:
             action_result["result"] = json.dumps({"success": False, "reason": "Typeerror when sending to backend URL %s" % url})
 
             self.logger.info(f"[DEBUG] Before typeerror stream result: {e}")
-            ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result)
+            ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result, verify=False)
             #self.logger.info(f"[DEBUG] Result: {ret.status_code}")
             #if ret.status_code != 200:
             #    pr
@@ -632,7 +693,7 @@ class AppBase:
             #self.logger.info(f"RET: {ret.text}")
             #self.logger.info(f"ID: {ret.status_code}")
             url = f"{self.url}/api/v1/orgs/{org_id}/validate_app_values"
-            ret = requests.post(url, json=data)
+            ret = requests.post(url, json=data, verify=False)
             if ret.status_code == 200:
                 json_value = ret.json()
                 if len(json_value["found"]) > 0: 
@@ -971,11 +1032,9 @@ class AppBase:
             for subparams in param_multiplier:
                 #self.logger.info(f"SUBPARAMS IN MULTI: {subparams}")
                 try:
-                    #tmp = await func(**subparams)
 
                     while True:
                         try:
-                            #tmp = await func(**subparams)
                             tmp = func(**subparams)
                             break
                         except TypeError as e:
@@ -988,7 +1047,7 @@ class AppBase:
                     
                                     try:
                                         del subparams[field]
-                                        self.logger.info("Removed field invalid field %s" % field)
+                                        self.logger.info("Removed invalid field %s (1)" % field)
                                     except KeyError:
                                         break
                             else:
@@ -1015,6 +1074,7 @@ class AppBase:
 
 
                 # An attempt at decomposing coroutine results
+                # Backwards compatibility
                 try:
                     if asyncio.iscoroutine(tmp):
                         self.logger.info("[DEBUG] In coroutine (2)")
@@ -1028,7 +1088,8 @@ class AppBase:
 
                         tmp = asyncio.run(parse_value(tmp))
                     else:
-                        self.logger.info("[DEBUG] Not in coroutine (2)")
+                        #self.logger.info("[DEBUG] Not in coroutine (2)")
+                        pass
                 except Exception as e:
                     self.logger.warning("[ERROR] Failed to parse coroutine value for old app: {e}")
 
@@ -1102,10 +1163,11 @@ class AppBase:
 
         get_path = "/api/v1/files/namespaces/%s?execution_id=%s&ids=true" % (category, self.full_execution["execution_id"])
         headers = {
-            "Authorization": "Bearer %s" % self.authorization
+            "Authorization": "Bearer %s" % self.authorization,
+            "User-Agent": "Shuffle 1.1.0",
         }
 
-        ret = requests.get("%s%s" % (self.url, get_path), headers=headers)
+        ret = requests.get("%s%s" % (self.url, get_path), headers=headers, verify=False)
         return ret.json()
         #if ret1.status_code != 200:
         #    return {
@@ -1127,10 +1189,11 @@ class AppBase:
 
         get_path = "/api/v1/files/namespaces/%s?execution_id=%s" % (namespace, self.full_execution["execution_id"])
         headers = {
-            "Authorization": "Bearer %s" % self.authorization
+            "Authorization": "Bearer %s" % self.authorization,
+            "User-Agent": "Shuffle 1.1.0",
         }
 
-        ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers)
+        ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers, verify=False)
         if ret1.status_code != 200:
             return None 
 
@@ -1189,10 +1252,11 @@ class AppBase:
             get_path = "/api/v1/files/%s?execution_id=%s" % (item, full_execution["execution_id"])
             headers = {
                 "Content-Type": "application/json",     
-                "Authorization": "Bearer %s" % self.authorization
+                "Authorization": "Bearer %s" % self.authorization,
+                "User-Agent": "Shuffle 1.1.0",
             }
 
-            ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers)
+            ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers, verify=False)
             self.logger.info("RET1 (file get): %s" % ret1.text)
             if ret1.status_code != 200:
                 returns.append({
@@ -1203,7 +1267,7 @@ class AppBase:
                 continue
 
             content_path = "/api/v1/files/%s/content?execution_id=%s" % (item, full_execution["execution_id"])
-            ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers)
+            ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers, verify=False)
             self.logger.info("RET2 (file get) done")
             if ret2.status_code == 200:
                 tmpdata = ret1.json()
@@ -1239,7 +1303,7 @@ class AppBase:
             "value": str(value),
         }
 
-        response = requests.post(url, json=data)
+        response = requests.post(url, json=data, verify=False)
         try:
             allvalues = response.json()
             allvalues["key"] = key
@@ -1261,7 +1325,7 @@ class AppBase:
             "key": key,
         }
 
-        value = requests.post(url, json=data)
+        value = requests.post(url, json=data, verify=False)
         try:
             allvalues = value.json()
             self.logger.info("VAL1: ", allvalues)
@@ -1291,7 +1355,8 @@ class AppBase:
         org_id = full_execution["workflow"]["execution_org"]["id"]
         headers = {
             "Content-Type": "application/json",     
-            "Authorization": "Bearer %s" % self.authorization
+            "Authorization": "Bearer %s" % self.authorization,
+            "User-Agent": "Shuffle 1.1.0",
         }
 
         if not isinstance(infiles, list):
@@ -1314,7 +1379,7 @@ class AppBase:
                 self.logger.info(f"KeyError in file setup: {e}")
                 pass
 
-            ret = requests.post("%s%s" % (self.url, create_path), headers=headers, json=data)
+            ret = requests.post("%s%s" % (self.url, create_path), headers=headers, json=data, verify=False)
             #self.logger.info(f"Ret CREATE: {ret.text}")
             cur_id = ""
             if ret.status_code == 200:
@@ -1337,6 +1402,7 @@ class AppBase:
 
             new_headers = {
                 "Authorization": f"Bearer {self.authorization}",
+                "User-Agent": "Shuffle 1.1.0",
             }
 
             upload_path = "/api/v1/files/%s/upload?execution_id=%s" % (cur_id, full_execution["execution_id"])
@@ -1345,7 +1411,7 @@ class AppBase:
             files={"shuffle_file": (filename, curfile["data"])}
             #open(filename,'rb')}
 
-            ret = requests.post("%s%s" % (self.url, upload_path), files=files, headers=new_headers)
+            ret = requests.post("%s%s" % (self.url, upload_path), files=files, headers=new_headers, verify=False)
             self.logger.info("Ret UPLOAD: %s" % ret.text)
             self.logger.info("Ret2 UPLOAD: %d" % ret.status_code)
 
@@ -1383,7 +1449,8 @@ class AppBase:
 
         headers = {
             "Content-Type": "application/json",     
-            "Authorization": f"Bearer {self.authorization}" 
+            "Authorization": f"Bearer {self.authorization}",
+            "User-Agent": "Shuffle 1.1.0",
         }
 
         if len(self.action) == 0:
@@ -1411,7 +1478,7 @@ class AppBase:
 
         # FIXME: Shouldn't skip this, but it's good for minimzing API calls
         #try:
-        #    ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result)
+        #    ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result, verify=False)
         #    self.logger.info("Workflow: %d" % ret.status_code)
         #    if ret.status_code != 200:
         #        self.logger.info(ret.text)
@@ -1426,7 +1493,7 @@ class AppBase:
         # If found, we get the full results list from backend
         fullexecution = {}
         if isinstance(self.full_execution, str) and len(self.full_execution) == 0:
-            self.logger.info("[DEBUG] NO EXECUTION - LOADING!")
+            #self.logger.info("[DEBUG] NO EXECUTION - LOADING!")
             try:
                 failed = False
                 rettext = ""
@@ -1440,7 +1507,8 @@ class AppBase:
                     ret = requests.post(
                         "%s/api/v1/streams/results" % (self.base_url), 
                         headers=headers, 
-                        json=tmpdata
+                        json=tmpdata,
+                        verify=False
                     )
 
                     if ret.status_code == 200:
@@ -1448,12 +1516,13 @@ class AppBase:
                         failed = False
                         break
 
-                    elif ret.status_code == 500:
+                    #elif ret.status_code == 500 or ret.status_code == 400:
+                    elif ret.status_code >= 400:
                         self.logger.info("[ERROR] (fails: %d) Error in app with status code %d for results (1). RETRYING because results can't be handled" % (i+1, ret.status_code))
                     
                         rettext = ret.text
                         failed = True 
-                        time.sleep(10)
+                        time.sleep(8)
                         continue
 
                     else:
@@ -1461,6 +1530,7 @@ class AppBase:
 
                         rettext = ret.text
                         failed = True 
+                        time.sleep(8)
                         break
 
                 if failed:
@@ -1497,6 +1567,13 @@ class AppBase:
 
         self.full_execution = fullexecution
 
+        #try:
+        #    if "backend_url" in self.full_execution:
+        #        self.url = self.full_execution["backend_url"]
+        #        self.base_url = self.full_execution["backend_url"]
+        #except KeyError:
+        #    pass
+
         try:
             if replace_params == True:
                 for inner_action in self.full_execution["workflow"]["actions"]:
@@ -1522,7 +1599,7 @@ class AppBase:
         except Exception as e:
             self.logger.info(f"[WARNING] Failed in replace params action parsing: {e}")
 
-        self.logger.info("[DEBUG] AFTER FULLEXEC stream result (init)")
+        self.logger.info(f"[DEBUG] AFTER FULLEXEC stream result (init): {self.current_execution_id}")
 
         # Gets the value at the parenthesis level you want
         def parse_nested_param(string, level):
@@ -1708,7 +1785,7 @@ class AppBase:
 
             # Because liquid can handle ALL of this now.
             # Implemented for >0.9.25
-            self.logger.info("[DEBUG] Skipping parser because use of its been deprecated >0.9.25 due to Liquid implementation")
+            #self.logger.info("[DEBUG] Skipping parser because use of its been deprecated >0.9.25 due to Liquid implementation")
             return data, False
 
             wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length", "lenght", "join", "replace"]
@@ -1761,7 +1838,7 @@ class AppBase:
                     else:
                         parse_string = inner_result
 
-            print("PARSE STRING: %s" % parse_string)
+            #print("PARSE STRING: %s" % parse_string)
             return parse_string, True
 
         # Looks for parantheses to grab special cases within a string, e.g:
@@ -1900,7 +1977,7 @@ class AppBase:
                         if isinstance(seconditem, int):
                             seconditem = str(seconditem)
 
-                        print("[DEBUG] ACTUAL PARSED: %s" % actualitem)
+                        #print("[DEBUG] ACTUAL PARSED: %s" % actualitem)
 
                         # Means it's a single item -> continue
                         if seconditem == "":
@@ -2022,13 +2099,13 @@ class AppBase:
             actionname_lower = parsersplit[0][1:].lower()
 
             #Actionname: Start_node
-            print(f"\n[INFO] Actionname: {actionname_lower}")
+            #print(f"\n[INFO] Actionname: {actionname_lower}")
 
             # 1. Find the action
             baseresult = ""
 
             appendresult = "" 
-            print("[INFO] Parsersplit length: %d" % len(parsersplit))
+            #print("[INFO] Parsersplit length: %d" % len(parsersplit))
             if (actionname_lower.startswith("exec ") or actionname_lower.startswith("webhook ") or actionname_lower.startswith("schedule ") or actionname_lower.startswith("userinput ") or actionname_lower.startswith("email_trigger ") or actionname_lower.startswith("trigger ")) and len(parsersplit) == 1:
                 record = False
                 for char in actionname_lower:
@@ -2071,7 +2148,6 @@ class AppBase:
                         print("[DEBUG] No results to get values from.")
                         baseresult = "$" + parsersplit[0][1:] 
                     
-                    print("[DEBUG] BEFORE VARIABLES!")
                     if len(baseresult) == 0:
                         try:
                             for variable in execution_data["workflow"]["workflow_variables"]:
@@ -2082,13 +2158,12 @@ class AppBase:
                                     break
 
                         except KeyError as e:
-                            print("[INFO] KeyError wf variables: %s" % e)
+                            #print("[INFO] KeyError wf variables: %s" % e)
                             pass
                         except TypeError as e:
-                            print("[INFO] TypeError wf variables: %s" % e)
+                            #print("[INFO] TypeError wf variables: %s" % e)
                             pass
         
-                    print("[DEBUG] BEFORE EXECUTION VAR")
                     if len(baseresult) == 0:
                         try:
                             for variable in execution_data["execution_variables"]:
@@ -2106,14 +2181,14 @@ class AppBase:
             except KeyError as error:
                 print(f"[DEBUG] KeyError in JSON: {error}")
         
-            print(f"[INFO] After first trycatch. Baseresult")#, baseresult)
+            #print(f"[INFO] After first trycatch. Baseresult")#, baseresult)
         
             # 2. Find the JSON data
             # Returns if there isn't any JSON in the base ($nodename)
             if len(baseresult) == 0:
                 return ""+appendresult, False
         
-            print("[INFO] After second return")
+            #print("[INFO] After second return")
             # Returns if the result is JUST something like $nodename, not $nodename.value
             if len(parsersplit) == 1:
                 returndata = str(baseresult)+str(appendresult)
@@ -2124,7 +2199,7 @@ class AppBase:
             baseresult = baseresult.replace(" False", " false,")
 
             # Tries to actually read it as JSON with some stupid formatting
-            print("[INFO] After third parser return - Formatted")#, baseresult)
+            #print("[INFO] After third parser return - Formatted")#, baseresult)
             basejson = {}
             try:
                 basejson = json.loads(baseresult)
@@ -2148,7 +2223,6 @@ class AppBase:
                     print("[WARNING] Parseditem issue: %s" % e)
                     pass
 
-            print("[DEBUG] DATA: (%s) %s" % (type(data), data))
             if is_loop:
                 print("[DEBUG] DATA IS A LOOP - SHOULD WRAP")
                 if parsersplit[-1] == "#":
@@ -2160,7 +2234,6 @@ class AppBase:
                     parseditem = "${%s%s}$" % (parsersplit[-1], json.dumps(data))
 
 
-            print("[DEBUG] Before last return with %s" % appendresult)
             returndata = str(parseditem)+str(appendresult)
 
             # New in 0.8.97: Don't return items without lists
@@ -2409,9 +2482,11 @@ class AppBase:
                         newvalue[key] = recurse_cleanup_script(value)
         
             except json.decoder.JSONDecodeError as e:
-                print(f"[WARNING] Failed JSON replacement for OpenAPI keys (3) {e}")
+                # Since here the data isn't at all JSON compatible..?
+                # Seems to happen with newlines in variables being parsed in as strings?
+                print(f"[ERROR] Failed JSON replacement for OpenAPI keys (3) {e}. Value: {data}")
             except Exception as e:
-                print(f"[WARNING] Failed as an exception (1): {e}") 
+                print(f"[ERROR] Failed as an exception (1): {e}") 
                 
             try:
                 for deletekey in deletekeys:
@@ -2476,9 +2551,17 @@ class AppBase:
             except:
                 self.logger.info("Error in initial replacement of escaped dollar!")
 
-            # Basic fix in case variant isn't set
+            paramname = ""
             try:
-                self.logger.info(f"[DEBUG] Parameter variant: {parameter['variant']} of length {len(parameter['value'])}")
+                paramname = parameter["name"]
+            except:
+                pass
+
+            # Basic fix in case variant isn't set
+            # Variant is ALWAYS STATIC_VALUE from mid 2021~ 
+            try:
+                #self.logger.info(f"[DEBUG] Parameter '{paramname}' of length {len(parameter['value'])}")
+                parameter["variant"] = parameter["variant"]
             except:
                 parameter["variant"] = "STATIC_VALUE"
 
@@ -2651,19 +2734,6 @@ class AppBase:
                     return True
 
                 return False
-                #if tmp == "[]":
-                #    tmp = []
-
-                #if type(tmp) == list and len(tmp) == 0 and not flip:
-                #    new_list.append(item)
-                #elif type(tmp) == list and len(tmp) > 0 and flip:
-                #    new_list.append(item)
-                #elif type(tmp) == str and not tmp and not flip:
-                #    new_list.append(item)
-                #elif type(tmp) == str and tmp and flip:
-                #    new_list.append(item)
-                #else:
-                #    failed_list.append(item)
 
             elif check.lower() == "contains_any_of":
                 newvalue = [destinationvalue.lower()]
@@ -2729,6 +2799,7 @@ class AppBase:
                     return True, ""
             except Exception as error:
                 self.logger.info(f"[WARNING] Failed checking startnode: {error}")
+                return True, ""
 
             available_checks = [
                 "=",
@@ -2783,9 +2854,6 @@ class AppBase:
                     correct_branches += 1
                     continue
 
-                # FIXME: Check if the previous node has a result or not 
-
-                #self.logger.info("[DEBUG] Relevant conditions: %s" % branch["conditions"])
                 successful_conditions = []
                 failed_conditions = []
                 successful_conditions = 0
@@ -2798,27 +2866,20 @@ class AppBase:
                     check, sourcevalue, is_loop = parse_params(action, fullexecution, condition["source"], self)
                     if check:
                         continue
-                        return False, {"success": False, "reason": "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check)}
 
-                    #sourcevalue = sourcevalue.encode("utf-8")
                     sourcevalue = parse_wrapper_start(sourcevalue, self)
                     destinationvalue = condition["destination"]["value"]
 
                     check, destinationvalue, is_loop = parse_params(action, fullexecution, condition["destination"], self)
                     if check:
                         continue
-                        return False, {"success": False, "reason": "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check)}
 
-                    #destinationvalue = destinationvalue.encode("utf-8")
                     destinationvalue = parse_wrapper_start(destinationvalue, self)
 
                     if not condition["condition"]["value"] in available_checks:
                         self.logger.warning("Skipping %s %s %s because %s is invalid." % (sourcevalue, condition["condition"]["value"], destinationvalue, condition["condition"]["value"]))
                         continue
 
-                    #self.logger.info(destinationvalue)
-                    # NEGATE 
-
                     # Configuration = negated because of WorkflowAppActionParam..
                     validation = run_validation(sourcevalue, condition["condition"]["value"], destinationvalue)
                     try:
@@ -2830,12 +2891,6 @@ class AppBase:
                     if validation == True:
                         successful_conditions += 1
 
-                    #if not validation:
-                    #    self.logger.info("Failed condition check for %s %s %s." % (sourcevalue, condition["condition"]["value"], destinationvalue))
-                    #    return False, {"success": False, "reason": "Failed condition (3): %s %s %s" % (sourcevalue, condition["condition"]["value"], destinationvalue)}
-
-                #self.logger.info("CONDITIONS VS SUCCESS: %d vs %d" % (total_conditions, successful_conditions))
-
                 if total_conditions == successful_conditions:
                     correct_branches += 1
     
@@ -2845,15 +2900,9 @@ class AppBase:
             if matching_branches > 0 and correct_branches > 0:
                 return True, ""
 
-            # FIXME: Check if previous branches are at all finished
-
             self.logger.info("[DEBUG] Correct branches vs matching branches: %d vs %d" % (correct_branches, matching_branches))
             return False, {"success": False, "reason": "Minimum of one branch's conditions must be correct to continue. Total: %d of %d" % (correct_branches, matching_branches)}
 
-            #Correct branches vs matching branches: 1 vs 1
-            #if 
-            return True, ""
-
 
         #
         #
@@ -2881,7 +2930,7 @@ class AppBase:
         # Checks whether conditions are met, otherwise set 
         branchcheck, tmpresult = check_branch_conditions(action, fullexecution, self)
         if isinstance(tmpresult, object) or isinstance(tmpresult, list) or isinstance(tmpresult, dict):
-            self.logger.info("[DEBUG] Fixing branch return as object -> string")
+            #self.logger.info("[DEBUG] Fixing branch return as object -> string")
             try:
                 #tmpresult = tmpresult.replace("'", "\"")
                 tmpresult = json.dumps(tmpresult) 
@@ -2904,7 +2953,8 @@ class AppBase:
         if " " in actionname:
             actionname.replace(" ", "_", -1) 
 
-        #print(action)
+        #print("ACTION: ", action)
+        #print("exec: ", self.full_execution)
         #if action.generated:
         #    actionname = actionname.lower()
 
@@ -2916,7 +2966,8 @@ class AppBase:
                 self.action_result["status"] = "FAILURE" 
                 self.action_result["result"] = json.dumps({
                     "success": False,
-                    "reason": f"Function {actionname} doesn't exist.",
+                    "reason": f"Function {actionname} doesn't exist, or the App is out of date.",
+                    "details": "If this persists, please restart delete the Docker image locally, restart your Orborus instance and then try again to force-download the latest version. Contact support@shuffler.io with this data if the issue persists.",
                 })
             elif callable(func):
                 try:
@@ -2931,12 +2982,14 @@ class AppBase:
                         # What variables are necessary here tho hmm
 
                         params = {}
-                        try:
+                        # This replacement should happen in backend as part of params
-                            for item in action["authentication"]:
+                        # error log is useless
-                                #self.logger.info("AUTH: ", key, value)
+                        #try:
-                                params[item["key"]] = item["value"]
+                        #    for item in action["authentication"]:
-                        except KeyError:
+                        #        self.logger.info("AUTH PARAM: ", key, value)
-                            self.logger.info("[DEBUG] No authentication specified!")
+                        #        #params[item["key"]] = item["value"]
+                        #except KeyError as e:
+                        #    self.logger.info(f"[WARNING] No authentication specified! Is this correct? err: {e}")
 
                         # Fixes OpenAPI body parameters for later.
                         newparams = []
@@ -3007,13 +3060,12 @@ class AppBase:
                                     pass
 
                                  
-                                self.logger.info(f"""HANDLING {action["parameters"][counter]["value"]}""")
+                                self.logger.info(f"""HANDLING BODY: {action["parameters"][counter]["value"]}""")
                                 action["parameters"][counter]["value"] = recurse_cleanup_script(action["parameters"][counter]["value"])
 
                         #self.logger.info(action["parameters"])
 
                         # This seems redundant now 
-                        self.logger.info("[DEBUG] Pre parameters")
                         for parameter in newparams:
                             action["parameters"].append(parameter)
 
@@ -3035,7 +3087,6 @@ class AppBase:
 
                         # Multi_parameter has the data for each. variable
                         minlength = 0
-                        self.logger.info("[DEBUG] Pre-loading parameters")
                         multi_parameters = json.loads(json.dumps(params))
                         multiexecution = False
                         multi_execution_lists = []
@@ -3293,9 +3344,6 @@ class AppBase:
                                 # This part has fucked over so many random JSON usages because of weird paranthesis parsing
 
                                 value = parse_wrapper_start(value, self)
-                                #self.logger.info("[DEBUG] Post return: %s" % value)
-
-                                #self.logger.info("POST data value: %s" % value)
 
                                 try:
                                     if str(value).startswith("b'") and str(value).endswith("'"):
@@ -3428,8 +3476,73 @@ class AppBase:
                                     break
 
                                 try:
-                                    newres = func(**params)
+                                    #try:
+                                    # Individual functions shouldn't take longer than this
+                                    # This is an attempt to make timeouts occur less, incentivizing users to make use efficient API's
+                                    # PS: Not implemented for lists - only single actions as of May 2023
+                                    timeout = 30 
+
+                                    # Check if current app is Shuffle Tools, then set to 55 due to certain actions being slow (ioc parser..) 
+                                    #uu In general, this should be disabled for onprem 
+                                    if self.action["app_name"].lower() == "shuffle tools":
+                                        timeout = 55
+
+                                    timeout = 30 
+
+                                    try:
+                                        executor = concurrent.futures.ThreadPoolExecutor()
+                                        future = executor.submit(func, **params)
+                                        newres = future.result(timeout)
+
+                                        if not future.done():
+                                            # The future is still running, so we need to cancel it
+                                            future.cancel()
+                                            newres = json.dumps({
+                                                "success": False,
+                                                "reason": "Timeout error within %d seconds. This happens if we can't reach or use the API you're trying to use within the time limit." % timeout,
+                                                "exception": str(e),
+                                            })
+
+                                        else:
+                                            # The future is done, so we can just get the result from newres :)
+                                            #newres = future.result()
+                                            #print("Future is done!")
+                                            pass
+
+                                    except concurrent.futures.TimeoutError as e:
+                                        newres = json.dumps({
+                                            "success": False,
+                                            "reason": "Timeout error within %d seconds (2). This happens if we can't reach or use the API you're trying to use within the time limit" % timeout
+                                        })
+
                                     break
+
+
+
+                                    #thread = threading.Thread(target=func, args=(**params,))
+                                    #thread.start()
+
+                                    #thread.join(timeout)
+
+                                    #if thread.is_alive():
+                                    #    # The thread is still running, so we need to stop it
+                                    #    # You can handle this as needed, such as raising an exception
+                                    #    timeout_handler()
+
+
+                                    #with Timeout(timeout):
+                                    #    newres = func(**params)
+                                    #    break
+                                    #except Timeout.Timeout as e:
+                                    #    self.logger.info(f"[DEBUG] Timeout error: {e}")
+                                    #    newres = json.dumps({
+                                    #        "success": False,
+                                    #        "reason": "Timeout error within %d seconds. This typically happens if we can't reach the API you're trying to reach." % timeout,
+                                    #        "exception": str(e),
+                                    #    })
+
+                                    #    break
+
                                 except TypeError as e:
                                     newres = ""
                                     self.logger.info(f"[DEBUG] Got exec type error: {e}")
@@ -3442,7 +3555,7 @@ class AppBase:
                                     errorstring = f"{e}"
 
                                     if "the JSON object must be" in errorstring:
-                                        self.logger.info("[ERROR] Something is wrong with the input for this function. Are lists and JSON data handled parsed properly (0)?")
+                                        self.logger.info("[ERROR] Something is wrong with the input for this function. Are lists and JSON data handled parsed properly (0)? the JSON object must be in...")
                                         try:
                                             e = json.loads(f"{e}")
                                         except:
@@ -3461,7 +3574,7 @@ class AppBase:
                             
                                             try:
                                                 del params[field]
-                                                self.logger.info("[WARNING] Removed field invalid field %s" % field)
+                                                self.logger.info("[WARNING] Removed invalid field %s (2)" % field)
                                             except KeyError:
                                                 break
                                     else:
@@ -3472,7 +3585,7 @@ class AppBase:
                                         })
                                         break
                                 except Exception as e:
-                                    self.logger.info("[ERROR] Something is wrong with the input for this function. Are lists and JSON data handled parsed properly (1)?")
+                                    self.logger.info(f"[ERROR] Something is wrong with the input for this function. Are lists and JSON data handled parsed properly (1)? err: {e}")
 
                                     try:
                                         e = json.loads(f"{e}")
@@ -3499,7 +3612,8 @@ class AppBase:
 
                                     newres = asyncio.run(parse_value(newres))
                                 else:
-                                    self.logger.info("[DEBUG] Not in coroutine (1)")
+                                    #self.logger.info("[DEBUG] Not in coroutine (1)")
+                                    pass
                             except Exception as e:
                                 self.logger.warning("[ERROR] Failed to parse coroutine value for old app: {e}")
 
@@ -3553,7 +3667,7 @@ class AppBase:
                                     result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres)
                                     self.logger.info("Can't handle type %s value from function" % (type(newres)))
 
-                            self.logger.info("[INFO] POST NEWRES RESULT!")#, result)
+                            #self.logger.info("[INFO] POST NEWRES RESULT!")#, result)
                         else:
                             #self.logger.info("[INFO] APP_SDK DONE: Starting MULTI execution (length: %d) with values %s" % (minlength, multi_parameters))
                             # 1. Use number of executions based on the arrays being similar
@@ -3712,7 +3826,6 @@ class AppBase:
             def execute():
                 if request.method == "POST":
                     #print(request.get_json(force=True))
-                    #print("DATA: ", request.data)
                     requestdata = {}
                     try:
                         requestdata = json.loads(request.data)
@@ -3725,7 +3838,6 @@ class AppBase:
                     #logger.info(f"[DEBUG] Datatype: {type(requestdata)}: {requestdata}")
 
                     # Remaking class for each request
-                    #print(f"APP: {app}")
         
                     app = cls(redis=None, logger=logger, console_logger=logger)
                     extra_info = ""
@@ -3754,7 +3866,7 @@ class AppBase:
                         # BASE URL (backend)
                         try:
                             app.url = requestdata["url"]
-                            logger.info(f"BACKEND URL: {app.url}")
+                            logger.info(f"BACKEND URL (url): {app.url}")
                         except Exception as e:
                             logger.info(f"[ERROR] Failed parsing url (backend): {e}")
                             extra_info += f"\n{e}"
@@ -3762,7 +3874,7 @@ class AppBase:
                         # URL (worker)
                         try:
                             app.base_url = requestdata["base_url"]
-                            logger.info(f"WORKER URL: {app.base_url}")
+                            logger.info(f"WORKER URL (base url): {app.base_url}")
                         except Exception as e:
                             logger.info(f"[ERROR] Failed parsing base url (worker): {e}")
                             extra_info += f"\n{e}"
@@ -3840,11 +3952,7 @@ class AppBase:
             else:
                 self.logger.info("ACTION TYPE (unhandled): %s" % type(action))
 
-            #await app.execute_action(app.action)
             app.execute_action(app.action)
 
-    #app.run(host="0.0.0.0", port=33334)
-
 if __name__ == "__main__":
     AppBase.run()
-    #asyncio.run(AppBase.run(), debug=True)

backend/app_sdk/build.sh

@@ -2,7 +2,7 @@
 
 ### DEFAULT 
 NAME=shuffle-app_sdk
-VERSION=1.1.0
+VERSION=1.2.0
 
 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force
 docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly -t shuffle/shuffle:app_sdk -t shuffle/$NAME:$VERSION -t docker.pkg.github.com/shuffle/shuffle/$NAME:$VERSION -t ghcr.io/shuffle/$NAME:$VERSION -t ghcr.io/shuffle/$NAME:nightly

backend/build.sh

@@ -1,10 +1,10 @@
 #!/bin/sh
 docker stop shuffle-backend
 docker rm shuffle-backend
-docker rmi frikky/shuffle:backend 
+docker rmi ghcr.io/shuffle/shuffle-backend:nightly
 
-docker build . -t frikky/shuffle:backend
+docker build . -t ghcr.io/shuffle/shuffle-backend:nightly
-docker push frikky/shuffle:backend
+docker push ghcr.io/shuffle/shuffle-backend:nightly
 
 echo "Starting server"
 #docker run -it \

backend/go-app/docker.go

@@ -241,16 +241,15 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin
 		BuildArgs: map[string]*string{},
 		Labels:    labels,
 	}
-
 	// NetworkMode: "host",
 
 	httpProxy := os.Getenv("HTTP_PROXY")
 	if len(httpProxy) > 0 {
-		buildOptions.BuildArgs["http_proxy"] = &httpProxy
+		buildOptions.BuildArgs["HTTP_PROXY"] = &httpProxy
 	}
 	httpsProxy := os.Getenv("HTTPS_PROXY")
 	if len(httpProxy) > 0 {
-		buildOptions.BuildArgs["https_proxy"] = &httpsProxy
+		buildOptions.BuildArgs["HTTPS_PROXY"] = &httpsProxy
 	}
 
 	// Build the actual image
@@ -352,7 +351,7 @@ func buildImage(tags []string, dockerfileFolder string) error {
 
 	httpProxy := os.Getenv("HTTP_PROXY")
 	if len(httpProxy) > 0 {
-		buildOptions.BuildArgs["http_proxy"] = &httpProxy
+		buildOptions.BuildArgs["HTTP_PROXY"] = &httpProxy
 	}
 	httpsProxy := os.Getenv("HTTPS_PROXY")
 	if len(httpProxy) > 0 {
@@ -386,170 +385,6 @@ func buildImage(tags []string, dockerfileFolder string) error {
 	return nil
 }
 
-// FIXME - very specific for webhooks. Make it easier?
-func stopWebhook(image string, identifier string) error {
-	ctx := context.Background()
-
-	containername := fmt.Sprintf("%s-%s", image, identifier)
-
-	cli, err := client.NewEnvClient()
-	if err != nil {
-		log.Println("Unable to create docker client")
-		return err
-	}
-
-	//	containers, err := cli.ContainerList(ctx, types.ContainerListOptions{
-	//		All: true,
-	//	})
-
-	if err := cli.ContainerStop(ctx, containername, nil); err != nil {
-		log.Printf("Unable to stop container %s - running removal anyway, just in case: %s", containername, err)
-	}
-
-	removeOptions := types.ContainerRemoveOptions{
-		RemoveVolumes: true,
-		Force:         true,
-	}
-
-	if err := cli.ContainerRemove(ctx, containername, removeOptions); err != nil {
-		log.Printf("Unable to remove container: %s", err)
-	}
-
-	return nil
-}
-
-// Starts a new webhook
-func handleStopHookDocker(resp http.ResponseWriter, request *http.Request) {
-	cors := shuffle.HandleCors(resp, request)
-	if cors {
-		return
-	}
-
-	location := strings.Split(request.URL.String(), "/")
-
-	var fileId string
-	if location[1] == "api" {
-		if len(location) <= 4 {
-			resp.WriteHeader(401)
-			resp.Write([]byte(`{"success": false}`))
-			return
-		}
-
-		fileId = location[4]
-	}
-
-	if len(fileId) != 32 {
-		resp.WriteHeader(401)
-		resp.Write([]byte(`{"success": false, "message": "ID not valid"}`))
-		return
-	}
-
-	ctx := context.Background()
-	hook, err := shuffle.GetHook(ctx, fileId)
-	if err != nil {
-		log.Printf("Failed getting hook %s (stop docker): %s", fileId, err)
-		resp.WriteHeader(401)
-		resp.Write([]byte(`{"success": false}`))
-		return
-	}
-
-	log.Printf("Status: %s", hook.Status)
-	log.Printf("Running: %t", hook.Running)
-	if !hook.Running {
-		message := fmt.Sprintf("Error: %s isn't running", hook.Id)
-		log.Println(message)
-		resp.WriteHeader(401)
-		resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "%s"}`, message)))
-		return
-	}
-
-	hook.Status = "stopped"
-	hook.Running = false
-	hook.Actions = []shuffle.HookAction{}
-	err = shuffle.SetHook(ctx, *hook)
-	if err != nil {
-		log.Printf("Failed setting hook: %s", err)
-		resp.WriteHeader(401)
-		resp.Write([]byte(`{"success": false}`))
-		return
-	}
-
-	image := "webhook"
-
-	// This is here to force stop and remove the old webhook
-	err = stopWebhook(image, fileId)
-	if err != nil {
-		log.Printf("Container stop issue for %s-%s: %s", image, fileId, err)
-	}
-
-	resp.WriteHeader(200)
-	resp.Write([]byte(`{"success": true, "message": "Stopped webhook"}`))
-}
-
-// THis is an example
-// Can also be used as base data?
-var webhook = `{
-	"id": "d6ef8912e8bd37776e654cbc14c2629c",
-	"info": {
-		"url": "http://localhost:5001",
-		"name": "TheHive",
-		"description": "Webhook for TheHive"
-	},
-	"transforms": {},
-	"actions": {},
-	"type": "webhook",
-	"running": false,
-	"status": "stopped"
-}`
-
-// Starts a new webhook
-func handleDeleteHookDocker(resp http.ResponseWriter, request *http.Request) {
-	ctx := context.Background()
-	cors := shuffle.HandleCors(resp, request)
-	if cors {
-		return
-	}
-
-	location := strings.Split(request.URL.String(), "/")
-
-	var fileId string
-	if location[1] == "api" {
-		if len(location) <= 4 {
-			resp.WriteHeader(401)
-			resp.Write([]byte(`{"success": false}`))
-			return
-		}
-
-		fileId = location[4]
-	}
-
-	if len(fileId) != 32 {
-		resp.WriteHeader(401)
-		resp.Write([]byte(`{"success": false, "message": "ID not valid"}`))
-		return
-	}
-
-	err := shuffle.DeleteKey(ctx, "hooks", fileId)
-	if err != nil {
-		resp.WriteHeader(401)
-		resp.Write([]byte(`{"success": false, "message": "Can't delete"}`))
-		return
-	}
-
-	image := "webhook"
-
-	// This is here to force stop and remove the old webhook
-	err = stopWebhook(image, fileId)
-	if err != nil {
-		log.Printf("Container stop issue for %s-%s: %s", image, fileId, err)
-		resp.Write([]byte(`{"success": false, "message": "Couldn't stop webhook"}`))
-		return
-	}
-
-	resp.WriteHeader(200)
-	resp.Write([]byte(`{"success": true, "message": "Deleted webhook"}`))
-}
-
 // Checks if an image exists
 func imageCheckBuilder(images []string) error {
 	//log.Printf("[FIXME] ImageNames to check: %#v", images)
@@ -595,32 +430,7 @@ func imageCheckBuilder(images []string) error {
 	return nil
 }
 
-func hookTest() {
+// https://stackoverflow.com/questions/23935141/how-to-copy-docker-images-from-one-host-to-another-without-using-a-repository
-	var hook shuffle.Hook
-	err := json.Unmarshal([]byte(webhook), &hook)
-	log.Println(webhook)
-	if err != nil {
-		log.Printf("Failed hook unmarshaling: %s", err)
-		return
-	}
-
-	ctx := context.Background()
-	err = shuffle.SetHook(ctx, hook)
-	if err != nil {
-		log.Printf("Failed setting hook: %s", err)
-	}
-
-	returnHook, err := shuffle.GetHook(ctx, hook.Id)
-	if err != nil {
-		log.Printf("Failed getting hook %s (test): %s", hook.Id, err)
-	}
-
-	if len(returnHook.Id) > 0 {
-		log.Printf("Success! - %s", returnHook.Id)
-	}
-}
-
-//https://stackoverflow.com/questions/23935141/how-to-copy-docker-images-from-one-host-to-another-without-using-a-repository
 func getDockerImage(resp http.ResponseWriter, request *http.Request) {
 	cors := shuffle.HandleCors(resp, request)
 	if cors {
@@ -643,14 +453,9 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) {
 		return
 	}
 
-	type requestCheck struct {
-		Name string `datastore:"name" json:"name" yaml:"name"`
-	}
-
 	// This has to be done in a weird way because Datastore doesn't
 	// support map[string]interface and similar (openapi3.Swagger)
-	var version requestCheck
+	var version shuffle.DockerRequestCheck
-
 	err = json.Unmarshal(body, &version)
 	if err != nil {
 		resp.WriteHeader(422)
@@ -684,8 +489,11 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) {
 		alternativeName = strings.Join(alternativeNameSplit[1:3], "/")
 	}
 
+	log.Printf("[INFO] Trying to download image: %s. Alt: %s", version.Name, alternativeName)
+
 	for _, image := range images {
 		for _, tag := range image.RepoTags {
+			//log.Printf("[DEBUG] Tag: %s", tag)
 			if strings.ToLower(tag) == strings.ToLower(version.Name) {
 				img = image
 				tagFound = tag
@@ -699,11 +507,34 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) {
 		}
 	}
 
+	pullOptions := types.ImagePullOptions{}
+	if len(img.ID) == 0 {
+		_, err := dockercli.ImagePull(context.Background(), version.Name, pullOptions)
+		if err == nil {
+			tagFound = version.Name
+			img.ID = version.Name
+			img2.ID = version.Name
+
+			dockercli.ImageTag(ctx, version.Name, alternativeName)
+		}
+	}
+
+	if len(img2.ID) == 0 {
+		_, err := dockercli.ImagePull(context.Background(), alternativeName, pullOptions)
+		if err == nil {
+			tagFound = alternativeName
+			img.ID = alternativeName
+			img2.ID = alternativeName
+
+			dockercli.ImageTag(ctx, alternativeName, version.Name)
+		}
+	}
+
 	// REBUILDS THE APP
 	if len(img.ID) == 0 {
 		if len(img2.ID) == 0 {
 			workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 0, 0)
-			//log.Printf("[INFO] Getting workflowapps for a rebuild. Got %d with err %#v", len(workflowapps), err)
+			log.Printf("[INFO] Getting workflowapps for a rebuild. Got %d with err %#v", len(workflowapps), err)
 			if err == nil {
 				imageName := ""
 				imageVersion := ""
@@ -728,7 +559,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) {
 					foundApp := shuffle.WorkflowApp{}
 					imageName = strings.ToLower(imageName)
 					imageVersion = strings.ToLower(imageVersion)
-					log.Printf("[DEBUG] Looking for appname %s with version %s", imageName, imageVersion)
+					log.Printf("[DEBUG] Docker Looking for appname %s with version %s", imageName, imageVersion)
 
 					for _, app := range workflowapps {
 						if strings.ToLower(strings.Replace(app.Name, " ", "_", -1)) == imageName && app.AppVersion == imageVersion {
@@ -848,7 +679,7 @@ func handleRemoteDownloadApp(resp http.ResponseWriter, ctx context.Context, user
 		app := tmpapp{}
 		err := json.Unmarshal(respBody, &app)
 		if err != nil || app.Success == false || len(app.OpenAPI) == 0 {
-			log.Printf("[ERROR] Failed app unmarshal during auto-download. Success%#v. Applength: %d: %s", app.Success, len(app.OpenAPI), err)
+			log.Printf("[ERROR] Failed app unmarshal during auto-download. Success: %#v. Applength: %d: %s", app.Success, len(app.OpenAPI), err)
 			resp.WriteHeader(401)
 			resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`))
 			return
@@ -966,37 +797,29 @@ func activateWorkflowAppDocker(resp http.ResponseWriter, request *http.Request)
 		}
 	}
 
-	if app.Sharing || app.Public {
+	// Just making sure it's being built properly
-		org, err := shuffle.GetOrg(ctx, user.ActiveOrg.Id)
+	if app == nil {
-		if err == nil {
+		log.Printf("[WARNING] App is nil. This shouldn't happen. Starting remote download(3)")
-			added := false
+		handleRemoteDownloadApp(resp, ctx, user, fileId)
-			if !shuffle.ArrayContains(org.ActiveApps, app.ID) {
-				org.ActiveApps = append(org.ActiveApps, app.ID)
-				added = true
-			}
-
-			if added {
-				err = shuffle.SetOrg(ctx, *org, org.Id)
-				if err != nil {
-					log.Printf("[WARNING] Failed setting org when autoadding apps on save: %s", err)
-				} else {
-					log.Printf("[INFO] Added public app %s (%s) to org %s (%s)", app.Name, app.ID, user.ActiveOrg.Name, user.ActiveOrg.Id)
-					cacheKey := fmt.Sprintf("apps_%s", user.Id)
-					shuffle.DeleteCache(ctx, cacheKey)
-				}
-			}
-		}
-	} else {
-		log.Printf("[WARNING] User is trying to activate %s which is NOT public", app.Name)
-		resp.WriteHeader(401)
-		resp.Write([]byte(`{"success": false}`))
 		return
 	}
 
-	log.Printf("[DEBUG] App %s (%s) activated for org %s by user %s", app.Name, app.ID, user.ActiveOrg.Id, user.Username)
+	// Check the app.. hmm
+	openApiApp, err := shuffle.GetOpenApiDatastore(ctx, app.ID)
+	if err != nil {
+		log.Printf("[WARNING] Error getting app %s (openapi config): %s", app.ID, err)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false, "reason": "Couldn't find app OpenAPI"}`))
+		return
+	}
 
-	// If onprem, it should autobuild the container(s) from here
+	log.Printf("[INFO] User %s (%s) is activating %s. Public: %t, Shared: %t", user.Username, user.Id, app.Name, app.Public, app.Sharing)
+	buildSwaggerApp(resp, []byte(openApiApp.Body), user, true)
 
-	resp.WriteHeader(200)
+	//app.Active = true
-	resp.Write([]byte(`{"success": true}`))
+	//app.Generated = true
+	//app, err := shuffle.SetApp(ctx, app)
+
+	//resp.WriteHeader(200)
+	//resp.Write([]byte(`{"success": true}`))
 }

backend/go-app/go.mod

@@ -1,100 +1,105 @@
-module main
+module shuffle-shared
+
+replace github.com/shuffle/shuffle-shared => ../../../../git/shuffle-shared
 
 go 1.19
 
-//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared
-
 require (
-	cloud.google.com/go/datastore v1.10.0
+	cloud.google.com/go/datastore v1.11.0
-	cloud.google.com/go/pubsub v1.28.0
+	cloud.google.com/go/pubsub v1.31.0
-	cloud.google.com/go/storage v1.28.1
+	cloud.google.com/go/storage v1.30.1
 	github.com/basgys/goxml2json v1.1.0
 	github.com/carlescere/scheduler v0.0.0-20170109141437-ee74d2f83d82
-	github.com/docker/docker v20.10.21+incompatible
+	github.com/docker/docker v24.0.2+incompatible
 	github.com/frikky/kin-openapi v0.42.0
-	github.com/fsouza/go-dockerclient v1.9.0
+	github.com/fsouza/go-dockerclient v1.9.7
 	github.com/ghodss/yaml v1.0.0
-	github.com/go-git/go-billy/v5 v5.3.1
+	github.com/go-git/go-billy/v5 v5.4.1
-	github.com/go-git/go-git/v5 v5.5.0
+	github.com/go-git/go-git/v5 v5.7.0
 	github.com/gorilla/mux v1.8.0
 	github.com/h2non/filetype v1.1.3
 	github.com/satori/go.uuid v1.2.0
-	github.com/shuffle/shuffle-shared v0.3.35
+	github.com/shuffle/shuffle-shared v0.4.19
-	golang.org/x/crypto v0.3.0
+	golang.org/x/crypto v0.9.0
-	google.golang.org/api v0.103.0
+	google.golang.org/api v0.125.0
 	google.golang.org/appengine v1.6.7
-	google.golang.org/grpc v1.51.0
+	google.golang.org/grpc v1.55.0
 	gopkg.in/src-d/go-git.v4 v4.13.1
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
-	cloud.google.com/go v0.105.0 // indirect
+	cloud.google.com/go v0.110.2 // indirect
-	cloud.google.com/go/compute v1.13.0 // indirect
+	cloud.google.com/go/compute v1.19.3 // indirect
-	cloud.google.com/go/compute/metadata v0.2.1 // indirect
+	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	cloud.google.com/go/iam v0.7.0 // indirect
+	cloud.google.com/go/iam v1.0.1 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/Microsoft/go-winio v0.6.0 // indirect
-	github.com/Microsoft/hcsshim v0.9.3 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 // indirect
-	github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 // indirect
+	github.com/acomagu/bufpipe v1.0.4 // indirect
-	github.com/acomagu/bufpipe v1.0.3 // indirect
 	github.com/adrg/strutil v0.2.3 // indirect
 	github.com/algolia/algoliasearch-client-go/v3 v3.18.1 // indirect
+	github.com/bitly/go-simplejson v0.5.0 // indirect
 	github.com/bradfitz/gomemcache v0.0.0-20221031212613-62deef7fc822 // indirect
 	github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 // indirect
-	github.com/cloudflare/circl v1.1.0 // indirect
+	github.com/cloudflare/circl v1.3.3 // indirect
-	github.com/containerd/cgroups v1.0.3 // indirect
+	github.com/containerd/containerd v1.6.18 // indirect
-	github.com/containerd/containerd v1.6.6 // indirect
+	github.com/docker/distribution v2.8.2+incompatible // indirect
-	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
-	github.com/frikky/go-elasticsearch/v8 v8.13.1 // indirect
+	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
-	github.com/go-git/gcfg v1.5.0 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/swag v0.19.5 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/go-github/v28 v28.1.1 // indirect
 	github.com/google/go-querystring v1.0.0 // indirect
+	github.com/google/s2a-go v0.1.4 // indirect
 	github.com/google/uuid v1.3.0 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
-	github.com/googleapis/gax-go/v2 v2.7.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.10.0 // indirect
-	github.com/imdario/mergo v0.3.13 // indirect
+	github.com/imdario/mergo v0.3.15 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
-	github.com/mailru/easyjson v0.7.0 // indirect
+	github.com/klauspost/compress v1.11.13 // indirect
-	github.com/moby/sys/mount v0.3.3 // indirect
+	github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e // indirect
-	github.com/moby/sys/mountinfo v0.6.2 // indirect
+	github.com/moby/patternmatcher v0.5.0 // indirect
+	github.com/moby/sys/sequential v0.5.0 // indirect
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
-	github.com/opencontainers/runc v1.1.2 // indirect
+	github.com/opencontainers/runc v1.1.5 // indirect
+	github.com/opensearch-project/opensearch-go v1.1.0 // indirect
+	github.com/opensearch-project/opensearch-go/v2 v2.3.0 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
-	github.com/pjbgf/sha1cd v0.2.0 // indirect
+	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/sergi/go-diff v1.1.0 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
-	github.com/skeema/knownhosts v1.1.0 // indirect
+	github.com/skeema/knownhosts v1.1.1 // indirect
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect
 	github.com/src-d/gcfg v1.4.0 // indirect
-	github.com/xanzy/ssh-agent v0.3.2 // indirect
+	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go4.org v0.0.0-20201209231011-d4a079459e60 // indirect
-	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
+	golang.org/x/mod v0.8.0 // indirect
-	golang.org/x/net v0.2.0 // indirect
+	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect
+	golang.org/x/oauth2 v0.8.0 // indirect
-	golang.org/x/sync v0.1.0 // indirect
+	golang.org/x/sync v0.2.0 // indirect
-	golang.org/x/sys v0.2.0 // indirect
+	golang.org/x/sys v0.8.0 // indirect
-	golang.org/x/text v0.4.0 // indirect
+	golang.org/x/text v0.9.0 // indirect
-	golang.org/x/tools v0.1.12 // indirect
+	golang.org/x/tools v0.6.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-	google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd // indirect
+	google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
+
 )

backend/go-app/main.go

@@ -13,7 +13,7 @@ import (
 
 	//"crypto/tls"
 	//"crypto/x509"
-	"encoding/base64"
+	//"encoding/base64"
 	"encoding/hex"
 	"encoding/json"
 	"errors"
@@ -21,7 +21,6 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
-	"math/rand"
 	"net/http"
 	"net/url"
 	"os"
@@ -38,9 +37,6 @@ import (
 	"cloud.google.com/go/storage"
 	"google.golang.org/appengine/mail"
 
-	//"github.com/elastic/go-elasticsearch/v7"
-	//"github.com/elastic/go-elasticsearch/v8/esapi"
-
 	"github.com/frikky/kin-openapi/openapi2"
 	"github.com/frikky/kin-openapi/openapi2conv"
 	"github.com/frikky/kin-openapi/openapi3"
@@ -82,18 +78,14 @@ import (
 var gceProject = "shuffle"
 var bucketName = "shuffler.appspot.com"
 var baseAppPath = "/home/frikky/git/shaffuru/tmp/apps"
+
 var baseDockerName = "frikky/shuffle"
 var registryName = "registry.hub.docker.com"
 var runningEnvironment = "onprem"
 
 var syncUrl = "https://shuffler.io"
-
-//var syncUrl = "http://localhost:5002"
 var syncSubUrl = "https://shuffler.io"
 
-//var syncUrl = "http://localhost:5002"
-//var syncSubUrl = "https://050196912a9d.ngrok.io"
-
 var dbclient *datastore.Client
 
 type Userapi struct {
@@ -701,9 +693,9 @@ func createNewUser(username, password, role, apikey string, org shuffle.OrgMini)
 		for tutorialIndex, tutorial := range neworg.Tutorials {
 			if tutorial.Name == "Invite teammates" {
 				neworg.Tutorials[tutorialIndex].Description = fmt.Sprintf("%d users are in your org. Org name and Image change next.", len(neworg.Users))
-				if len(neworg.Users) > 0 {
+				if len(neworg.Users) > 1 {
 					neworg.Tutorials[tutorialIndex].Done = true
-					neworg.Tutorials[tutorialIndex].Link = "/admin"
+					neworg.Tutorials[tutorialIndex].Link = "/admin?tab=users"
 				}
 
 				break
@@ -796,7 +788,7 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) {
 				CloudSync: false,
 			}
 
-			err = shuffle.SetOrg(ctx, newOrg, orgId)
+			err = shuffle.SetOrg(ctx, newOrg, newOrg.Id)
 			if err != nil {
 				log.Printf("[WARNING] Failed setting init organization: %s", err)
 			} else {
@@ -948,48 +940,104 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) {
 	})
 
 	// Updating user info if there's something wrong
-	if (len(userInfo.ActiveOrg.Name) == 0 || len(userInfo.ActiveOrg.Id) == 0) && len(userInfo.Orgs) > 0 {
+	if len(userInfo.ActiveOrg.Name) == 0 || len(userInfo.ActiveOrg.Id) == 0 {
-		_, err := shuffle.GetOrg(ctx, userInfo.Orgs[0])
+		if len(userInfo.Orgs) == 0 || (len(userInfo.Orgs) > 0 && userInfo.Orgs[0] == "") {
-		if err != nil {
 			orgs, err := shuffle.GetAllOrgs(ctx)
-			if err == nil {
+			log.Printf("[INFO] Fixing organization for user %s (%s). Found orgs: %d", userInfo.Username, userInfo.Id, len(orgs))
-				newStringOrgs := []string{}
+			if err == nil && len(orgs) > 0 {
-				newOrgs := []shuffle.Org{}
 				for _, org := range orgs {
-					if strings.ToLower(org.Name) == strings.ToLower(userInfo.Orgs[0]) {
+					if len(org.Id) == 0 {
-						newOrgs = append(newOrgs, org)
+						continue
-						newStringOrgs = append(newStringOrgs, org.Id)
 					}
-				}
 
-				if len(newOrgs) > 0 {
+					// Prolly some way here to jump into another org
+					// when you have access to the DB
 					userInfo.ActiveOrg = shuffle.OrgMini{
-						Id:   newOrgs[0].Id,
+						Name: org.Name,
-						Name: newOrgs[0].Name,
+						Id:   org.Id,
-					}
+						Role: "admin",
-
-					userInfo.Orgs = newStringOrgs
-
-					err = shuffle.SetUser(ctx, &userInfo, true)
-					if err != nil {
-						log.Printf("Error patching User for activeOrg: %s", err)
-					} else {
-						log.Printf("Updated the users' org")
 					}
+					userInfo.Orgs = []string{org.Id}
+					break
 				}
-			} else {
-				log.Printf("Failed getting orgs for user. Major issue.: %s", err)
 			}
 
-		} else {
+			// Make a new one in case we couldn't find one
-			// 1. Check if the org exists by ID
+			if len(userInfo.ActiveOrg.Id) == 0 {
-			// 2. if it does, overwrite user
+				orgSetupName := "default"
-			userInfo.ActiveOrg = shuffle.OrgMini{
+				orgId := uuid.NewV4().String()
-				Id: userInfo.Orgs[0],
+				newOrg := shuffle.Org{
+					Name:      orgSetupName,
+					Id:        orgId,
+					Org:       orgSetupName,
+					Users:     []shuffle.User{},
+					Roles:     []string{"admin", "user"},
+					CloudSync: false,
+				}
+
+				err = shuffle.SetOrg(ctx, newOrg, newOrg.Id)
+				if err == nil {
+					userInfo.ActiveOrg = shuffle.OrgMini{
+						Name: newOrg.Name,
+						Id:   newOrg.Id,
+						Role: "admin",
+					}
+					userInfo.Orgs = []string{newOrg.Id}
+				} else {
+					log.Printf("[WARNING] Failed to set new org: %s", err)
+				}
 			}
+
+			// Set user
 			err = shuffle.SetUser(ctx, &userInfo, true)
 			if err != nil {
-				log.Printf("[INFO] Error patching User for activeOrg: %s", err)
+				log.Printf("[WARNING] Failed fixing org info for user %s (%s)", userInfo.Username, userInfo.Id)
+			} else {
+				log.Printf("[INFO] Set organization for %s (%s) to be %s (%s)", userInfo.Username, userInfo.Id, userInfo.ActiveOrg.Name, userInfo.ActiveOrg.Id)
+			}
+		} else if len(userInfo.Orgs) > 0 && userInfo.Orgs[0] != "" {
+			_, err := shuffle.GetOrg(ctx, userInfo.Orgs[0])
+			if err != nil {
+				orgs, err := shuffle.GetAllOrgs(ctx)
+				if err == nil {
+					newStringOrgs := []string{}
+					newOrgs := []shuffle.Org{}
+					for _, org := range orgs {
+						if strings.ToLower(org.Name) == strings.ToLower(userInfo.Orgs[0]) {
+							newOrgs = append(newOrgs, org)
+							newStringOrgs = append(newStringOrgs, org.Id)
+						}
+					}
+
+					if len(newOrgs) > 0 {
+						userInfo.ActiveOrg = shuffle.OrgMini{
+							Id:   newOrgs[0].Id,
+							Name: newOrgs[0].Name,
+						}
+
+						userInfo.Orgs = newStringOrgs
+
+						err = shuffle.SetUser(ctx, &userInfo, true)
+						if err != nil {
+							log.Printf("Error patching User for activeOrg: %s", err)
+						} else {
+							log.Printf("Updated the users' org")
+						}
+					}
+				} else {
+					log.Printf("Failed getting orgs for user. Major issue.: %s", err)
+				}
+
+			} else {
+				// 1. Check if the org exists by ID
+				// 2. if it does, overwrite user
+				userInfo.ActiveOrg = shuffle.OrgMini{
+					Id: userInfo.Orgs[0],
+				}
+				err = shuffle.SetUser(ctx, &userInfo, true)
+				if err != nil {
+					log.Printf("[INFO] Error patching User for activeOrg: %s", err)
+				}
 			}
 		}
 	}
@@ -1052,15 +1100,18 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) {
 		chatDisabled = true
 	}
 
+	userOrgs = shuffle.SortOrgList(userOrgs)
 	orgPriorities := org.Priorities
-	if len(org.Priorities) < 5 {
+	if len(org.Priorities) < 10 {
-		log.Printf("[WARNING] Should find and add priorities as length is less than 5 for org %s", userInfo.ActiveOrg.Id)
+		log.Printf("[WARNING] Should find and add priorities as length is less than 10 for org %s", userInfo.ActiveOrg.Id)
 		newPriorities, err := shuffle.GetPriorities(ctx, userInfo, org)
 		if err != nil {
 			log.Printf("[WARNING] Failed getting new priorities for org %s: %s", org.Id, err)
 			//orgPriorities = []shuffle.Priority{}
 		} else {
 			orgPriorities = newPriorities
+
+			// A way to manage them over time
 		}
 	}
 
@@ -1225,15 +1276,6 @@ func handleContact(resp http.ResponseWriter, request *http.Request) {
 	resp.Write([]byte(fmt.Sprintf(`{"success": true, "message": "Thanks for reaching out. We will contact you soon!"}`)))
 }
 
-func verifier() (*shuffle.CodeVerifier, error) {
-	r := rand.New(rand.NewSource(time.Now().UnixNano()))
-	b := make([]byte, 32, 32)
-	for i := 0; i < 32; i++ {
-		b[i] = byte(r.Intn(255))
-	}
-	return shuffle.CreateCodeVerifierFromBytes(b)
-}
-
 func checkAdminLogin(resp http.ResponseWriter, request *http.Request) {
 	cors := shuffle.HandleCors(resp, request)
 	if cors {
@@ -1278,51 +1320,7 @@ func checkAdminLogin(resp http.ResponseWriter, request *http.Request) {
 
 		// Should run calculations
 		if len(org.SSOConfig.OpenIdAuthorization) > 0 {
-			baseSSOUrl = org.SSOConfig.OpenIdAuthorization
+			baseSSOUrl = shuffle.GetOpenIdUrl(request, *org)
-
-			codeChallenge := uuid.NewV4().String()
-			//h.Write([]byte(v.Value))
-			verifier, verifiererr := verifier()
-			if verifiererr == nil {
-				codeChallenge = verifier.Value
-			}
-
-			//log.Printf("[DEBUG] Got challenge value %s (pre state)", codeChallenge)
-
-			// https://192.168.55.222:3443/api/v1/login_openid
-			//location := strings.Split(request.URL.String(), "/")
-			//redirectUrl := url.QueryEscape("http://localhost:5001/api/v1/login_openid")
-			redirectUrl := url.QueryEscape(fmt.Sprintf("http://%s/api/v1/login_openid", request.Host))
-			if strings.Contains(request.Host, "shuffle-backend") && !strings.Contains(os.Getenv("BASE_URL"), "shuffle-backend") {
-				redirectUrl = url.QueryEscape(fmt.Sprintf("%s/api/v1/login_openid", os.Getenv("BASE_URL")))
-			}
-
-			if len(os.Getenv("SSO_REDIRECT_URL")) > 0 {
-				redirectUrl = url.QueryEscape(fmt.Sprintf("%s/api/v1/login_openid", os.Getenv("SSO_REDIRECT_URL")))
-			}
-
-			state := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("org=%s&challenge=%s&redirect=%s", org.Id, codeChallenge, redirectUrl)))
-
-			// has to happen after initial value is stored
-			if verifiererr == nil {
-				codeChallenge = verifier.CodeChallengeS256()
-			}
-
-			//log.Printf("[DEBUG] Got challenge value %s (POST state)", codeChallenge)
-
-			if len(org.SSOConfig.OpenIdClientSecret) > 0 {
-
-				//baseSSOUrl += fmt.Sprintf("?client_id=%s&response_type=code&scope=openid&redirect_uri=%s&state=%s&client_secret=%s", org.SSOConfig.OpenIdClientId, redirectUrl, state, org.SSOConfig.OpenIdClientSecret)
-				state := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("org=%s&redirect=%s&challenge=%s", org.Id, redirectUrl, org.SSOConfig.OpenIdClientSecret)))
-				log.Printf("URL: %s", redirectUrl)
-
-				baseSSOUrl += fmt.Sprintf("?client_id=%s&response_type=id_token&scope=openid&redirect_uri=%s&state=%s&response_mode=form_post&nonce=%s", org.SSOConfig.OpenIdClientId, redirectUrl, state, state)
-				//baseSSOUrl += fmt.Sprintf("&client_secret=%s", org.SSOConfig.OpenIdClientSecret)
-				log.Printf("[DEBUG] Found OpenID url (client secret). Extra redirect check: %s - %s", request.URL.String(), baseSSOUrl)
-			} else {
-				log.Printf("[DEBUG] Found OpenID url (PKCE!!). Extra redirect check: %s", request.URL.String())
-				baseSSOUrl += fmt.Sprintf("?client_id=%s&response_type=code&scope=openid&redirect_uri=%s&state=%s&code_challenge_method=S256&code_challenge=%s", org.SSOConfig.OpenIdClientId, redirectUrl, state, codeChallenge)
-			}
 
 			break
 		}
@@ -1339,156 +1337,6 @@ func checkAdminLogin(resp http.ResponseWriter, request *http.Request) {
 	resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "redirect", "sso_url": "%s"}`, baseSSOUrl)))
 }
 
-func handleLogin(resp http.ResponseWriter, request *http.Request) {
-	cors := shuffle.HandleCors(resp, request)
-	if cors {
-		return
-	}
-
-	// Gets a struct of Username, password
-	data, err := shuffle.ParseLoginParameters(resp, request)
-	if err != nil {
-		resp.WriteHeader(401)
-		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
-		return
-	}
-
-	log.Printf("[INFO] Handling login of %s", data.Username)
-
-	err = checkUsername(data.Username)
-	if err != nil {
-		resp.WriteHeader(401)
-		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
-		return
-	}
-
-	ctx := context.Background()
-	log.Printf("[INFO] Login Username: %s", data.Username)
-	users, err := shuffle.FindUser(ctx, strings.ToLower(strings.TrimSpace(data.Username)))
-	if err != nil && len(users) == 0 {
-		log.Printf("[WARNING] Failed getting user %s: %s", data.Username, err)
-		resp.WriteHeader(401)
-		resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`))
-		return
-	}
-
-	if len(users) != 1 {
-		log.Printf(`Found multiple or no users with the same username: %s: %d`, data.Username, len(users))
-		resp.WriteHeader(401)
-		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error: %d users with username %s"}`, len(users), data.Username)))
-		return
-	}
-
-	Userdata := users[0]
-
-	err = bcrypt.CompareHashAndPassword([]byte(Userdata.Password), []byte(data.Password))
-	if err != nil {
-		log.Printf("Password for %s is incorrect: %s", data.Username, err)
-		resp.WriteHeader(401)
-		resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`))
-		return
-	}
-
-	if !Userdata.Active {
-		log.Printf("%s is not active, but tried to login. Error: %v", data.Username, err)
-		resp.WriteHeader(401)
-		resp.Write([]byte(`{"success": false, "reason": "This user is deactivated"}`))
-		return
-	}
-
-	tutorialsFinished := []shuffle.Tutorial{}
-	for _, tutorial := range Userdata.PersonalInfo.Tutorials {
-		tutorialsFinished = append(tutorialsFinished, shuffle.Tutorial{
-			Name: tutorial,
-		})
-	}
-	returnValue := shuffle.HandleInfo{
-		Success:   true,
-		Tutorials: tutorialsFinished,
-	}
-
-	loginData := `{"success": true}`
-	newData, err := json.Marshal(returnValue)
-	if err == nil {
-		loginData = string(newData)
-	}
-
-	if len(Userdata.Session) != 0 {
-		log.Println("[INFO] User session already exists - resetting it")
-		expiration := time.Now().Add(3600 * time.Second)
-
-		http.SetCookie(resp, &http.Cookie{
-			Name:    "session_token",
-			Value:   Userdata.Session,
-			Expires: expiration,
-		})
-
-		returnValue.Cookies = append(returnValue.Cookies, shuffle.SessionCookie{
-			Key:        "session_token",
-			Value:      Userdata.Session,
-			Expiration: expiration.Unix(),
-		})
-
-		loginData = fmt.Sprintf(`{"success": true, "cookies": [{"key": "session_token", "value": "%s", "expiration": %d}]}`, Userdata.Session, expiration.Unix())
-		newData, err := json.Marshal(returnValue)
-		if err == nil {
-			loginData = string(newData)
-		}
-		//log.Printf("SESSION LENGTH MORE THAN 0 IN LOGIN: %s", Userdata.Session)
-
-		err = shuffle.SetSession(ctx, Userdata, Userdata.Session)
-		if err != nil {
-			log.Printf("Error adding session to database: %s", err)
-		}
-
-		resp.WriteHeader(200)
-		resp.Write([]byte(loginData))
-		return
-	} else {
-		log.Printf("[INFO] User session is empty - create one!")
-
-		sessionToken := uuid.NewV4().String()
-		expiration := time.Now().Add(3600 * time.Second)
-		http.SetCookie(resp, &http.Cookie{
-			Name:    "session_token",
-			Value:   sessionToken,
-			Expires: expiration,
-		})
-
-		// ADD TO DATABASE
-		err = shuffle.SetSession(ctx, Userdata, sessionToken)
-		if err != nil {
-			log.Printf("Error adding session to database: %s", err)
-		}
-
-		Userdata.Session = sessionToken
-		err = shuffle.SetUser(ctx, &Userdata, true)
-		if err != nil {
-			log.Printf("Failed updating user when setting session: %s", err)
-			resp.WriteHeader(500)
-			resp.Write([]byte(`{"success": false}`))
-			return
-		}
-
-		returnValue.Cookies = append(returnValue.Cookies, shuffle.SessionCookie{
-			Key:        "session_token",
-			Value:      sessionToken,
-			Expiration: expiration.Unix(),
-		})
-
-		loginData = fmt.Sprintf(`{"success": true, "cookies": [{"key": "session_token", "value": "%s", "expiration": %d}]}`, sessionToken, expiration.Unix())
-		newData, err := json.Marshal(returnValue)
-		if err == nil {
-			loginData = string(newData)
-		}
-	}
-
-	log.Printf("[INFO] %s SUCCESSFULLY LOGGED IN with session %s", data.Username, Userdata.Session)
-
-	resp.WriteHeader(200)
-	resp.Write([]byte(loginData))
-}
-
 func fixOrgUser(ctx context.Context, org *shuffle.Org) *shuffle.Org {
 	//found := false
 	//for _, id := range user.Orgs {
@@ -1590,7 +1438,7 @@ func fixUserOrg(ctx context.Context, user *shuffle.User) *shuffle.User {
 			org.Users = append(org.Users, *user)
 		}
 
-		err = shuffle.SetOrg(ctx, *org, orgId)
+		err = shuffle.SetOrg(ctx, *org, org.Id)
 		if err != nil {
 			log.Printf("Failed setting org %s", orgId)
 		}
@@ -2211,6 +2059,15 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) {
 		}
 	}
 
+	// Find user agent header
+	userAgent := request.Header.Get("User-Agent")
+	if strings.Contains(strings.ToLower(userAgent), "microsoftpreview") || strings.Contains(strings.ToLower(userAgent), "googlebot") {
+		log.Printf("[AUDIT] Blocking googlebot and microsoftbot for webhooks. UA: '%s'", userAgent)
+		resp.WriteHeader(400)
+		resp.Write([]byte(`{"success": false, "reason": "Google/Microsoft preview bots not allowed. Please change the useragent."}`))
+		return
+	}
+
 	// ID: webhook_<UID>
 	if len(hookId) != 44 {
 		log.Printf("[INFO] Couldn't handle hookId. Too short in webhook: %d", len(hookId))
@@ -2296,7 +2153,7 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) {
 		if err == nil {
 			for _, branch := range workflow.Branches {
 				if branch.SourceID == hook.Id {
-					log.Printf("[INFO] Found ID %s for hook", hook.Id)
+					log.Printf("[DEBUG] Found ID %s for hook", hook.Id)
 					if branch.DestinationID != hook.Start {
 						newBody.Start = branch.DestinationID
 						break
@@ -2308,57 +2165,76 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) {
 
 	b, err := json.Marshal(newBody)
 	if err != nil {
-		log.Printf("Failed newBody marshaling: %s", err)
+		log.Printf("[ERROR] Failed newBody marshaling for webhook: %s", err)
-		resp.WriteHeader(401)
+		resp.WriteHeader(500)
 		resp.Write([]byte(`{"success": false}`))
 		return
 	}
 
+	// Should wrap the response input Body as well?
 	for _, item := range hook.Workflows {
-		//log.Printf("Running webhook for workflow %s with startnode %s", item, hook.Start)
+		log.Printf("[INFO] Running webhook for workflow %s with startnode %s", item, hook.Start)
+
+		// This ID is empty to force it to get the webhook within the execution
 		workflow := shuffle.Workflow{
 			ID: "",
 		}
 
-		//parsedBody := string(body)
+		if len(hook.Start) == 0 {
-		//parsedBody = strings.Replace(parsedBody, "\"", "\\\"", -1)
+			log.Printf("[WARNING] No start node for hook %s - running with workflow default.", hook.Id)
-		//if len(parsedBody) > 0 {
+			//bodyWrapper = string(parsedBody)
-		//	if string(parsedBody[0]) == `"` && string(parsedBody[len(parsedBody)-1]) == "\"" {
+		}
-		//		parsedBody = parsedBody[1 : len(parsedBody)-1]
-		//	}
-		//}
-
-		//bodyWrapper := fmt.Sprintf(`{"start": "%s", "execution_source": "webhook", "execution_argument": "%s"}`, hook.Start, string(parsedBody))
-		//if len(hook.Start) == 0 {
-		//	log.Printf("No start node for hook %s - running with workflow default.", hook.Id)
-		//	bodyWrapper = string(parsedBody)
-		//}
 
 		newRequest := &http.Request{
 			URL:    &url.URL{},
 			Method: "POST",
 			Body:   ioutil.NopCloser(bytes.NewReader(b)),
 		}
-		//start, startok := request.URL.Query()["start"]
 
 		// OrgId: activeOrgs[0].Id,
 		workflowExecution, executionResp, err := handleExecution(item, workflow, newRequest, hook.OrgId)
-		if err == nil {
-			/*
-				err = increaseStatisticsField(ctx, "total_webhooks_ran", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg)
-				if err != nil {
-					log.Printf("Failed to increase total apps loaded stats: %s", err)
-				}
-			*/
 
+		if err == nil {
+			if hook.Version == "v2" {
+				timeout := 15
+				//if hook.VersionTimeout != 0 {
+				//	timeout = hook.VersionTimeout
+				//}
+
+				log.Printf("[DEBUG] Waiting for Webhook response from %s for max %d seconds! Checking every 1 second. Hook ID: %s", workflowExecution.ExecutionId, timeout, hook.Id)
+				// Try every second for 15 seconds
+				for i := 0; i < timeout; i++ {
+					time.Sleep(1 * time.Second)
+
+					newExec, err := shuffle.GetWorkflowExecution(ctx, workflowExecution.ExecutionId)
+					if err != nil {
+						log.Printf("[ERROR] Failed to get workflow execution: %s", err)
+						break
+					}
+
+					if newExec.Status != "EXECUTING" {
+						log.Printf("[INFO] Got response from webhook v2 of length '%d' <- %s", len(newExec.Result), newExec.ExecutionId)
+						resp.WriteHeader(200)
+						resp.Write([]byte(newExec.Result))
+						return
+					}
+				}
+			}
+
+			// Fallback
 			resp.WriteHeader(200)
-			resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s"}`, workflowExecution.ExecutionId)))
+			if len(hook.CustomResponse) > 0 {
+				resp.Write([]byte(hook.CustomResponse))
+			} else {
+				resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s"}`, workflowExecution.ExecutionId)))
+			}
 			return
 		}
 
 		resp.WriteHeader(500)
 		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, executionResp)))
 	}
+
 }
 
 func executeCloudAction(action shuffle.CloudSyncJob, apikey string) error {
@@ -2408,6 +2284,8 @@ func executeCloudAction(action shuffle.CloudSyncJob, apikey string) error {
 		return errors.New(fmt.Sprintf("Cloud error from Shuffler: %s", responseData.Reason))
 	}
 
+	log.Printf("[INFO] Cloud action executed successfully for '%s'", action.Action)
+
 	return nil
 }
 
@@ -3688,7 +3566,7 @@ func handleCloudJob(job shuffle.CloudSyncJob) error {
 
 	} else if job.Type == "schedule" {
 		if job.Action == "execute" {
-			log.Printf("Should handle schedule for workflow %s with start node %s and data %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem)
+			log.Printf("[INFO] Should handle schedule for workflow %s with start node %s and data %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem)
 			err := handleCloudExecutionOnprem(job.PrimaryItemId, job.SecondaryItem, "schedule", job.ThirdItem)
 			if err != nil {
 				log.Printf("[INFO] Failed executing workflow from cloud schedule: %s", err)
@@ -3698,7 +3576,7 @@ func handleCloudJob(job shuffle.CloudSyncJob) error {
 		}
 	} else if job.Type == "email_trigger" {
 		if job.Action == "execute" {
-			log.Printf("Should handle email for workflow %s with start node %s and data %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem)
+			log.Printf("[INFO] Should handle email for workflow %s with start node %s and data %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem)
 			err := handleCloudExecutionOnprem(job.PrimaryItemId, job.SecondaryItem, "email_trigger", job.ThirdItem)
 			if err != nil {
 				log.Printf("Failed executing workflow from email trigger: %s", err)
@@ -3709,7 +3587,7 @@ func handleCloudJob(job shuffle.CloudSyncJob) error {
 
 	} else if job.Type == "user_input" {
 		if job.Action == "continue" {
-			log.Printf("Should handle user_input CONTINUE for workflow %s with start node %s and execution ID %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem)
+			log.Printf("[INFO] Should handle user_input CONTINUE for workflow %s with start node %s and execution ID %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem)
 			// FIXME: Handle authorization
 			ctx := context.Background()
 			workflowExecution, err := shuffle.GetWorkflowExecution(ctx, job.ThirdItem)
@@ -3807,47 +3685,50 @@ func remoteOrgJobController(org shuffle.Org, body []byte) error {
 	if !responseData.Success {
 		log.Printf("[WARNING] Should stop org job controller because no success?")
 
-		if strings.Contains(responseData.Reason, "Bad apikey") || strings.Contains(responseData.Reason, "Error getting the organization") || strings.Contains(responseData.Reason, "Organization isn't syncing") {
+		if strings.Contains(strings.ToLower(responseData.Reason), "bad apikey") || strings.Contains(responseData.Reason, "Error getting the organization") || strings.Contains(responseData.Reason, "Organization isn't syncing") {
 			log.Printf("[WARNING] Remote error; Bad apikey or org error. Stopping sync for org: %s", responseData.Reason)
 
 			if value, exists := scheduledOrgs[org.Id]; exists {
 				// Looks like this does the trick? Hurr
-				log.Printf("[WARNING] STOPPING ORG SCHEDULE for: %s", org.Id)
+				log.Printf("[INFO] STOPPING ORG SCHEDULE for: %s", org.Id)
-
 				value.Lock()
-				org, err := shuffle.GetOrg(ctx, org.Id)
-				if err != nil {
-					log.Printf("[WARNING] Failed finding org %s: %s", org.Id, err)
-					return err
-				}
-
-				org.SyncConfig.Interval = 0
-				org.SyncConfig.Apikey = ""
-				org.CloudSync = false
-
-				// Just in case
-				org, err = handleStopCloudSync(syncUrl, *org)
-
-				startDate := time.Now().Unix()
-				org.SyncFeatures.Webhook = shuffle.SyncData{Active: false, Type: "trigger", Name: "Webhook", StartDate: startDate}
-				org.SyncFeatures.UserInput = shuffle.SyncData{Active: false, Type: "trigger", Name: "User Input", StartDate: startDate}
-				org.SyncFeatures.EmailTrigger = shuffle.SyncData{Active: false, Type: "action", Name: "Email Trigger", StartDate: startDate}
-				org.SyncFeatures.Schedules = shuffle.SyncData{Active: false, Type: "trigger", Name: "Schedule", StartDate: startDate, Limit: 0}
-				org.SyncFeatures.SendMail = shuffle.SyncData{Active: false, Type: "action", Name: "Send Email", StartDate: startDate, Limit: 0}
-				org.SyncFeatures.SendSms = shuffle.SyncData{Active: false, Type: "action", Name: "Send SMS", StartDate: startDate, Limit: 0}
-				org.CloudSyncActive = false
-
-				err = shuffle.SetOrg(ctx, *org, org.Id)
-				if err != nil {
-					log.Printf("[WARNING] Failed setting organization when stopping sync: %s", err)
-				} else {
-					log.Printf("[INFO] Successfully STOPPED org cloud sync for %s (%s)", org.Name, org.Id)
-				}
-
-				return errors.New("Stopped schedule for org locally because of bad apikey.")
 			} else {
-				return errors.New(fmt.Sprintf("Failed finding the schedule for org %s (%s)", org.Name, org.Id))
+				log.Printf("[INFO] Failed finding the schedule for org %s (%s)", org.Name, org.Id)
 			}
+
+			org, err := shuffle.GetOrg(ctx, org.Id)
+			if err != nil {
+				log.Printf("[WARNING] Failed finding org %s: %s", org.Id, err)
+				return err
+			}
+
+			// Just in case
+			org, err = handleStopCloudSync(syncUrl, *org)
+			if err != nil {
+				log.Printf("[ERROR] Failed stopping cloud sync remotely: %s", err)
+			}
+
+			org.SyncConfig.Interval = 0
+			org.CloudSync = false
+			org.SyncConfig.Apikey = ""
+
+			startDate := time.Now().Unix()
+			org.SyncFeatures.Webhook = shuffle.SyncData{Active: false, Type: "trigger", Name: "Webhook", StartDate: startDate}
+			org.SyncFeatures.UserInput = shuffle.SyncData{Active: false, Type: "trigger", Name: "User Input", StartDate: startDate}
+			org.SyncFeatures.EmailTrigger = shuffle.SyncData{Active: false, Type: "action", Name: "Email Trigger", StartDate: startDate}
+			org.SyncFeatures.Schedules = shuffle.SyncData{Active: false, Type: "trigger", Name: "Schedule", StartDate: startDate, Limit: 0}
+			org.SyncFeatures.SendMail = shuffle.SyncData{Active: false, Type: "action", Name: "Send Email", StartDate: startDate, Limit: 0}
+			org.SyncFeatures.SendSms = shuffle.SyncData{Active: false, Type: "action", Name: "Send SMS", StartDate: startDate, Limit: 0}
+			org.CloudSyncActive = false
+
+			err = shuffle.SetOrg(ctx, *org, org.Id)
+			if err != nil {
+				log.Printf("[WARNING] Failed setting organization when stopping sync: %s", err)
+			} else {
+				log.Printf("[INFO] Successfully STOPPED org cloud sync for %s (%s)", org.Name, org.Id)
+			}
+
+			return nil
 		}
 
 		return errors.New("[ERROR] Remote job handler issues.")
@@ -3916,7 +3797,7 @@ func runInitCloudSetup() {
 }
 
 func runInitEs(ctx context.Context) {
-	log.Printf("[DEBUG] Starting INIT setup (ES)")
+	log.Printf("[DEBUG] Starting INIT setup for Elasticsearch/Opensearch")
 
 	httpProxy := os.Getenv("HTTP_PROXY")
 	if len(httpProxy) > 0 {
@@ -3933,7 +3814,7 @@ func runInitEs(ctx context.Context) {
 		log.Printf("[DEBUG] Setting default environment for org to %s", defaultEnv)
 	}
 
-	log.Printf("[DEBUG] Getting organizations")
+	log.Printf("[DEBUG] Getting organizations for Elasticsearch/Opensearch")
 	activeOrgs, err := shuffle.GetAllOrgs(ctx)
 
 	setUsers := false
@@ -4023,6 +3904,11 @@ func runInitEs(ctx context.Context) {
 		}
 	}
 
+	if strings.Contains(os.Getenv("SHUFFLE_OPENSEARCH_URL"), "https") {
+		log.Printf("[INFO] Waiting during init to make sure the opensearch instance is up and running with security features properly")
+		time.Sleep(30 * time.Second)
+	}
+
 	_ = setUsers
 	schedules, err := shuffle.GetAllSchedules(ctx, "ALL")
 	if err != nil {
@@ -4098,7 +3984,7 @@ func runInitEs(ctx context.Context) {
 				CloudSync: false,
 			}
 
-			err = shuffle.SetOrg(ctx, newOrg, orgId)
+			err = shuffle.SetOrg(ctx, newOrg, newOrg.Id)
 			setUsers := false
 			if err != nil {
 				log.Printf("[WARNING] Failed setting organization when creating original user: %s", err)
@@ -4150,8 +4036,13 @@ func runInitEs(ctx context.Context) {
 	}
 
 	for _, org := range activeOrgs {
+		if len(org.Id) == 0 {
+			log.Printf("[DEBUG] No ID found for org with name '%s'. Why was it made?", org.Name)
+			continue
+		}
+
 		if !org.CloudSync {
-			log.Printf("[INFO] Skipping org syncCheck for %s because sync isn't set (1).", org.Id)
+			log.Printf("[INFO] Skipping org syncCheck for '%s' because sync isn't set (1).", org.Id)
 			continue
 		}
 
@@ -4301,7 +4192,10 @@ func runInitEs(ctx context.Context) {
 
 		url := os.Getenv("SHUFFLE_APP_DOWNLOAD_LOCATION")
 		if len(url) == 0 {
-			url = "https://github.com/frikky/shuffle-apps"
+			log.Printf("[INFO] Skipping download of apps since no URL is set. Default would be https://github.com/shuffle/shuffle-apps")
+			url = "https://github.com/shuffle/shuffle-apps"
+			//url = ""
+			//return
 		}
 
 		username := os.Getenv("SHUFFLE_DOWNLOAD_AUTH_USERNAME")
@@ -4323,10 +4217,9 @@ func runInitEs(ctx context.Context) {
 			cloneOptions.ReferenceName = plumbing.ReferenceName(branch)
 		}
 
-		log.Printf("[DEBUG] Getting apps from %s", url)
+		log.Printf("[DEBUG] Getting apps from url '%s'", url)
 
 		r, err := git.Clone(storer, fs, cloneOptions)
-
 		if err != nil {
 			log.Printf("[WARNING] Failed loading repo into memory (init): %s", err)
 		}
@@ -4338,7 +4231,6 @@ func runInitEs(ctx context.Context) {
 		_ = r
 		//iterateAppGithubFolders(fs, dir, "", "testing")
 
-		// FIXME: Get all the apps?
 		_, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate)
 		if err != nil {
 			log.Printf("[WARNING] Error from app load in init: %s", err)
@@ -4353,7 +4245,7 @@ func runInitEs(ctx context.Context) {
 	}
 
 	log.Printf("[INFO] Downloading OpenAPI data for search - EXTRA APPS")
-	apis := "https://github.com/frikky/security-openapis"
+	apis := "https://github.com/shuffle/security-openapis"
 
 	// THis gets memory problems hahah
 	//apis := "https://github.com/APIs-guru/openapi-directory"
@@ -4388,7 +4280,7 @@ func runInit(ctx context.Context) {
 	//}
 	//log.Printf("[DEBUG] Finalized init statistics update")
 
-	log.Printf("[DEBUG] Starting INIT setup")
+	log.Printf("[DEBUG] Starting INIT setup (NOT Opensearch/Elasticsearch!)")
 	httpProxy := os.Getenv("HTTP_PROXY")
 	if len(httpProxy) > 0 {
 		log.Printf("Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy)
@@ -4460,11 +4352,11 @@ func runInit(ctx context.Context) {
 				CloudSync: false,
 			}
 
-			err = shuffle.SetOrg(ctx, newOrg, orgId)
+			err = shuffle.SetOrg(ctx, newOrg, newOrg.Id)
 			if err != nil {
-				log.Printf("Failed setting organization: %s", err)
+				log.Printf("[WARNING] Failed setting organization: %s", err)
 			} else {
-				log.Printf("Successfully created the default org!")
+				log.Printf("[WARNING] Successfully created the default org!")
 				setUsers = true
 			}
 		} else {
@@ -4961,7 +4853,7 @@ func runInit(ctx context.Context) {
 
 		url := os.Getenv("SHUFFLE_APP_DOWNLOAD_LOCATION")
 		if len(url) == 0 {
-			url = "https://github.com/frikky/shuffle-apps"
+			url = "https://github.com/shuffle/shuffle-apps"
 		}
 
 		username := os.Getenv("SHUFFLE_DOWNLOAD_AUTH_USERNAME")
@@ -4982,7 +4874,7 @@ func runInit(ctx context.Context) {
 			cloneOptions.ReferenceName = plumbing.ReferenceName(branch)
 		}
 
-		log.Printf("[DEBUG] Getting apps from %s", url)
+		log.Printf("[DEBUG] Getting apps from URL '%s'", url)
 
 		r, err := git.Clone(storer, fs, cloneOptions)
 
@@ -5012,7 +4904,7 @@ func runInit(ctx context.Context) {
 	}
 
 	log.Printf("[INFO] Downloading OpenAPI data for search - EXTRA APPS")
-	apis := "https://github.com/frikky/security-openapis"
+	apis := "https://github.com/shuffle/security-openapis"
 
 	// FIXME: This part gets memory problems. Fix in the future to load these apps too.
 	//apis := "https://github.com/APIs-guru/openapi-directory"
@@ -5139,7 +5031,7 @@ func handleStopCloudSync(syncUrl string, org shuffle.Org) (*shuffle.Org, error)
 	if err != nil {
 		return &org, err
 	}
-	log.Printf("Remote disable ret: %s", string(respBody))
+	log.Printf("[INFO] Remote disable ret: %s", string(respBody))
 
 	responseData := retStruct{}
 	err = json.Unmarshal(respBody, &responseData)
@@ -6018,7 +5910,7 @@ func initHandlers() {
 
 	// General - duplicates and old.
 	r.HandleFunc("/api/v1/getusers", shuffle.HandleGetUsers).Methods("GET", "OPTIONS")
-	r.HandleFunc("/api/v1/login", handleLogin).Methods("POST", "OPTIONS")
+	r.HandleFunc("/api/v1/login", shuffle.HandleLogin).Methods("POST", "OPTIONS")
 	r.HandleFunc("/api/v1/logout", shuffle.HandleLogout).Methods("POST", "OPTIONS")
 	r.HandleFunc("/api/v1/register", handleRegister).Methods("POST", "OPTIONS")
 	r.HandleFunc("/api/v1/checkusers", checkAdminLogin).Methods("GET", "OPTIONS")
@@ -6041,11 +5933,14 @@ func initHandlers() {
 	r.HandleFunc("/api/v1/streams/results", handleGetStreamResults).Methods("POST", "OPTIONS")
 
 	// Used by orborus
-	r.HandleFunc("/api/v1/workflows/queue", handleGetWorkflowqueue).Methods("GET")
+	r.HandleFunc("/api/v1/workflows/queue", handleGetWorkflowqueue).Methods("GET", "POST")
 	r.HandleFunc("/api/v1/workflows/queue/confirm", handleGetWorkflowqueueConfirm).Methods("POST")
 
 	// App specific
 	// From here down isnt checked for org specific
+	r.HandleFunc("/api/v1/apps/{key}/execute", executeSingleAction).Methods("POST", "OPTIONS")
+	r.HandleFunc("/api/v1/apps/categories", shuffle.GetActiveCategories).Methods("GET", "OPTIONS")
+	r.HandleFunc("/api/v1/apps/categories/run", shuffle.RunCategoryAction).Methods("POST", "OPTIONS")
 	r.HandleFunc("/api/v1/apps/upload", handleAppZipUpload).Methods("POST", "OPTIONS")
 	r.HandleFunc("/api/v1/apps/{appId}/activate", activateWorkflowAppDocker).Methods("GET", "OPTIONS")
 	r.HandleFunc("/api/v1/apps/frameworkConfiguration", shuffle.GetFrameworkConfiguration).Methods("GET", "OPTIONS")
@@ -6097,6 +5992,12 @@ func initHandlers() {
 	r.HandleFunc("/api/v1/workflows/{key}", deleteWorkflow).Methods("DELETE", "OPTIONS")
 	r.HandleFunc("/api/v1/workflows/{key}", shuffle.SaveWorkflow).Methods("PUT", "OPTIONS")
 	r.HandleFunc("/api/v1/workflows/{key}", shuffle.GetSpecificWorkflow).Methods("GET", "OPTIONS")
+	r.HandleFunc("/api/v1/workflows/recommend", shuffle.HandleActionRecommendation).Methods("POST", "OPTIONS")
+
+	// New for recommendations in Shuffle
+	r.HandleFunc("/api/v1/recommendations/get_actions", shuffle.HandleActionRecommendation).Methods("POST", "OPTIONS")
+	r.HandleFunc("/api/v1/recommendations/modify", shuffle.HandleRecommendationAction).Methods("POST", "OPTIONS")
+	r.HandleFunc("/api/v1/workflows/{key}/revisions", shuffle.GetWorkflowRevisions).Methods("GET", "OPTIONS")
 
 	// Triggers
 	r.HandleFunc("/api/v1/hooks/new", shuffle.HandleNewHook).Methods("POST", "OPTIONS")
@@ -6149,10 +6050,12 @@ func initHandlers() {
 	r.HandleFunc("/api/v1/environments/{key}/rerun", shuffle.HandleRerunExecutions).Methods("GET", "POST", "OPTIONS")
 
 	r.HandleFunc("/api/v1/orgs/{orgId}/validate_app_values", shuffle.HandleKeyValueCheck).Methods("POST", "OPTIONS")
+	r.HandleFunc("/api/v1/orgs/{orgId}/list_cache", shuffle.HandleListCacheKeys).Methods("GET", "OPTIONS")
 	r.HandleFunc("/api/v1/orgs/{orgId}/get_cache", shuffle.HandleGetCacheKey).Methods("POST", "OPTIONS")
 	r.HandleFunc("/api/v1/orgs/{orgId}/set_cache", shuffle.HandleSetCacheKey).Methods("POST", "OPTIONS")
+	r.HandleFunc("/api/v1/orgs/{orgId}/cache/{cache_key}", shuffle.HandleDeleteCacheKey).Methods("DELETE", "OPTIONS")
 	r.HandleFunc("/api/v1/orgs/{orgId}/stats", shuffle.HandleGetStatistics).Methods("GET", "OPTIONS")
-	r.HandleFunc("/api/v1/apps/{key}/execute", executeSingleAction).Methods("POST", "OPTIONS")
+	r.HandleFunc("/api/v1/orgs/{orgId}/revisions", shuffle.GetWorkflowRevisions).Methods("GET", "OPTIONS")
 
 	// Docker orborus specific - downloads an image
 	r.HandleFunc("/api/v1/get_docker_image", getDockerImage).Methods("POST", "OPTIONS")
@@ -6182,6 +6085,8 @@ func initHandlers() {
 	r.HandleFunc("/api/v1/users/notifications/clear", shuffle.HandleClearNotifications).Methods("GET", "OPTIONS")
 	r.HandleFunc("/api/v1/users/notifications/{notificationId}/markasread", shuffle.HandleMarkAsRead).Methods("GET", "OPTIONS")
 
+	r.HandleFunc("/api/v1/conversation", shuffle.RunActionAI).Methods("POST", "OPTIONS")
+
 	//r.HandleFunc("/api/v1/users/notifications/{notificationId}/markasread", shuffle.HandleMarkAsRead).Methods("GET", "OPTIONS")
 	r.HandleFunc("/api/v1/dashboards/{key}/widgets", shuffle.HandleNewWidget).Methods("POST", "OPTIONS")
 	r.HandleFunc("/api/v1/dashboards/{key}/widgets/{widget_id}", shuffle.HandleGetWidget).Methods("GET", "OPTIONS")

backend/go-app/main_test.go

@@ -24,6 +24,7 @@ type endpoint struct {
 	handler http.HandlerFunc
 	path    string
 	method  string
+	body    []byte
 }
 
 func init() {
@@ -44,7 +45,7 @@ func TestAuthenticationRequired(t *testing.T) {
 		{handler: shuffle.HandleNewOutlookRegister, path: "/functions/outlook/register", method: "GET"},
 		{handler: shuffle.HandleGetOutlookFolders, path: "/functions/outlook/getFolders", method: "GET"},
 		{handler: shuffle.HandleApiGeneration, path: "/api/v1/users/generateapikey", method: "GET"},
-		{handler: handleLogin, path: "/api/v1/users/login", method: "POST"}, // prob not this one
+		{handler: shuffle.HandleLogin, path: "/api/v1/users/login", method: "POST"}, // prob not this one
 		// handleRegister generates nil pointer exception. Not necessary for this anyway.
 		//{handler: handleRegister, path: "/api/v1/users/register", method: "POST"},
 		{handler: shuffle.HandleGetUsers, path: "/api/v1/users/getusers", method: "GET"},
@@ -108,8 +109,8 @@ func TestAuthenticationRequired(t *testing.T) {
 
 		{handler: verifySwagger, path: "/api/v1/verify_swagger", method: "POST"},
 		{handler: verifySwagger, path: "/api/v1/verify_openapi", method: "POST"},
-		{handler: echoOpenapiData, path: "/api/v1/get_openapi_uri", method: "POST"},
+		{handler: shuffle.EchoOpenapiData, path: "/api/v1/get_openapi_uri", method: "POST"},
-		{handler: echoOpenapiData, path: "/api/v1/validate_openapi", method: "POST"},
+		{handler: shuffle.EchoOpenapiData, path: "/api/v1/validate_openapi", method: "POST"},
 		{handler: shuffle.ValidateSwagger, path: "/api/v1/validate_openapi", method: "POST"},
 		{handler: getOpenapi, path: "/api/v1/get_openapi", method: "GET"},
 
@@ -117,7 +118,7 @@ func TestAuthenticationRequired(t *testing.T) {
 
 		{handler: handleCloudSetup, path: "/api/v1/cloud/setup", method: "POST"},
 		{handler: shuffle.HandleGetOrgs, path: "/api/v1/orgs", method: "POST"},
-		{handler: shuffle.HandleGetFileContent, path: "/api/v1/files/{fileId}/content", method: "POST"},
+		{handler: shuffle.HandleGetFileContent, path: "/api/v1/files/{fileId}/content", method: "POST", body: []byte("hi")},
 	}
 
 	var err error
@@ -197,10 +198,11 @@ func TestAuthenticationNotRequired(t *testing.T) {
 // requirements might change after the refactor.
 func TestCors(t *testing.T) {
 	handlers := []endpoint{
+		{handler: handleLogin, path: "/api/v1/users/login", method: "POST"}, // prob not this one
+
 		{handler: shuffle.HandleNewOutlookRegister, path: "/functions/outlook/register", method: "GET"},
 		{handler: shuffle.HandleGetOutlookFolders, path: "/functions/outlook/getFolders", method: "GET"},
 		{handler: shuffle.HandleApiGeneration, path: "/api/v1/users/generateapikey", method: "GET"},
-		{handler: handleLogin, path: "/api/v1/users/login", method: "POST"}, // prob not this one
 		// handleRegister generates nil pointer exception
 		{handler: handleRegister, path: "/api/v1/users/register", method: "POST"},
 		{handler: shuffle.HandleGetUsers, path: "/api/v1/users/getusers", method: "GET"},

backend/tests/files.sh

@@ -16,7 +16,7 @@
 	#r.HandleFunc("/api/v1/files/{fileId}", handleDeleteFile).Methods("DELETE", "OPTIONS")
 
 
-#curl http://localhost:5001/api/v1/files/create -H "Authorization: Bearer 09627dcb-7e2a-4843-819b-417d268ff840" -d '{"filename": "rule2.yar", "org_id": "11f67b76-6051-4425-b0d6-be23daac6d12", "workflow_id": "global", "namespace": "yara"}'
+curl http://localhost:5001/api/v1/files/create -H "Authorization: Bearer 317f5066-395c-414d-aa3d-479cf27f47dd" -d '{"filename": "rule2.yar", "org_id": "292c7e25-40ad-4f05-904f-77d3c7b735e6", "workflow_id": "global", "namespace": "yara"}'
-curl http://localhost:5002/api/v1/files/file_366ee8d2-1af6-4270-8639-213af30b4a29/upload -H "Authorization: Bearer 09627dcb-7e2a-4843-819b-417d268ff840" -F 'shuffle_file=@upload.sh'
+curl http://localhost:5001/api/v1/files/file_eb89e315-eb66-4d76-9df7-530fb003fc84/upload -H "Authorization: Bearer 317f5066-395c-414d-aa3d-479cf27f47dd" -F 'shuffle_file=@upload.sh'
 
 #curl http://localhost:5001/api/v1/files/namespaces/yara -H "Authorization: Bearer c5b4c827-65ec-47f4-9e8a-234cdba38959" --output rules.zip

docker-compose.yml

@@ -43,7 +43,6 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock
     environment:
       #- DOCKER_HOST=tcp://docker-socket-proxy:2375
-      - SHUFFLE_WORKER_VERSION=latest
       - ENVIRONMENT_NAME=${ENVIRONMENT_NAME}
       - BASE_URL=http://${OUTER_HOSTNAME}:5001
       - DOCKER_API_VERSION=1.40
@@ -114,4 +113,10 @@ services:
 networks:
   shuffle:
     driver: bridge
-    #driver: overlay 
+
+    # uncomment to set MTU for swarm mode.
+    # MTU should be whatever is your host's preferred MTU is.
+    # Refer to this doc to figure out what your host's MTU is:
+    # https://shuffler.io/docs/troubleshooting#TLS_timeout_error/Timeout_Errors/EOF_Errors 
+    # driver_opts:
+    #   com.docker.network.driver.mtu: 1460

frontend/package.json

@@ -1,13 +1,14 @@
 {
   "name": "shuffler",
   "homepage": "https://shuffler.io",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "private": true,
   "dependencies": {
-    "@babel/core": "^7.15.8",
+    "@codemirror/commands": "^6.2.2",
     "@emotion/is-prop-valid": "^1.1.1",
     "@emotion/react": "^11.7.0",
     "@emotion/styled": "^11.6.0",
+    "@lezer/highlight": "^1.1.3",
     "@material-ui/core": "^4.5.2",
     "@material-ui/icons": "^4.5.1",
     "@material-ui/lab": "^4.0.0-alpha.58",
@@ -17,10 +18,10 @@
     "@mui/icons-material": "^5.2.1",
     "@mui/material": "^5.2.3",
     "@mui/x-data-grid": "^5.17.11",
+    "@uiw/codemirror-themes": "^4.19.9",
     "@uiw/react-codemirror": "^3.2.1",
     "@use-it/interval": "^1.0.0",
     "algoliasearch": "^4.13.1",
-    "babel-eslint": "^10.1.0",
     "class-transformer": "^0.4.0",
     "create-react-app": "^4.0.3",
     "cytoscape": "^3.15.1",
@@ -58,7 +59,7 @@
     "react-draggable": "^3.3.2",
     "react-driftjs": "^1.2.2",
     "react-dropzone": "^10.1.10",
-    "react-ga": "^2.7.0",
+    "react-ga4": "^2.0.0",
     "react-iframe": "^1.8.0",
     "react-instantsearch-dom": "^6.28.0",
     "react-json-pretty": "^2.2.0",
@@ -76,14 +77,12 @@
     "shellwords": "^0.1.1",
     "simplebar": "^4.2.3",
     "styled-components": "^4.4.0",
-    "webpack": "^4.44.2",
-    "websocket": "^1.0.30",
     "yaml": "^1.7.2",
     "yamljs": "^0.3.0",
     "zone.js": "~0.11.4"
   },
   "scripts": {
-    "start": "react-scripts start",
+    "start": "HTTPS=false&&PORT=3000 react-scripts --openssl-legacy-provider start",
     "build": "react-scripts build",
     "test": "react-scripts test",
     "eject": "react-scripts eject",
@@ -106,6 +105,10 @@
   ],
   "devDependencies": {
     "prettier": "2.4.1",
-    "promise-window": "^1.2.1"
+    "promise-window": "^1.2.1",
+    "@babel/core": "^7.15.8",
+    "babel-eslint": "^10.1.0",
+    "webpack": "^4.44.2",
+	"@babel/plugin-proposal-private-property-in-object": "^7.21.11"
   }
 }

frontend/public/images/logos/orange_logo.svg

@@ -0,0 +1,5 @@
+<svg width="22" height="22" viewBox="0 0 22 22" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M0 0L-1.48522e-08 13.3913L4.40052 13.3913L4.40052 4.46465L22 4.46465L22 2.44001e-08L0 0Z" fill="#FF8444"/>
+<path d="M17.5995 8.60864L17.5995 17.5353L-9.90052e-09 17.5353L-1.48522e-08 22L22 22L22 8.60864L17.5995 8.60864Z" fill="#FF8444"/>
+<path d="M13.3915 8.60864L8.60889 8.60864L8.60889 13.3913L13.3915 13.3913L13.3915 8.60864Z" fill="#FF8444"/>
+</svg>

frontend/src/App.jsx

@@ -1,6 +1,5 @@
 import React, { useState, useEffect } from "react";
 
-//import { Route, Routes } from "react-router";
 import { Route, Routes, BrowserRouter } from "react-router-dom";
 import { CookiesProvider } from "react-cookie";
 import { removeCookies, useCookies } from "react-cookie";
@@ -10,7 +9,7 @@ import GettingStarted from "./views/GettingStarted";
 import EditWebhook from "./views/EditWebhook";
 import AngularWorkflow from "./views/AngularWorkflow";
 
-import Header from "./components/Header";
+import Header from "./components/Header.jsx";
 import theme from "./theme";
 import Apps from "./views/Apps";
 import AppCreator from "./views/AppCreator";
@@ -25,6 +24,7 @@ import Introduction from "./views/Introduction";
 import SetAuthentication from "./views/SetAuthentication";
 import SetAuthenticationSSO from "./views/SetAuthenticationSSO";
 import Search from "./views/Search.jsx";
+import RunWorkflow from "./views/RunWorkflow.jsx";
 
 import LandingPageNew from "./views/LandingpageNew";
 import LoginPage from "./views/LoginPage";
@@ -335,7 +335,9 @@ const App = (message, props) => {
           userdata={userdata}
           {...props}
         />
+				{/*
         <div style={{ height: 60 }} />
+				*/}
 				<Routes>
         	<Route
         	  exact
@@ -367,6 +369,7 @@ const App = (message, props) => {
         	      globalUrl={globalUrl}
         	      setCookie={setCookie}
         	      cookies={cookies}
+        	      checkLogin={checkLogin}
         	      {...props}
         	    />
         	  }
@@ -499,6 +502,7 @@ const App = (message, props) => {
         	  path="/workflows"
         	  element={
         	    <Workflows
+          			checkLogin={checkLogin}
         	      cookies={cookies}
         	      removeCookie={removeCookie}
         	      isLoaded={isLoaded}
@@ -540,6 +544,8 @@ const App = (message, props) => {
         	    />
         	  }
         	/>
+					<Route exact path="/workflows/:key/run" element={<RunWorkflow  userdata={userdata} globalUrl={globalUrl} isLoaded={isLoaded} isLoggedIn={isLoggedIn} surfaceColor={theme.palette.surfaceColor} inputColor={theme.palette.inputColor}{...props} /> } />
+					<Route exact path="/workflows/:key/execute" element={<RunWorkflow  userdata={userdata} globalUrl={globalUrl} isLoaded={isLoaded} isLoggedIn={isLoggedIn} surfaceColor={theme.palette.surfaceColor} inputColor={theme.palette.inputColor}{...props} /> } />
         	<Route
         	  exact
         	  path="/docs/:key"