Ton of minor fixes all pulled in. More info in the latest release for 1.2.0

backend/app_sdk/Dockerfile

@@ -1,6 +1,6 @@
 #FROM python:3.9.1-alpine as base
-#FROM python:3.10.0-alpine as base
+FROM python:3.10.0-alpine as base
-FROM python:3.11.3-alpine as base
+#FROM python:3.11.3-alpine as base
 
 FROM base as builder
 RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev tzdata coreutils

backend/app_sdk/app_base.py

@@ -2279,13 +2279,6 @@ class AppBase:
 
                 # Can't handle self yet (?)
                 ret = run.render(**globals())
-                    
-                # Load output as JSON
-                try:
-                    ret = json.loads(ret)
-                except: 
-                    pass
-
                 return ret
             except jinja2.exceptions.TemplateNotFound as e:
                 self.logger.info(f"[ERROR] Liquid Template error: {e}")
@@ -3071,6 +3064,7 @@ class AppBase:
                         #self.logger.info(action["parameters"])
 
                         # This seems redundant now 
+                        self.logger.info("[DEBUG] Pre parameters")
                         for parameter in newparams:
                             action["parameters"].append(parameter)
 
@@ -3092,6 +3086,7 @@ class AppBase:
 
                         # Multi_parameter has the data for each. variable
                         minlength = 0
+                        self.logger.info("[DEBUG] Pre-loading parameters")
                         multi_parameters = json.loads(json.dumps(params))
                         multiexecution = False
                         multi_execution_lists = []
@@ -3518,11 +3513,8 @@ class AppBase:
                                             try:
                                                 del params[field]
                                                 self.logger.info("[WARNING] Removed field invalid field %s" % field)
-                                            except KeyError as e:
+                                            except KeyError:
-                                                self.logger.info("[WARNING] Tried to remove field %s but it didn't exist" % field)
                                                 break
-                                        else:
-                                            self.logger.info("[ERROR] Couldn't find fieldsplit in error. Raw error: %s" % errorstring)
                                     else:
                                         newres = json.dumps({
                                             "success": False,
@@ -3899,11 +3891,7 @@ class AppBase:
             else:
                 self.logger.info("ACTION TYPE (unhandled): %s" % type(action))
 
-            #await app.execute_action(app.action)
             app.execute_action(app.action)
 
-    #app.run(host="0.0.0.0", port=33334)
-
 if __name__ == "__main__":
     AppBase.run()
-    #asyncio.run(AppBase.run(), debug=True)

backend/app_sdk/build.sh

@@ -2,7 +2,7 @@
 
 ### DEFAULT 
 NAME=shuffle-app_sdk
-VERSION=1.1.0
+VERSION=1.2.0
 
 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force
 docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly -t shuffle/shuffle:app_sdk -t shuffle/$NAME:$VERSION -t docker.pkg.github.com/shuffle/shuffle/$NAME:$VERSION -t ghcr.io/shuffle/$NAME:$VERSION -t ghcr.io/shuffle/$NAME:nightly

backend/go-app/docker.go

@@ -678,8 +678,11 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) {
 		alternativeName = strings.Join(alternativeNameSplit[1:3], "/")
 	}
 
+	log.Printf("[INFO] Trying to download image: %s. Alt: %s", version.Name, alternativeName)
+
 	for _, image := range images {
 		for _, tag := range image.RepoTags {
+			//log.Printf("[DEBUG] Tag: %s", tag)
 			if strings.ToLower(tag) == strings.ToLower(version.Name) {
 				img = image
 				tagFound = tag
@@ -693,6 +696,29 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) {
 		}
 	}
 
+	pullOptions := types.ImagePullOptions{}
+	if len(img.ID) == 0 {
+		_, err := dockercli.ImagePull(context.Background(), version.Name, pullOptions)
+		if err == nil {
+			tagFound = version.Name
+			img.ID = version.Name
+			img2.ID = version.Name
+
+			dockercli.ImageTag(ctx, version.Name, alternativeName)
+		}
+	}
+
+	if len(img2.ID) == 0 {
+		_, err := dockercli.ImagePull(context.Background(), alternativeName, pullOptions)
+		if err == nil {
+			tagFound = alternativeName
+			img.ID = alternativeName
+			img2.ID = alternativeName
+
+			dockercli.ImageTag(ctx, alternativeName, version.Name)
+		}
+	}
+
 	// REBUILDS THE APP
 	if len(img.ID) == 0 {
 		if len(img2.ID) == 0 {
@@ -722,7 +748,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) {
 					foundApp := shuffle.WorkflowApp{}
 					imageName = strings.ToLower(imageName)
 					imageVersion = strings.ToLower(imageVersion)
-					log.Printf("[DEBUG] Looking for appname %s with version %s", imageName, imageVersion)
+					log.Printf("[DEBUG] Docker Looking for appname %s with version %s", imageName, imageVersion)
 
 					for _, app := range workflowapps {
 						if strings.ToLower(strings.Replace(app.Name, " ", "_", -1)) == imageName && app.AppVersion == imageVersion {

backend/go-app/go.mod

@@ -19,7 +19,7 @@ require (
 	github.com/gorilla/mux v1.8.0
 	github.com/h2non/filetype v1.1.3
 	github.com/satori/go.uuid v1.2.0
-	github.com/shuffle/shuffle-shared v0.4.2
+	github.com/shuffle/shuffle-shared v0.4.9
 	golang.org/x/crypto v0.3.0
 	google.golang.org/api v0.103.0
 	google.golang.org/appengine v1.6.7

backend/go-app/main.go

@@ -81,18 +81,14 @@ import (
 var gceProject = "shuffle"
 var bucketName = "shuffler.appspot.com"
 var baseAppPath = "/home/frikky/git/shaffuru/tmp/apps"
+
 var baseDockerName = "frikky/shuffle"
 var registryName = "registry.hub.docker.com"
 var runningEnvironment = "onprem"
 
 var syncUrl = "https://shuffler.io"
-
-// var syncUrl = "http://localhost:5002"
 var syncSubUrl = "https://shuffler.io"
 
-//var syncUrl = "http://localhost:5002"
-//var syncSubUrl = "https://050196912a9d.ngrok.io"
-
 var dbclient *datastore.Client
 
 type Userapi struct {
@@ -5873,7 +5869,7 @@ func initHandlers() {
 	r.HandleFunc("/api/v1/streams/results", handleGetStreamResults).Methods("POST", "OPTIONS")
 
 	// Used by orborus
-	r.HandleFunc("/api/v1/workflows/queue", handleGetWorkflowqueue).Methods("GET")
+	r.HandleFunc("/api/v1/workflows/queue", handleGetWorkflowqueue).Methods("GET", "POST")
 	r.HandleFunc("/api/v1/workflows/queue/confirm", handleGetWorkflowqueueConfirm).Methods("POST")
 
 	// App specific
@@ -6018,6 +6014,8 @@ func initHandlers() {
 	r.HandleFunc("/api/v1/users/notifications/clear", shuffle.HandleClearNotifications).Methods("GET", "OPTIONS")
 	r.HandleFunc("/api/v1/users/notifications/{notificationId}/markasread", shuffle.HandleMarkAsRead).Methods("GET", "OPTIONS")
 
+	r.HandleFunc("/api/v1/conversation", shuffle.RunActionAI).Methods("POST", "OPTIONS")
+
 	//r.HandleFunc("/api/v1/users/notifications/{notificationId}/markasread", shuffle.HandleMarkAsRead).Methods("GET", "OPTIONS")
 	r.HandleFunc("/api/v1/dashboards/{key}/widgets", shuffle.HandleNewWidget).Methods("POST", "OPTIONS")
 	r.HandleFunc("/api/v1/dashboards/{key}/widgets/{widget_id}", shuffle.HandleGetWidget).Methods("GET", "OPTIONS")

backend/go-app/walkoff.go

@@ -11,6 +11,7 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"math/rand"
 	"net/http"
 	"net/url"
 	"os"
@@ -251,16 +252,42 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
 		return
 	}
 
-	id := request.Header.Get("Org-Id")
+	// This is really the environment's name - NOT org-id
-	if len(id) == 0 {
+	orgId := request.Header.Get("Org-Id")
-		log.Printf("[INFO] No org-id header set")
+	if len(orgId) == 0 {
+		log.Printf("[AUDIT] No org-id header set")
 		resp.WriteHeader(401)
 		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Specify the org-id header."}`)))
 		return
 	}
 
-	ctx := context.Background()
+	environment := request.Header.Get("org")
-	env, err := shuffle.GetEnvironment(ctx, id, "")
+	if len(environment) == 0 {
+		log.Printf("[AUDIT] No 'org' header set (get workflow queue). Required for cloud.")
+		/*
+			resp.WriteHeader(403)
+			resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Specify the org header. This can be done by setting the 'ORG' environment variable for Orborus to your Org ID in Shuffle"}`)))
+			return
+		*/
+	}
+
+	orborusLabel := request.Header.Get("x-orborus-label")
+
+	// This section is cloud custom for now
+	auth := request.Header.Get("Authorization")
+	if len(auth) == 0 {
+		log.Printf("[AUDIT] No Authorization header set. Required for cloud. Env: %s, org: %s", orgId, environment)
+		/*
+			resp.WriteHeader(401)
+			resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Specify the auth header (only applicable for cloud for now)."}`)))
+			return
+		*/
+	}
+
+	//log.Printf("[AUDIT] Get workflow queue for org %s, env %s, orborus label %s", orgId, environment, orborusLabel)
+
+	ctx := shuffle.GetContext(request)
+	env, err := shuffle.GetEnvironment(ctx, orgId, "")
 	timeNow := time.Now().Unix()
 	if err == nil && len(env.Id) > 0 && len(env.Name) > 0 {
 		if time.Now().Unix() > env.Edited+60 {
@@ -273,7 +300,154 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
 		}
 	}
 
-	executionRequests, err := shuffle.GetWorkflowQueue(ctx, id, 100)
+	//log.Printf("Found env: %#v", env)
+	if len(env.OrgId) > 0 {
+		environment = env.OrgId
+	}
+
+	if request.Method == "POST" {
+		if rand.Intn(1) == 0 {
+			// Parse out body
+			body, err := ioutil.ReadAll(request.Body)
+			if err == nil {
+
+				// Parse out CPU, memory and disk.
+
+				var envData shuffle.OrborusStats
+				err = json.Unmarshal(body, &envData)
+				if err == nil && !envData.Swarm && !envData.Kubernetes && (envData.CPU > 0 || envData.Memory > 0 || envData.Disk > 0) {
+
+					// Set the input in memory
+					envData.OrgId = orgId
+					envData.Environment = environment
+					envData.OrborusLabel = orborusLabel
+					envData.Timestamp = time.Now().Unix()
+
+					if envData.CPU > 0 && envData.MaxCPU > 0 {
+						envData.CPUPercent = float64(envData.CPU) / float64(envData.MaxCPU)
+					}
+
+					if envData.Memory > 0 && envData.MaxMemory > 0 {
+						envData.MemoryPercent = float64(envData.Memory) / float64(envData.MaxMemory)
+					}
+
+					// Check if CPU percent constantly has stayed above X% for the last Y requests
+					percentageCheck := 90
+					concurrentChecks := 0
+
+					//if int(envData.CPUPercent) > percentageCheck {
+					// Get cached data
+					percentages := []float64{}
+					cacheKey := fmt.Sprintf("%s_%s_percent", orgId, strings.ToLower(environment))
+
+					// Marshal float list into []byte
+					cacheData := []byte{}
+					cache, err := shuffle.GetCache(ctx, cacheKey)
+					if err == nil {
+						// Unmarshal into percentages
+						cacheData := []byte(cache.([]uint8))
+						err = json.Unmarshal(cacheData, &percentages)
+						if err != nil {
+							log.Printf("[INFO] error in cache unmarshal for percentages: %s", err)
+						}
+
+						if len(percentages) > concurrentChecks {
+							percentages = percentages[:concurrentChecks]
+						}
+
+						percentages = append(percentages, envData.CPUPercent)
+						if len(percentages) > concurrentChecks {
+							//log.Printf("[INFO] Checking percentages: %v", percentages)
+
+							// percentageCheck := 1
+							sendAlert := true
+							for _, p := range percentages {
+								if int(p) < percentageCheck {
+									//log.Printf("[AUDIT] CPU percent is below %d: %d", percentageCheck, int(p))
+									sendAlert = false
+									break
+								}
+							}
+
+							if sendAlert {
+								log.Printf("[INFO] CPU percent has been above %d percent for the last 5 requests. Sending alert. Env: %s, org: %s", percentageCheck, environment, orgId)
+
+								// Set notification + alert for organization
+								err = shuffle.CreateOrgNotification(
+									ctx,
+									fmt.Sprintf("CPU percent has been above %d percent", percentageCheck),
+									fmt.Sprintf("A environment %s has been using more than %d\\% CPU for the last 5 requests.", environment, percentageCheck),
+									fmt.Sprintf("/admin?tab=environments"),
+									environment,
+									true,
+								)
+
+								if err != nil {
+									log.Printf("[ERROR] error creating notification: %s", err)
+								}
+
+								org, err := shuffle.GetOrg(ctx, environment)
+								if err == nil {
+									foundRecommendation := false
+									for _, recommendation := range org.Priorities {
+										if strings.Contains(recommendation.Name, "CPU") {
+											foundRecommendation = true
+											break
+										}
+									}
+
+									if !foundRecommendation {
+										// Add to start of org.Priorities
+										org.Priorities = append(org.Priorities, shuffle.Priority{
+											Name:        fmt.Sprintf("High CPU in environment %s", orgId),
+											Description: fmt.Sprintf("The environment %s has been using more than %d percent CPU.", orgId, percentageCheck),
+											Type:        "scale",
+											Active:      true,
+											URL:         fmt.Sprintf("/admin?tab=environments"),
+										})
+
+										//Make last item the first item
+										org.Priorities = append([]shuffle.Priority{org.Priorities[len(org.Priorities)-1]}, org.Priorities[:len(org.Priorities)-1]...)
+										err = shuffle.SetOrg(ctx, *org, org.Id)
+										if err != nil {
+											log.Printf("[ERROR] Problem setting org: %s", err)
+										}
+									}
+								}
+							}
+
+							if len(percentages) > 1 {
+								percentages = percentages[1:]
+							}
+						}
+
+						// Marshal float list into []byte
+					} else {
+						//log.Printf("[ERROR] Failed getting cache: %s", err)
+						percentages = append(percentages, envData.CPUPercent)
+					}
+
+					if len(percentages) > 0 {
+						//log.Printf("[DEBUG] Setting cache for %s: %#v", cacheKey, percentages)
+						cacheData, err = json.Marshal(percentages)
+						if err != nil {
+							log.Printf("[INFO] error in cache marshal: %s", err)
+						}
+
+						// Add the new data
+						go shuffle.SetCache(ctx, cacheKey, cacheData, 5)
+					}
+				}
+
+				//log.Printf("CPU percent: %f", envData.CPUPercent)
+				//log.Printf("Memory percent: %f", envData.MemoryPercent*100)
+
+				go shuffle.SetenvStats(ctx, envData)
+			}
+		}
+	}
+
+	executionRequests, err := shuffle.GetWorkflowQueue(ctx, orgId, 100)
 	if err != nil {
 		// Skipping as this comes up over and over
 		//log.Printf("(2) Failed reading body for workflowqueue: %s", err)
@@ -290,21 +464,21 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
 
 		// Try again :)
 		if len(env.Id) == 0 && len(env.Name) == 0 {
-			orgId := ""
+			foundId := ""
 			for _, requestData := range executionRequests.Data {
 				execution, err := shuffle.GetWorkflowExecution(ctx, requestData.ExecutionId)
 				if err == nil {
 					if len(execution.ExecutionOrg) > 0 {
-						orgId = execution.ExecutionOrg
+						foundId = execution.ExecutionOrg
 						break
 					}
 				}
 			}
 
 			if len(orgId) > 0 {
-				env, err := shuffle.GetEnvironment(ctx, id, orgId)
+				env, err := shuffle.GetEnvironment(ctx, orgId, foundId)
 				if err != nil {
-					log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", id, err)
+					log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", orgId, err)
 					//resp.WriteHeader(401)
 					//resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No env found matching %s"}`, id)))
 					//return
@@ -361,9 +535,9 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) {
 	err = json.Unmarshal(body, &actionResult)
 	if err != nil {
 		log.Printf("[WARNING] Failed ActionResult unmarshaling (stream result): %s", err)
-		resp.WriteHeader(401)
+		//resp.WriteHeader(401)
-		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
+		//resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
-		return
+		//return
 	}
 
 	ctx := context.Background()
@@ -476,9 +650,9 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
 	err = json.Unmarshal(body, &actionResult)
 	if err != nil {
 		log.Printf("[WARNING] Failed ActionResult unmarshaling (queue): %s", err)
-		resp.WriteHeader(401)
+		//resp.WriteHeader(401)
-		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
+		//resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
-		return
+		//return
 	}
 
 	//log.Printf("Received action: %#v", actionResult)

docker-compose.yml

@@ -1,62 +1,62 @@
 version: '3'
 services:
-  frontend:
+  #frontend:
-    image: ghcr.io/shuffle/shuffle-frontend:latest
+  #  image: ghcr.io/shuffle/shuffle-frontend:latest
-    container_name: shuffle-frontend
+  #  container_name: shuffle-frontend
-    hostname: shuffle-frontend
+  #  hostname: shuffle-frontend
-    ports:
+  #  ports:
-      - "${FRONTEND_PORT}:80"
+  #    - "${FRONTEND_PORT}:80"
-      - "${FRONTEND_PORT_HTTPS}:443"
+  #    - "${FRONTEND_PORT_HTTPS}:443"
-    networks:
+  #  networks:
-      - shuffle
+  #    - shuffle
-    environment:
+  #  environment:
-      - BACKEND_HOSTNAME=${BACKEND_HOSTNAME}
+  #    - BACKEND_HOSTNAME=${BACKEND_HOSTNAME}
-    restart: unless-stopped
+  #  restart: unless-stopped
-    depends_on:
+  #  depends_on:
-      - backend
+  #    - backend
-  backend:
+  #backend:
-    image: ghcr.io/shuffle/shuffle-backend:latest
+  #  image: ghcr.io/shuffle/shuffle-backend:latest
-    container_name: shuffle-backend
+  #  container_name: shuffle-backend
-    hostname: ${BACKEND_HOSTNAME}
+  #  hostname: ${BACKEND_HOSTNAME}
-    # Here for debugging:
+  #  # Here for debugging:
-    ports:
+  #  ports:
-      - "${BACKEND_PORT}:5001"
+  #    - "${BACKEND_PORT}:5001"
-    networks:
+  #  networks:
-      - shuffle
+  #    - shuffle
-    volumes:
+  #  volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
+  #    - /var/run/docker.sock:/var/run/docker.sock
-      - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps:z
+  #    - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps:z
-      - ${SHUFFLE_FILE_LOCATION}:/shuffle-files:z
+  #    - ${SHUFFLE_FILE_LOCATION}:/shuffle-files:z
-    env_file: .env
+  #  env_file: .env
-    environment:
+  #  environment:
-      #- DOCKER_HOST=tcp://docker-socket-proxy:2375
+  #    #- DOCKER_HOST=tcp://docker-socket-proxy:2375
-      - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps
+  #    - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps
-      - SHUFFLE_FILE_LOCATION=/shuffle-files
+  #    - SHUFFLE_FILE_LOCATION=/shuffle-files
-    restart: unless-stopped
+  #  restart: unless-stopped
-  orborus:
+  #orborus:
-    image: ghcr.io/shuffle/shuffle-orborus:latest
+  #  image: ghcr.io/shuffle/shuffle-orborus:latest
-    container_name: shuffle-orborus
+  #  container_name: shuffle-orborus
-    hostname: shuffle-orborus
+  #  hostname: shuffle-orborus
-    networks:
+  #  networks:
-      - shuffle
+  #    - shuffle
-    volumes:
+  #  volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
+  #    - /var/run/docker.sock:/var/run/docker.sock
-    environment:
+  #  environment:
-      #- DOCKER_HOST=tcp://docker-socket-proxy:2375
+  #    #- DOCKER_HOST=tcp://docker-socket-proxy:2375
-      - SHUFFLE_WORKER_VERSION=latest
+  #    - SHUFFLE_WORKER_VERSION=latest
-      - ENVIRONMENT_NAME=${ENVIRONMENT_NAME}
+  #    - ENVIRONMENT_NAME=${ENVIRONMENT_NAME}
-      - BASE_URL=http://${OUTER_HOSTNAME}:5001
+  #    - BASE_URL=http://${OUTER_HOSTNAME}:5001
-      - DOCKER_API_VERSION=1.40
+  #    - DOCKER_API_VERSION=1.40
-      - SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME}
+  #    - SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME}
-      - SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY}
+  #    - SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY}
-      - SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX}
+  #    - SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX}
-      - HTTP_PROXY=${HTTP_PROXY}
+  #    - HTTP_PROXY=${HTTP_PROXY}
-      - HTTPS_PROXY=${HTTPS_PROXY}
+  #    - HTTPS_PROXY=${HTTPS_PROXY}
-      - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY}
+  #    - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY}
-      - SHUFFLE_PASS_APP_PROXY=${SHUFFLE_PASS_APP_PROXY}
+  #    - SHUFFLE_PASS_APP_PROXY=${SHUFFLE_PASS_APP_PROXY}
-    restart: unless-stopped
+  #  restart: unless-stopped
-    security_opt:
+  #  security_opt:
-      - seccomp:unconfined
+  #    - seccomp:unconfined
   opensearch:
     image: opensearchproject/opensearch:2.5.0
     hostname: shuffle-opensearch

frontend/public/images/logos/orange_logo.svg

@@ -0,0 +1,5 @@
+<svg width="22" height="22" viewBox="0 0 22 22" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M0 0L-1.48522e-08 13.3913L4.40052 13.3913L4.40052 4.46465L22 4.46465L22 2.44001e-08L0 0Z" fill="#FF8444"/>
+<path d="M17.5995 8.60864L17.5995 17.5353L-9.90052e-09 17.5353L-1.48522e-08 22L22 22L22 8.60864L17.5995 8.60864Z" fill="#FF8444"/>
+<path d="M13.3915 8.60864L8.60889 8.60864L8.60889 13.3913L13.3915 13.3913L13.3915 8.60864Z" fill="#FF8444"/>
+</svg>

frontend/src/App.jsx

@@ -10,7 +10,7 @@ import GettingStarted from "./views/GettingStarted";
 import EditWebhook from "./views/EditWebhook";
 import AngularWorkflow from "./views/AngularWorkflow";
 
-import Header from "./components/Header";
+import Header from "./components/Header.jsx";
 import theme from "./theme";
 import Apps from "./views/Apps";
 import AppCreator from "./views/AppCreator";
@@ -336,7 +336,9 @@ const App = (message, props) => {
           userdata={userdata}
           {...props}
         />
+				{/*
         <div style={{ height: 60 }} />
+				*/}
 				<Routes>
         	<Route
         	  exact

frontend/src/components/EditWorkflow.jsx

@@ -145,9 +145,11 @@ const EditWorkflow = (props) => {
         style: {
           backgroundColor: theme.palette.surfaceColor,
           color: "white",
-          minWidth: isMobile ? "90%" : newWorkflow === true ? 1000 : 550,
+          minWidth: isMobile ? "90%" : 550,
-          maxWidth: isMobile ? "90%" : newWorkflow === true ? 1000 : 550,
+          maxWidth: isMobile ? "90%" : 550,
 					minHeight: 400,
+          //minWidth: isMobile ? "90%" : newWorkflow === true ? 1000 : 550,
+          //maxWidth: isMobile ? "90%" : newWorkflow === true ? 1000 : 550,
         },
       }}
     >
@@ -196,7 +198,7 @@ const EditWorkflow = (props) => {
         	  	</div>
 						: null}
         	</div>
-					{newWorkflow === true ? 
+					{/*newWorkflow === true ? 
 						<div style={{flex: 1, marginLeft: 45, }}>
 							<Typography variant="h6">
 								Use a Template
@@ -205,7 +207,7 @@ const EditWorkflow = (props) => {
 								Start your workflow from our templating system. This uses publied workflows from our <a href="/creators" rel="noopener noreferrer" target="_blank" style={{ textDecoration: "none", color: "#f86a3e"}}>Creators</a> to generate full Usecases or parts of your Workflow.
 							</Typography>
 						</div>
-					: null}
+					: null*/}
 				</div>
       </DialogTitle>
       <FormControl>
@@ -438,7 +440,7 @@ const EditWorkflow = (props) => {
 							</IconButton>
 						</Tooltip>
 					</div>
-					{newWorkflow === true ? 
+					{/*newWorkflow === true ? 
 						<div style={{marginLeft: 50, maxWidth: 400, minWidth: 400, position: "relative",}}>
 							<UsecaseSearch
 								globalUrl={globalUrl}
@@ -449,7 +451,7 @@ const EditWorkflow = (props) => {
 								userdata={userdata}
 							/>
 						</div>
-					: null}
+					: null*/}
         </DialogContent>
         <DialogActions>
           <Button

frontend/src/components/Oauth2Auth.jsx

@@ -1,4 +1,5 @@
 import React, { useRef, useState, useEffect, useLayoutEffect } from "react";
+import { useParams, useNavigate, Link } from "react-router-dom";
 import { useTheme } from "@material-ui/core/styles";
 import theme from '../theme';
 
@@ -75,6 +76,7 @@ const registeredApps = [
 	"jira",
 	"jira_service_desk",
 	"jira_service_management",
+	"github",
 ]
 
 const AuthenticationOauth2 = (props) => {
@@ -92,8 +94,11 @@ const AuthenticationOauth2 = (props) => {
 		isCloud,
 		autoAuth, 
 		authButtonOnly, 
+		isLoggedIn,
   } = props;
 
+  let navigate = useNavigate();
+
   //const [update, setUpdate] = React.useState("|")
   const [defaultConfigSet, setDefaultConfigSet] = React.useState(
     authenticationType.client_id !== undefined &&
@@ -113,7 +118,7 @@ const AuthenticationOauth2 = (props) => {
   const [oauthUrl, setOauthUrl] = React.useState("");
   const [buttonClicked, setButtonClicked] = React.useState(false);
 
-  const [offlineAccess, setOfflineAccess] = React.useState(true);
+  const [offlineAccess, setOfflineAccess] = React.useState(false);
   const allscopes = authenticationType.scope !== undefined ? authenticationType.scope : [];
     
 
@@ -134,8 +139,13 @@ const AuthenticationOauth2 = (props) => {
     active: true,
   });
 
+
 	useEffect(() => {
-		console.log("Should automatically click the auto-auth button?")
+		if (isLoggedIn === false) {
+			navigate(`/login?view=${window.location.pathname}&message=Log in to authenticate this app`)
+		}
+
+		console.log("Should automatically click the auto-auth button?: ", autoAuth)
 		if (autoAuth === true && selectedApp !== undefined) {
 			startOauth2Request() 
 		}
@@ -147,7 +157,8 @@ const AuthenticationOauth2 = (props) => {
 
 	const startOauth2Request = (admin_consent) => {
 		// Admin consent also means to add refresh tokens
-
+		console.log("Inside oauth2 request for app: ", selectedApp.name)
+		selectedApp.name = selectedApp.name.replace(" ", "_").toLowerCase()
 
 		//console.log("APP: ", selectedApp)
 		if (selectedApp.name.toLowerCase() == "outlook_graph" || selectedApp.name.toLowerCase() == "outlook_office365") {
@@ -185,10 +196,10 @@ const AuthenticationOauth2 = (props) => {
 			)
 		} else if (selectedApp.name.toLowerCase() == "slack") {
 			handleOauth2Request(
-				"151779186901.2448678750935",
+				"5155508477298.5168162485601",
 				"",
 				"https://slack.com",
-				["chat:write:user", "im:read", "im:write", "search:read", "usergroups:read", "usergroups:write", "offline_access"],
+				["chat:write:user", "im:read", "im:write", "search:read", "usergroups:read", "usergroups:write",],
 				admin_consent,
 			)
 		} else if (selectedApp.name.toLowerCase() == "webex") {
@@ -196,7 +207,7 @@ const AuthenticationOauth2 = (props) => {
 				"Cab184f3d7271f540443c79b5b79845e3387abbbdb3db4233a87ea3a5432fb3d5",
 				"",
 				"https://webexapis.com",
-				["spark:all", "offline_access"],
+				["spark:all"],
 				admin_consent,
 			)
 		} else if (selectedApp.name.toLowerCase().includes("microsoft_teams")) {
@@ -212,7 +223,7 @@ const AuthenticationOauth2 = (props) => {
 				"35fa3a384040470db0c8527e90a3c2eb",
 				"",
 				"https://api.todoist.com",
-				["task:add", "offline_access"],
+				["task:add",],
 				admin_consent,
 			)
 		} else if (selectedApp.name.toLowerCase().includes("microsoft_sentinel")) {
@@ -220,7 +231,7 @@ const AuthenticationOauth2 = (props) => {
 				"4c16e8c4-3d34-4aa1-ac94-262ea170b7f7",
 				"",
 				"https://management.azure.com",
-				["https://management.azure.com/user_impersonation", "offline_access"],
+				["https://management.azure.com/user_impersonation",],
 				admin_consent,
 			)
 		} else if (selectedApp.name.toLowerCase().includes("microsoft_365_defender")) {
@@ -228,7 +239,7 @@ const AuthenticationOauth2 = (props) => {
 				"4c16e8c4-3d34-4aa1-ac94-262ea170b7f7",
 				"",
 				"https://graph.microsoft.com",
-				["SecurityEvents.ReadWrite.All", "offline_access"],
+				["SecurityEvents.ReadWrite.All",],
 				admin_consent,
 			)
 		} else if (selectedApp.name.toLowerCase().includes("google_sheets")) {
@@ -244,7 +255,7 @@ const AuthenticationOauth2 = (props) => {
 			handleOauth2Request(
 				"253565968129-6pij4g6ojim4gpum0h9m9u3bc357qsq7.apps.googleusercontent.com",
 				"",
-				"https://www.googleapis.com/drive/v3",
+				"https://www.googleapis.com",
 				["https://www.googleapis.com/auth/drive",],
 				admin_consent,
 				"consent",
@@ -254,9 +265,19 @@ const AuthenticationOauth2 = (props) => {
 				"AI02egeCQh1Zskm1QAJaaR6dzjR97V2F",
 				"",
 				"https://api.atlassian.com",
-				["read:jira-work", "write:jira-work", "read:servicedesk:jira-service-management", "write:servicedesk:jira-service-management", "read:request:jira-service-management", "write:request:jira-service-management", "offline_access"],
+				["read:jira-work", "write:jira-work", "read:servicedesk:jira-service-management", "write:servicedesk:jira-service-management", "read:request:jira-service-management", "write:request:jira-service-management",],
 				admin_consent,
 			)
+		} else if (selectedApp.name.toLowerCase().includes("github")) {
+			handleOauth2Request(
+				"3d272b1b782b100b1e61",
+				"",
+				"https://api.github.com",
+				["repo","user","project","notifications",],
+				admin_consent,
+			)
+		} else {
+			console.log("No match found for: ", selectedApp.name)
 		}
 		// write:request:jira-service-management
 	}
@@ -315,6 +336,7 @@ const AuthenticationOauth2 = (props) => {
 		}
 		
 		var url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&prompt=${defaultPrompt}&scope=${resources}&state=${state}&access_type=offline`;
+
 		if (admin_consent === true) {
 			console.log("Running Oauth2 WITH admin consent")
     	//url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&prompt=consent&scope=${resources}&state=${state}&access_type=offline`;
@@ -605,7 +627,8 @@ const AuthenticationOauth2 = (props) => {
 										onClick={() => {
 											// Hardcode some stuff?
 											// This could prolly be added to the app itself with a "default" client ID 
-											startOauth2Request(true)
+											//startOauth2Request(true)
+											startOauth2Request()
 										}}
 										color="primary"
 									>

frontend/src/components/ParsedAction.jsx

@@ -167,6 +167,7 @@ const ParsedAction = (props) => {
 		setLastSaved,
 		setShowVideo,
 		toolsAppId,
+		aiSubmit,
 		//expansionModalOpen,
 		//setExpansionModalOpen,
   } = props;
@@ -1458,7 +1459,7 @@ const ParsedAction = (props) => {
                 if (found === null) {
                   setActivateHidingBodyButton(true);
                 } else {
-									console.log("In found: ", found, hideBody)
+									//console.log("In found: ", found, hideBody)
 								}
               } else {
                 //console.log("SHOW BUTTON");
@@ -1532,6 +1533,12 @@ const ParsedAction = (props) => {
 								expansionModalOpen={expansionModalOpen}
 								setExpansionModalOpen={setExpansionModalOpen}
 								globalUrl={globalUrl}
+
+								workflowExecutions={workflowExecutions}
+								getParents={getParents}
+								selectedAction={selectedAction}
+								parameterName={data.name}
+								aiSubmit={aiSubmit}
 							/>
 						)
 
@@ -3222,6 +3229,7 @@ const ParsedAction = (props) => {
           </Tooltip>
         </div>
       ) : null}
+
       {selectedAction.authentication !== undefined &&
 				selectedAction.authentication !== null &&
 				selectedAction.authentication.length > 0 ? (

frontend/src/components/Searchfield.jsx

@@ -83,28 +83,32 @@ const SearchField = props => {
 			*/
 
 		return (
-		  <form id="search_form" noValidate type="searchbox" action="" role="search" style={{margin: 0, }} onClick={() => {
+		  <form id="search_form" noValidate type="searchbox" action="" role="search" style={{margin: "10px 0px 0px 0px", }} onClick={() => {
 			}}>
 				<TextField 
 					fullWidth
-					style={{backgroundColor: theme.palette.inputColor, borderRadius: borderRadius, margin: 10, width: "100%",}} 
+					style={{backgroundColor: theme.palette.surfaceColor, borderRadius: borderRadius, minWidth: 403, maxWidth: 403, }} 
 					InputProps={{
 						style:{
 							color: "white",
 							fontSize: "1em",
 							height: 50,
 							margin: 0, 
+							fontSize: "0.9em",
+							paddingLeft: 10,
 						},
-						startAdornment: (
+						disableUnderline: true,
+						endAdornment: (
 							<InputAdornment position="start">
-								<SearchIcon style={{marginLeft: 5}}/>
+								<SearchIcon style={{marginLeft: 5, color: "#f86a3e",}}/>
+								
 							</InputAdornment>
 						),
 					}}
 					autoComplete='off'
 					type="search"
 					color="primary"
-					placeholder="Find Public Apps, Workflows, Documentation and more"
+					placeholder="Find Public Apps, Workflows, Documentation..."
 					value={currentRefinement}
 					id="shuffle_search_field"
 					onClick={(event) => {

frontend/src/components/ShuffleCodeEditor.jsx

@@ -20,6 +20,7 @@ import { orange } from '@mui/material/colors';
 import { isMobile } from "react-device-detect" 
 import NestedMenuItem from "material-ui-nested-menu-item";
 import { GetParsedPaths, FindJsonPath } from "../views/Apps.jsx";
+import { SetJsonDotnotation } from "../views/AngularWorkflow.jsx";
 
 import {
 	FullscreenExit as FullscreenExitIcon,
@@ -114,7 +115,24 @@ const pythonFilters = [
 //});
 
 const CodeEditor = (props) => {
-	const { globalUrl, fieldCount, setFieldCount, actionlist, changeActionParameterCodeMirror, expansionModalOpen, setExpansionModalOpen, codedata, setcodedata, isFileEditor, runUpdateText, toolsAppId } = props
+	const { 
+		globalUrl, 
+		fieldCount, 
+		setFieldCount, 
+		actionlist, 
+		changeActionParameterCodeMirror, 
+		expansionModalOpen, 
+		setExpansionModalOpen, 
+		codedata, 
+		setcodedata, 
+		isFileEditor, 
+		runUpdateText, 
+		toolsAppId, 
+		parameterName, 
+		selectedAction ,
+		workflowExecutions,
+		getParents,
+	} = props
 
 	const [localcodedata, setlocalcodedata] = React.useState(codedata === undefined || codedata === null || codedata.length === 0 ? "" : codedata);
   	// const {codelang, setcodelang} = props
@@ -140,6 +158,8 @@ const CodeEditor = (props) => {
   const [menuPosition, setMenuPosition] = useState(null);
   const [showAutocomplete, setShowAutocomplete] = React.useState(false);
 
+	const [isAiLoading, setIsAiLoading] = React.useState(false);
+
 	const baseResult = ""
 	const [executionResult, setExecutionResult] = useState({
 		"valid": false,		
@@ -188,8 +208,201 @@ const CodeEditor = (props) => {
 		setMainVariables(tmpVariables)
 	}, [])
 
+	const aiSubmit = (value, inputAction) => {
+		if (value === undefined || value === "") {
+			console.log("No value input!")
+			return
+		}
+
+		setIsAiLoading(true)
+
+		// Time to construct this huh... Hmm
+		var AppContext = []
+		if (inputAction !== undefined && inputAction !== null && getParents !== undefined && getParents !== null && workflowExecutions !== undefined && workflowExecutions !== null) {
+			const parents = getParents(inputAction)
+
+			console.log("Parents: ", parents)
+			var actionlist = []
+			if (parents.length > 1) {
+				for (let [key,keyval] in Object.entries(parents)) {
+					const item = parents[key];
+					if (item.label === "Execution Argument") {
+						continue;
+					}
+
+					var exampledata = item.example === undefined || item.example === null ? "" : item.example;
+					// Find previous execution and their variables
+					//exampledata === "" &&
+					if (workflowExecutions.length > 0) {
+						// Look for the ID
+						const found = false;
+						for (let [key,keyval] in Object.entries(workflowExecutions)) {
+							if (workflowExecutions[key].results === undefined || workflowExecutions[key].results === null) {
+								continue;
+							}
+
+							var foundResult = workflowExecutions[key].results.find((result) => result.action.id === item.id);
+							if (foundResult === undefined || foundResult === null) {
+								continue;
+							}
+
+							if (foundResult.result !== undefined && foundResult.result !== null) {
+								foundResult = foundResult.result
+							}
+
+							const valid = validateJson(foundResult, true)
+							if (valid.valid) {
+								if (valid.result.success === false) {
+									//console.log("Skipping success false autocomplete")
+								} else {
+									exampledata = valid.result;
+									break;
+								}
+							} else {
+								exampledata = foundResult;
+							}
+						}
+					}
+
+					// 1. Take
+					const itemlabelComplete = item.label === null || item.label === undefined ? "" : item.label.split(" ").join("_");
+
+					const actionvalue = {
+						app_name: item.app_name,
+						action_name: item.name,
+						label: item.label,
+
+						type: "action",
+						id: item.id,
+						name: item.label,
+						autocomplete: itemlabelComplete,
+						example: exampledata,
+					};
+
+					actionlist.push(actionvalue);
+				}
+			}
+
+			var fixedResults = []
+			for (var i = 0; i < actionlist.length; i++) {
+				const item = actionlist[i];
+				const responseFix = SetJsonDotnotation(item.example, "") 
+				
+				// Check if json
+				const validated = validateJson(responseFix)
+				var exampledata = responseFix;
+				if (validated.valid) {
+					exampledata = JSON.stringify(validated.result)
+				}
+
+				AppContext.push({
+					"app_name": item.app_name,
+					"action_name": item.action_name,
+					"label": item.label,
+					"example": exampledata,
+				})
+			}
+		}
+
+		var conversationData = {
+			"query": value,
+			"output_format": "action",
+			"app_context": AppContext,
+		}
+
+		if (inputAction !== undefined) {
+			console.log("Add app context! This should them get parameters directly")
+			conversationData.output_format = "action_parameters"
+
+			conversationData.app_id = inputAction.app_id
+			conversationData.app_name = inputAction.app_name
+			conversationData.action_name = inputAction.name
+			conversationData.parameters = inputAction.parameters
+		}
+
+		fetch(`${globalUrl}/api/v1/conversation`, {
+			method: "POST",
+			headers: {
+				"Content-Type": "application/json",
+				Accept: "application/json",
+			},
+			body: JSON.stringify(conversationData),
+			credentials: "include",
+		})
+		.then((response) => {
+			if (response.status !== 200) {
+				console.log("Status not 200 for stream results :O!");
+			}
+
+			return response.json();
+		})
+		.then((responseJson) => {
+			console.log("Conversation response: ", responseJson)
+			setIsAiLoading(false)
+			if (responseJson.success === false) {
+				if (responseJson.reason !== undefined) {
+				}
+
+				return
+			}
+
+			if (inputAction !== undefined) {
+				console.log("In input action! Should check params if they match, and add suggestions")
+
+				if (responseJson.parameters === undefined || responseJson.parameters.length === 0) {
+					return
+				}
+
+				for (let respParam of responseJson.parameters) {
+					if (respParam.name !== parameterName) {
+						continue
+					}
+
+					if (respParam.value === "") {
+						break
+					}
+
+					setlocalcodedata(respParam.value)
+					break
+				}
+
+				return
+			}
+		})
+		.catch((error) => {
+			setIsAiLoading(false)
+			console.log("Conv response error: ", error);
+		});
+	}
+
 	const autoFormat = (input) => {
+		// Check if it's default too
 		if (validation !== true) {
+
+			// Should try to automatically fix this input
+			console.log("Running AI input fixer")
+			if (aiSubmit !== undefined && parameterName !== undefined && selectedAction !== undefined) {
+
+				// Should remove params from selectedAction that aren't parameterName  
+				var tmpAction = JSON.parse(JSON.stringify(selectedAction))
+				var tmpParams = selectedAction.parameters.filter((param) => param.name === parameterName)
+
+				var aiMsg = `Make it valid for action ${tmpAction.label} with parameter ${parameterName}: `
+				if (tmpParams.length > 0) {
+					aiMsg += tmpParams[0].value
+				}
+
+
+				if (localcodedata.startsWith("//")) {
+					aiMsg = localcodedata
+				}
+
+				tmpAction.parameters = tmpParams
+				console.log("Parameters: ", tmpParams.length)
+
+				aiSubmit(aiMsg, tmpAction)
+			}
+
 			return
 		}
 
@@ -295,6 +508,8 @@ const CodeEditor = (props) => {
 		var removedIndexes = 0
 		for (var key in itemsplit) {
 			var tmpitem = itemsplit[key]
+
+			// Makes sure #0 and # are same, as we only visualize first one anyway
 			if (tmpitem.startsWith("#")) {
 				removedIndexes += tmpitem.length-1
 				tmpitem = "#"
@@ -303,7 +518,7 @@ const CodeEditor = (props) => {
 			newitem.push(tmpitem)
 		}
 
-		//console.log("Fixed item: ", newitem, "removed length: ", removedIndexes)
+		console.log("Fixed item: ", newitem, "removed length: ", removedIndexes)
 
 		return newitem.join(".")
 		//return inputvariable
@@ -450,12 +665,17 @@ const CodeEditor = (props) => {
 		const found = input.match(/[$]{1}([a-zA-Z0-9_-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,}/g)
 
 		console.log("FOUND: ", found)
+
+
 		// Whelp this is inefficient af. Single loop pls
 		// When the found array is empty.
 		if (found !== null && found !== undefined) {
 			try { 
 				for (var i = 0; i < found.length; i++) {
 					try {
+						// For found specifically, should replace .#\d with .# with regex
+						
+
 						//found[i] = found[i].toLowerCase()
 						const fixedVariable = fixVariable(found[i])
 						//var correctVariable = availableVariables.includes(fixedVariable)
@@ -468,15 +688,15 @@ const CodeEditor = (props) => {
 
 								try {
 									if (typeof actionlist[j].example === "object") {
-										input = input.replace(fixedVariable, JSON.stringify(actionlist[j].example));
+										input = input.replace(found[i], JSON.stringify(actionlist[j].example), -1);
 
 									} else if (actionlist[j].example.trim().startsWith("{") || actionlist[j].example.trim().startsWith("[")) {
-										input = input.replace(fixedVariable, JSON.stringify(actionlist[j].example));
+										input = input.replace(found[i], JSON.stringify(actionlist[j].example), -1);
 									} else {
-										input = input.replace(fixedVariable, actionlist[j].example)
+										input = input.replace(found[i], actionlist[j].example, -1)
 									}
 								} catch (e) { 
-									input = input.replace(fixedVariable, actionlist[j].example)
+									input = input.replace(found[i], actionlist[j].example, -1)
 								}
 							} else {
 								// Couldn't find the correct example value
@@ -523,7 +743,8 @@ const CodeEditor = (props) => {
 										}
 
 										//console.log("FOUND2: ", fixedVariable, actionlist[j].example)
-										input = input.replace(fixedVariable, new_input)
+										input = input.replace(fixedVariable, new_input, -1)
+										input = input.replace(found[i], new_input, -1)
 
 										//} catch (e) {
 										//	input = input.replace(found[i], actionlist[k].example)
@@ -769,6 +990,7 @@ const CodeEditor = (props) => {
 									height: 50, 
 									width: 50, 
 								}}
+								disabled={isAiLoading}
 								onClick={() => {
 									autoFormat(localcodedata) 
 								}}
@@ -778,7 +1000,11 @@ const CodeEditor = (props) => {
 									title={"Auto format data"}
 									placement="top"
 								>
+									{isAiLoading ? 
+										<CircularProgress style={{height: 20, width: 20, color: "rgba(255,255,255,0.7)"}}/>
+										:
 										<AutoFixHighIcon style={{color: "rgba(255,255,255,0.7)"}}/>
+									}
 								</Tooltip>
 							</IconButton>
 						</div>

frontend/src/views/Admin.jsx

@@ -5,10 +5,11 @@ import { useNavigate, Link } from "react-router-dom";
 import countries from "../components/Countries.jsx";
 import CodeEditor from "../components/ShuffleCodeEditor.jsx";
 import getLocalCodeData from "../components/ShuffleCodeEditor.jsx";
-
+import CacheView from "../components/CacheView.jsx";
 import theme from "../theme";
 import AddIcon from "@mui/icons-material/Add";
 import ClearIcon from '@mui/icons-material/Clear';
+import StorageIcon from '@mui/icons-material/Storage';
 //import ToggleButton from '@mui/material/ToggleButton';
 import {
   FormControl,
@@ -81,6 +82,7 @@ import Billing from "../components/Billing.jsx";
 import Branding from "../components/Branding.jsx";
 import Files from "../components/Files.jsx";
 import { display, style } from "@mui/system";
+//import EnvironmentStats from "../components/EnvironmentStats.jsx";
 
 const useStyles = makeStyles({
   notchedOutline: {
@@ -695,13 +697,15 @@ const Admin = (props) => {
 
   const handleGetOrg = (orgId) => {
     if (orgId.length === 0) {
-      alert.error("Organization ID not defined. Please contact us on https://shuffler.io if this persists logout.");
+      alert.error(
+        "Organization ID not defined. Please contact us on https://shuffler.io if this persists logout."
+      );
       return;
     }
 
     // Just use this one?
-    const url = `${globalUrl}/api/v1/orgs/${orgId}`
+    
-    fetch(url, {
+    fetch(`${globalUrl}/api/v1/orgs/${orgId}`, {
       method: "GET",
       credentials: "include",
       headers: {
@@ -716,11 +720,7 @@ const Admin = (props) => {
       })
       .then((responseJson) => {
         if (responseJson["success"] === false) {
-					if (responseJson.reason !== undefined) {
-						alert.error(responseJson.reason);
-					} else {
           alert.error("Failed getting your org. If this persists, please contact support.");
-					}
         } else {
           if (
             responseJson.sync_features === undefined ||
@@ -1258,9 +1258,10 @@ const Admin = (props) => {
     1: "users",
     2: "app_auth",
     3: "files",
-    4: "schedules",
+    4: "cache",
-    5: "environments",
+    5: "schedules",
-    6: "suborgs",
+    6: "environments",
+    7: "suborgs",
   };
 
   const admin_views = {
@@ -1268,6 +1269,7 @@ const Admin = (props) => {
     1: "cloud_sync",
     2: "billing",
     3: "branding",
+    4: "cache",
   };
 
   const setConfig = (event, inputValue) => {
@@ -1281,21 +1283,25 @@ const Admin = (props) => {
       document.title = "Shuffle - admin - app authentication";
       getAppAuthentication();
     } else if (newValue === 3) {
-      document.title = "Shuffle - admin - files";
+      document.title = "Shuffle - admin - Files";
     } else if (newValue === 4) {
+      document.title = "Shuffle - admin - Datastore";
+
+      //listOrgCache("3fd181b9-fb29-41b7-b2f5-15292265d420");
+    } else if (newValue === 5) {
       document.title = "Shuffle - admin - schedules";
       getSchedules();
-    } else if (newValue === 5) {
+    } else if (newValue === 6) {
       document.title = "Shuffle - admin - environments";
       getEnvironments();
-    } else if (newValue === 6) {
+    } else if (newValue === 7) {
       document.title = "Shuffle - admin - orgs";
       getOrgs();
     } else {
       document.title = "Shuffle - admin";
     }
 
-    if (newValue === 6) {
+    if (newValue === 8) {
       console.log("Should get apps for categories.");
     }
 
@@ -1394,7 +1400,8 @@ const Admin = (props) => {
         if (!responseJson.success && responseJson.reason !== undefined) {
           alert.error("Failed setting user: " + responseJson.reason);
         } else {
-          alert.success("Set the user field " + field + " to " + value);
+          //alert.success("Set the user field " + field + " to " + value);
+          alert.success("Successfully updated user field " + field)
 
           if (field !== "suborgs") {
             setSelectedUserModalOpen(false);
@@ -2407,6 +2414,7 @@ const Admin = (props) => {
 									If not otherwise specified, Usage will reset monthly
             		</Typography>
             		<Grid container style={{ width: "100%", marginBottom: 15 }}>
+
             		  {selectedOrganization.sync_features === undefined ||
             		  selectedOrganization.sync_features === null
             		    ? null
@@ -2507,11 +2515,7 @@ const Admin = (props) => {
     >
       <DialogTitle>
         <span style={{ color: "white" }}>
-          {curTab === 1
+          {curTab === 1 ? "Add user" : curTab === 7 ? "Add Sub-Organization" : "Add environment"}
-            ? "Add user"
-            : curTab === 6
-            ? "Add Sub-Organization"
-            : "Add environment"}
         </span>
       </DialogTitle>
       <DialogContent>
@@ -2519,7 +2523,7 @@ const Admin = (props) => {
           <Typography variant="body1" style={{ marginBottom: 10 }}>
             We will send an email to invite them to your organization.
           </Typography>
-        ) : curTab === 6 ? (
+        ) : curTab === 7 ? (
           <Typography variant="body1" style={{ marginBottom: 10 }}>
             The organization created will become a child of your current
             organization, and be available to you.
@@ -2578,7 +2582,7 @@ const Admin = (props) => {
               </span>
             )}
           </div>
-        ) : curTab === 6 ? (
+        ) : curTab === 7 ? (
           <div>
             Name
             <TextField
@@ -2603,7 +2607,7 @@ const Admin = (props) => {
               }}
             />
           </div>
-        ) : curTab === 5 ? (
+        ) : curTab === 6 ? (
           <div>
             Environment Name
             <TextField
@@ -2649,9 +2653,9 @@ const Admin = (props) => {
               } else {
                 submitUser(modalUser);
               }
-            } else if (curTab === 6) {
+            } else if (curTab === 7) {
               createSubOrg(selectedOrganization.id, orgName);
-            } else if (curTab === 5) {
+            } else if (curTab === 6) {
               submitEnvironment(modalUser);
             }
           }}
@@ -3002,7 +3006,7 @@ const Admin = (props) => {
 		/>
 
   const schedulesView =
-    curTab === 4 ? (
+    curTab === 5 ? (
       <div>
         <div style={{ marginTop: 20, marginBottom: 20 }}>
           <h2 style={{ display: "inline" }}>Schedules</h2>
@@ -3108,7 +3112,7 @@ const Admin = (props) => {
     ) : null;
 
   const appCategoryView =
-    curTab === 7 ? (
+    curTab === 8 ? (
       <div>
         <div style={{ marginTop: 20, marginBottom: 20 }}>
           <h2 style={{ display: "inline" }}>Categories</h2>
@@ -3434,7 +3438,7 @@ const Admin = (props) => {
     ) : null;
 
   const environmentView =
-    curTab === 5 ? (
+    curTab === 6 ? (
       <div>
         <div style={{ marginTop: 20, marginBottom: 20 }}>
           <h2 style={{ display: "inline" }}>Environments</h2>
@@ -3697,11 +3701,12 @@ const Admin = (props) => {
                 );
               })}
         </List>
+				{/*<EnvironmentStats />*/}
       </div>
     ) : null;
 
   const organizationsTab =
-    curTab === 6 ? (
+    curTab === 7 ? (
       <div>
         <div style={{ marginTop: 20, marginBottom: 20 }}>
           <h2 style={{ display: "inline" }}>Organizations</h2>
@@ -3831,7 +3836,7 @@ const Admin = (props) => {
     ) : null;
 
   const hybridTab =
-    curTab === 7 ? (
+    curTab === 8 ? (
       <div>
         <div style={{ marginTop: 20, marginBottom: 20 }}>
           <h2 style={{ display: "inline" }}>Hybrid</h2>
@@ -3882,6 +3887,16 @@ const Admin = (props) => {
       </div>
     ) : null;
 
+    const cacheOrgView =
+    curTab === 4 ? (
+      <div>
+        <CacheView
+					globalUrl={globalUrl}
+					orgId = {selectedOrganization.id}
+        />
+      </div>
+    ) : null;
+
   // primary={environment.Registered ? "true" : "false"}
 
   const iconStyle = { marginRight: 10 };
@@ -3901,6 +3916,8 @@ const Admin = (props) => {
           textColor="secondary"
           onChange={setConfig}
           aria-label="disabled tabs example"
+          variant="scrollable"
+          scrollButtons="auto"
         >
           <Tab
             label=<span>
@@ -3927,6 +3944,11 @@ const Admin = (props) => {
               Files
             </span>
           />
+          <Tab
+            label=<span>
+              <StorageIcon style={iconStyle} /> Datastore 
+            </span>
+          />
           <Tab
             disabled={userdata.admin !== "true"}
             label=<span>
@@ -3935,7 +3957,6 @@ const Admin = (props) => {
             </span>
           />
           <Tab
-            index={5}
             disabled={userdata.admin !== "true"}
             label=<span>
               <EcoIcon style={iconStyle} />
@@ -3943,12 +3964,11 @@ const Admin = (props) => {
             </span>
           />
           <Tab
-            index={6}
-            value={6}
             label=<span>
               <BusinessIcon style={iconStyle} /> Tenants
             </span>
           />
+
           {/*window.location.protocol == "http:" && window.location.port === "3000" ? <Tab label=<span><CloudIcon style={iconStyle} /> Hybrid</span>/> : null*/}
           {/*window.location.protocol === "http:" && window.location.port === "3000" ? <Tab label=<span><LockIcon style={iconStyle} />Categories</span>/> : null*/}
         </Tabs>
@@ -3969,6 +3989,7 @@ const Admin = (props) => {
           {hybridTab}
           {organizationsTab}
           {appCategoryView}
+          {cacheOrgView}
         </div>
       </Paper>
     </div>

frontend/src/views/AngularWorkflow.jsx

@@ -107,6 +107,7 @@ import {
   VpnKey as VpnKeyIcon,
   AddComment as AddCommentIcon,
   Edit as EditIcon,
+	Send as SendIcon,
 } from "@material-ui/icons";
 
 import {
@@ -114,6 +115,7 @@ import {
   ContentCopy as ContentCopyIcon,
   Circle as  CircleIcon,
 	SquareFoot as SquareFootIcon,
+	AutoFixHigh as AutoFixHighIcon,
 } from '@mui/icons-material';
 
 import Autocomplete from "@material-ui/lab/Autocomplete";
@@ -282,6 +284,51 @@ export function sortByKey(array, key) {
   });
 }
 
+// look for keys and set values with shuffle dotnotation 
+// used primarily for AI autocompletions
+export function SetJsonDotnotation(jsonInput, inputKey) {
+
+	if (jsonInput === undefined || jsonInput === null) {
+		return jsonInput;
+	}
+
+	// Check for array
+	if (Array.isArray(jsonInput)) {
+		for (var i = 0; i < jsonInput.length; i++) {
+			jsonInput[i] = SetJsonDotnotation(jsonInput[i], inputKey+".#");
+		}
+
+		return jsonInput;
+		// Check for dict
+	} else if (typeof jsonInput === "object") {
+		// Loop keys and values
+	
+		for (var key in jsonInput) {
+			if (!jsonInput.hasOwnProperty(key)) {
+				continue
+			}
+
+			const value = jsonInput[key];
+			// Check if array
+			if (Array.isArray(value)) {
+				for (var i = 0; i < value.length; i++) {
+					jsonInput[key][i] = SetJsonDotnotation(jsonInput[key][i], inputKey+"."+key+".#");
+				}
+			} else if (typeof value === "object") {
+				jsonInput[key] = SetJsonDotnotation(jsonInput[key], inputKey+"."+key);
+			} else {
+				jsonInput[key] = inputKey+"."+key
+			}
+		}
+	} else {
+		//jsonInput = inputKey
+
+		console.log("SetJsonDotnotation: jsonInput is not an object or array, but key ", jsonInput, typeof jsonInput);
+	}
+
+	return jsonInput;
+}
+
 export const green = "#86c142";
 export const yellow = "#FECC00";
 
@@ -417,8 +464,7 @@ const AngularWorkflow = (defaultprops) => {
   const [rightSideBarOpen, setRightSideBarOpen] = React.useState(false);
   const [showSkippedActions, setShowSkippedActions] = React.useState(false);
   const [lastExecution, setLastExecution] = React.useState("");
-  const [configureWorkflowModalOpen, setConfigureWorkflowModalOpen] =
+  const [configureWorkflowModalOpen, setConfigureWorkflowModalOpen] = React.useState(false);
-    React.useState(false);
 
   const curpath =
     typeof window === "undefined" || window.location === undefined
@@ -486,6 +532,18 @@ const AngularWorkflow = (defaultprops) => {
   const [defaultEnvironmentIndex, setDefaultEnvironmentIndex] = React.useState(0);
   const [workflowRecommendations, setWorkflowRecommendations] = React.useState([]);
 
+	const [listCache, setListCache] = React.useState([]);
+	const [suggestionBox, setSuggestionBox] = React.useState({
+		"position": {
+			"top": 500,
+			"left": 500,
+		},
+		"open": false,
+		"attachedTo": "",
+	});
+
+	
+
   // This should all be set once, not on every iteration
   // Use states and don't update lol
   const cloudSyncEnabled =
@@ -565,6 +623,31 @@ const AngularWorkflow = (defaultprops) => {
     }
   }, [authenticationModalOpen])
 
+	const listOrgCache = (orgId) => {
+		fetch(`${globalUrl}/api/v1/orgs/${orgId}/list_cache`, {
+				method: "GET",
+				headers: {
+						"Content-Type": "application/json",
+						Accept: "application/json",
+				},
+				credentials: "include",
+		})
+		.then((response) => {
+				if (response.status !== 200) {
+						console.log("Status not 200 for apps :O!");
+						return;
+				}
+
+				return response.json();
+		})
+		.then((responseJson) => {
+				setListCache(responseJson);
+		})
+		.catch((error) => {
+				alert.error(error.toString());
+		});
+	};
+
   const getAvailableWorkflows = (trigger_index) => {
     fetch(globalUrl + "/api/v1/workflows", {
       method: "GET",
@@ -2718,10 +2801,60 @@ const AngularWorkflow = (defaultprops) => {
   const onNodeSelect = (event, newAppAuth) => {
     // Forces all states to update at the same time,
 		// Otherwise everything is SUPER slow
+
 		ReactDOM.unstable_batchedUpdates(() => {
       const data = event.target.data();
       if (data.isButton) {
-        if (data.buttonType === "delete") {
+        if (data.buttonType === "suggestion") {
+					if (cy === undefined) {
+						console.log("Cy not defined yet")
+						return
+					}
+
+					// Inject HTML at a fixed location
+					//const newHtml = "<div id='suggestion' style='position: absolute; top: 0; left: 0; width: 100%; height: 100%; background-color: white; z-index: 1000;'><div style='position: absolute; top: 50%; left: 50%; transform: translate(-50%, -50%);'><h1>Do you want to add this suggestion?</h1><button id='suggestion_yes'>Yes</button><button id='suggestion_no'>No</button></div></div>"
+	
+					// Find mouse cursor position on screen
+					console.log("Suggestion html to be added at location: ", event)
+					/*
+					const position = {
+						"top": cy.pan().y, 
+						"left": cy.pan().x, 
+					}
+					*/
+
+
+					const position = event.target.renderedPosition();
+					const container = cy.container();
+					const offset = {
+  					left: container.offsetLeft,
+  					top: container.offsetTop
+					};
+					
+					// Calculate the actual screen position for the box
+					const screenPosition = {
+					  left: position.x + offset.left - 150,
+					  top: position.y + offset.top,
+					};
+					
+					// Log the position to the console
+					console.log('Node screen position:', screenPosition);
+
+					const newbox = {
+						"position": screenPosition,
+						"node_position": event.target.position(),
+						"open": true,
+						"attachedTo": data.attachedTo,
+					}
+
+					console.log("Rendered position: ", newbox.node_position)
+
+					setSuggestionBox(newbox)
+
+					// Unselect
+        	event.target.unselect();
+					
+        } else if (data.buttonType === "delete") {
           const parentNode = cy.getElementById(data.attachedTo);
           if (parentNode !== null && parentNode !== undefined) {
             removeNode(data.attachedTo)
@@ -3187,6 +3320,14 @@ const AngularWorkflow = (defaultprops) => {
 
       console.log("DOne in the node update")
 
+			setSuggestionBox({
+				"position": {
+					"top": 500,
+					"left": 500,
+				},
+				"open": false,
+				"attachedTo": "",
+			});
       sendStreamRequest({
         "item": "node",
         "type": "select",
@@ -3582,7 +3723,7 @@ const AngularWorkflow = (defaultprops) => {
     }
 
     const eventTarget = event.target.target()
-    console.log("BUTTON ADDED! Find parent from: ", eventTarget)
+    //console.log("BUTTON ADDED! Find parent from: ", eventTarget)
     if (eventTarget.data("isButton") === true) {
       const parentNode = cy.getElementById(eventTarget.data("attachedTo"))
       event.target.remove()
@@ -4211,6 +4352,8 @@ const AngularWorkflow = (defaultprops) => {
 					item = newitem
 					item.type = "ACTION"
 					item.isStartNode = false
+					item.data.type = "ACTION"
+					item.data.isStartNode = false
 				}
 
 				item.data.id = uuidv4()
@@ -4473,7 +4616,46 @@ const AngularWorkflow = (defaultprops) => {
   };
 
   const addSuggestionButtons = (nodedata, event) => {
-		console.log("In suggestion buttons: ", nodedata, ". This is not enabled yet. Backend needs further work.")
+		//console.log("In suggestion buttons: ", nodedata, ". This is not enabled yet. Backend needs further work.")
+		// Skipping add for now. Should Re-enable
+
+		// Add a button for autocompletion based on input
+		console.log("Type: ", nodedata.type)
+		if (nodedata.type === "ACTION") {
+			const color = "#34a853" 
+
+			// Fix icon
+    	const iconInfo = {
+    	  icon: "M7.5 5.6 10 7 8.6 4.5 10 2 7.5 3.4 5 2l1.4 2.5L5 7zm12 9.8L17 14l1.4 2.5L17 19l2.5-1.4L22 19l-1.4-2.5L22 14zM22 2l-2.5 1.4L17 2l1.4 2.5L17 7l2.5-1.4L22 7l-1.4-2.5zm-7.63 5.29a.9959.9959 0 0 0-1.41 0L1.29 18.96c-.39.39-.39 1.02 0 1.41l2.34 2.34c.39.39 1.02.39 1.41 0L16.7 11.05c.39-.39.39-1.02 0-1.41l-2.33-2.35zm-1.03 5.49-2.12-2.12 2.44-2.44 2.12 2.12-2.44 2.44z",
+    	  iconColor: buttonColor,
+    	  iconBackgroundColor: buttonBackgroundColor,
+    	};
+
+    	const svg_pin = `<svg width="${svgSize}" height="${svgSize}" viewBox="0 0 ${svgSize} ${svgSize}" version="1.1" xmlns="http://www.w3.org/2000/svg"><path d="${iconInfo.icon}" fill="${iconInfo.iconColor}"></path></svg>`;
+    	const svgpin_Url = encodeURI("data:image/svg+xml;utf-8," + svg_pin);
+
+			const decoratorNode = {
+				position: {
+					x: event.target.position().x + 0,
+					y: event.target.position().y + 65,
+				},
+				locked: true,
+				data: {
+					isButton: true,
+					isValid: true,
+					is_valid: true,
+					//label: "+",
+					attachedTo: nodedata.id,
+					imageColor: color,
+        	buttonType: "suggestion", 
+					icon: svgpin_Url,
+					iconBackground: iconInfo.iconBackgroundColor,
+				},
+			};
+
+			cy.add(decoratorNode);
+		}
+
 		return
 	
 		//setWorkflowRecommendations(responseJson.actions)
@@ -7199,7 +7381,6 @@ const AngularWorkflow = (defaultprops) => {
     setSelectedAction(newSelectedAction);
     setUpdate(Math.random());
 
-    // FIXME - should change icon-node (descriptor) as well
     const allNodes = cy.nodes().jsons();
 		if (allNodes !== undefined && allNodes !== null) {
 			for (let nodekey in allNodes) {
@@ -7221,6 +7402,11 @@ const AngularWorkflow = (defaultprops) => {
 				}
 			}
 		}
+
+		// Send it in here, after all fields are filled
+		// Disabled for now :(
+		// const aiMsg = "Fill based on previous values"
+		// aiSubmit(aiMsg, undefined, undefined, newSelectedAction)
 	};
 
   // APPSELECT at top
@@ -10905,7 +11091,6 @@ const AngularWorkflow = (defaultprops) => {
                   fullWidth
                   multiline
                   rows="2"
-                  defaultValue={trigger_header_auth}
                   color="primary"
                   disabled={selectedTrigger.status === "running"}
                   placeholder={"OK"}
@@ -12612,6 +12797,8 @@ const AngularWorkflow = (defaultprops) => {
         requiresAuthentication={requiresAuthentication}
         setLastSaved={setLastSaved}
         lastSaved={lastSaved}
+
+				aiSubmit={aiSubmit}
       />
 
     } else if (Object.getOwnPropertyNames(selectedComment).length > 0) {
@@ -12929,6 +13116,9 @@ const AngularWorkflow = (defaultprops) => {
               getSettings();
               getFiles()
 
+							// For loading datastore
+							// listOrgCache(workflow.org_id) 
+
               setUpdate(Math.random());
             }}
           >
@@ -14566,8 +14756,8 @@ const AngularWorkflow = (defaultprops) => {
         <div style={{ display: "flex", marginBottom: 15 }}>
           {curapp === null ? null : (
             <img
-              alt={selectedResult.app_name}
+              alt={selectedResult.action.app_name}
-              src={curapp === undefined ? theme.palette.defaultImage : curapp.app_name === "shuffle-subflow" ? triggers[4].large_image : curapp.app_name === "User Input" ? triggers[5].large_image : curapp.large_image}
+              src={selectedResult === undefined ? theme.palette.defaultImage : selectedResult.action.app_name === "shuffle-subflow" ? triggers[4].large_image : selectedResult.action.app_name === "User Input" ? triggers[5].large_image : curapp.large_image}
               style={{
                 marginRight: 20,
                 width: imgsize,
@@ -14596,6 +14786,15 @@ const AngularWorkflow = (defaultprops) => {
         <div style={{ marginBottom: 5 }}>
           <b>Status </b> {selectedResult.status}
         </div>
+
+
+					<h1 onClick={() => {
+						console.log("APP: ", curapp)
+						console.log("Data: ", selectedResult)
+					}}>
+						HIYA: {selectedResult.action.app_name}
+					</h1>
+
         {validate.valid ? (
           <ReactJson
             src={validate.result}
@@ -14613,8 +14812,12 @@ const AngularWorkflow = (defaultprops) => {
           />
         ) : (
           <div>
-            <b>Result</b>&nbsp;
+            <b>Result</b>
+						<br/>
             <span
+							style={{
+								wordBreak: "break-word",
+							}}
               onClick={() => {
                 console.log("IN HERE TO CLICK");
                 to_be_copied = selectedResult.result;
@@ -15734,6 +15937,358 @@ const AngularWorkflow = (defaultprops) => {
       </div>
     </Dialog>
   ) : null;
+
+	// Should get AI autocompletes
+	const aiSubmit = (value, setResponseMsg, setSuggestionLoading, inputAction) => {
+		if (setResponseMsg !== undefined) {
+			setResponseMsg("")
+		}
+
+		if (value === undefined || value === "") {
+			console.log("No value input!")
+			return
+		}
+
+		if (setSuggestionLoading !== undefined) {
+			setSuggestionLoading(true)
+		}
+	
+		console.log("Submit conversation with value: ", value);
+
+		// This is to find sample response and parse it as string
+		
+		var AppContext = []
+		if (inputAction !== undefined && inputAction !== null) {
+			const parents = getParents(inputAction)
+
+			console.log("Parents: ", parents)
+			var actionlist = []
+			if (parents.length > 1) {
+				for (let [key,keyval] in Object.entries(parents)) {
+					const item = parents[key];
+					if (item.label === "Execution Argument") {
+						continue;
+					}
+
+					var exampledata = item.example === undefined || item.example === null ? "" : item.example;
+					// Find previous execution and their variables
+					//exampledata === "" &&
+					if (workflowExecutions.length > 0) {
+						// Look for the ID
+						const found = false;
+						for (let [key,keyval] in Object.entries(workflowExecutions)) {
+							if (workflowExecutions[key].results === undefined || workflowExecutions[key].results === null) {
+								continue;
+							}
+
+							var foundResult = workflowExecutions[key].results.find((result) => result.action.id === item.id);
+							if (foundResult === undefined || foundResult === null) {
+								continue;
+							}
+
+							if (foundResult.result !== undefined && foundResult.result !== null) {
+								foundResult = foundResult.result
+							}
+
+							const valid = validateJson(foundResult, true)
+							if (valid.valid) {
+								if (valid.result.success === false) {
+									//console.log("Skipping success false autocomplete")
+								} else {
+									exampledata = valid.result;
+									break;
+								}
+							} else {
+								exampledata = foundResult;
+							}
+						}
+					}
+
+					// 1. Take
+					const itemlabelComplete = item.label === null || item.label === undefined ? "" : item.label.split(" ").join("_");
+
+					const actionvalue = {
+						app_name: item.app_name,
+						action_name: item.name,
+						label: item.label,
+
+						type: "action",
+						id: item.id,
+						name: item.label,
+						autocomplete: itemlabelComplete,
+						example: exampledata,
+					};
+
+					actionlist.push(actionvalue);
+				}
+			}
+
+			var fixedResults = []
+			for (var i = 0; i < actionlist.length; i++) {
+				const item = actionlist[i];
+				const responseFix = SetJsonDotnotation(item.example, "") 
+				
+				// Check if json
+				const validated = validateJson(responseFix)
+				var exampledata = responseFix;
+				if (validated.valid) {
+					exampledata = JSON.stringify(validated.result)
+				}
+
+				AppContext.push({
+					"app_name": item.app_name,
+					"action_name": item.action_name,
+					"label": item.label,
+					"example": exampledata,
+				})
+			}
+		}
+
+		var conversationData = {
+			"query": value,
+			"output_format": "action",
+			"app_context": AppContext,
+
+			"workflow_id": workflow.id,
+		}
+
+		if (inputAction !== undefined) {
+			console.log("Add app context! This should them get parameters directly")
+			conversationData.output_format = "action_parameters"
+
+			conversationData.app_id = inputAction.app_id
+			conversationData.app_name = inputAction.app_name
+			conversationData.action_name = inputAction.name
+			conversationData.parameters = inputAction.parameters
+
+			if (!value.includes(inputAction.label)) {
+				conversationData.query = inputAction.label.replace("_", " ", -1)
+			}
+		}
+
+		// Onprem not available yet (April 2023)
+		// Should: Make OpenAI work for them with their own key
+		//fetch("https://shuffler.io/api/v1/conversation", {
+		fetch(`${globalUrl}/api/v1/conversation`, {
+			method: "POST",
+			headers: {
+				"Content-Type": "application/json",
+				Accept: "application/json",
+			},
+			body: JSON.stringify(conversationData),
+			credentials: "include",
+		})
+		.then((response) => {
+			if (setSuggestionLoading !== undefined) {
+				setSuggestionLoading(false)
+			}
+
+			if (response.status !== 200) {
+				console.log("Status not 200 for stream results :O!");
+			}
+
+			return response.json();
+		})
+		.then((responseJson) => {
+			console.log("Conversation response: ", responseJson)
+			if (responseJson.success === false) {
+				if (responseJson.reason !== undefined) {
+					if (setResponseMsg !== undefined) {
+						setResponseMsg(responseJson.reason)
+					}
+				}
+
+				return
+			}
+
+			if (inputAction !== undefined) {
+				console.log("In input action! Should check params if they match, and add suggestions")
+
+				if (responseJson.parameters === undefined || responseJson.parameters.length === 0) {
+					return
+				}
+
+				var changed = false
+
+				for (let paramkey in inputAction.parameters) {
+					const actionParam = inputAction.parameters[paramkey]
+
+					if (actionParam.autocompleted === true) {
+						continue
+					}
+
+					if (actionParam.configuration === true && actionParam.name !== "url") {
+						continue
+					}
+
+					if (actionParam.value !== "" && actionParam.value !== actionParam.example) {
+						console.log("Skipping: ", actionParam)
+						continue
+					}
+
+					for (let respParam of responseJson.parameters) {
+						if (respParam.name === actionParam.name) {
+							console.log("Found match for param: ", respParam)
+
+							if (respParam.value === "") {
+								break
+							}
+
+							changed = true
+			
+							inputAction.parameters[paramkey].autocompleted = true
+							inputAction.parameters[paramkey].value = respParam.value
+							break
+						}
+					}
+				}
+
+				if (changed === true) {
+					console.log("Setting action! Force update pls :)")
+					setUpdate(Math.random())
+					setSelectedAction(inputAction)
+				}
+
+				return
+			}
+
+			console.log("Suggestionbox location: ", suggestionBox)
+
+			// Add action
+			if (responseJson.app_name !== undefined && responseJson.app_name !== null) {
+				// Always added to 0, 0
+				// Should use suggestionBox.position.x, suggestionBox.position.y
+				var newitem = {
+					"data": responseJson,
+					"position": {
+						"x": suggestionBox.node_position.x !== undefined ? suggestionBox.node_position.x : 0,
+						"y": suggestionBox.node_position.y !== undefined ? suggestionBox.node_position.y + 100 : 0,
+					},
+					"group": "nodes",
+				}
+
+				newitem.type = "ACTION"
+				newitem.isStartNode = false
+				newitem.data.id = uuidv4()
+				newitem.data.type = "ACTION"
+				newitem.data.isStartNode = false
+
+				newitem.data.is_valid = true
+				newitem.data.isValid = true
+
+				cy.add({
+					group: 	newitem.group,
+					data: 	newitem.data,
+					position: newitem.position,
+				});
+
+				// Add edge
+				const newId = uuidv4()
+				cy.add({
+					group: "edges",
+					data: {
+						id: newId,
+						_id: newId,
+						source: suggestionBox.attachedTo,
+						target: newitem.data.id,
+					}
+				})
+				//label: "Generated",
+		
+				setSuggestionBox({
+					"position": {
+						"top": 500,
+						"left": 500,
+					},
+					"open": false,
+					"attachedTo": "",
+				});
+			}
+		})
+		.catch((error) => {
+			if (setSuggestionLoading !== undefined) {
+				setSuggestionLoading(false)
+			}
+
+			console.log("Conv response error: ", error);
+		});
+	}
+
+	const SuggestionBoxUi = () => {
+		const [suggestionValue, setSuggestionValue] = useState("");
+		const [suggestionLoading, setSuggestionLoading] = useState(false);
+		const [responseMsg, setResponseMsg] = useState("");
+
+		if (suggestionBox === undefined || suggestionBox.open === false) {
+			return false
+		}
+
+		return ( 
+			<div style={{width: 350, padding: 15, position: "fixed", top: suggestionBox.position.top, left: suggestionBox.position.left, borderRadius: theme.palette.borderRadius, backgroundColor: theme.palette.surfaceColor, border: "1px solid rgba(255,255,255,0.3)", }}>
+				{/*
+				<AutoFixHighIcon style={{height: 12, width: 12, color: "white", position: "absolute", top: 10, right: 24, }} />
+				*/}
+				<Tooltip
+					title="Close"
+					placement="top"
+					style={{ zIndex: 10011 }}
+				>
+					<IconButton
+						style={{ zIndex: 5000, position: "absolute", top: 0, right: 0}}
+						onClick={(e) => {
+							e.preventDefault();
+							setSuggestionBox({
+								"position": {
+									"top": 500,
+									"left": 500,
+								},
+								"open": false,
+								"value": "",
+								"loading": false,
+							});
+						}}
+					>
+						<CloseIcon style={{ color: "white", height: 12, width: 12, }} />
+					</IconButton>
+				</Tooltip>
+				<form onSubmit={(e, value) => {
+					e.preventDefault();
+					aiSubmit(suggestionValue, setResponseMsg, setSuggestionLoading)
+				}}>
+					<TextField
+						id="suggestion-textfield"
+						style={{width: "90%"}}
+						disabled={suggestionLoading}
+						label="What action do you want to add?"
+						onChange={(e) => {
+							setSuggestionValue(e.target.value)
+						}}
+						InputProps={{
+							endAdornment: (
+								<InputAdornment position="end">
+									<Tooltip title="Run search" placement="top">
+										<SendIcon style={{ cursor: "pointer" }} onClick={(e) => {
+											e.preventDefault();
+											aiSubmit(suggestionValue, setResponseMsg, setSuggestionLoading)
+										}} />
+									</Tooltip>
+								</InputAdornment>
+							),
+						}}
+					/>
+				</form>
+				{suggestionLoading === true ?
+					<CircularProgress style={{height: 15, width: 15, marginTop: 5, }} />
+				: null}
+				{responseMsg.length > 0 ?	
+					<Typography variant="body2">
+						{responseMsg}
+					</Typography>
+				: null}
+			</div>
+		)
+	}
+
   const loadedCheck =
     isLoaded && workflowDone ? (
       <div>
@@ -15746,6 +16301,8 @@ const AngularWorkflow = (defaultprops) => {
         {configureWorkflowModal}
         {editWorkflowModal}
 
+				<SuggestionBoxUi />
+
         {editWorkflowModalOpen === true ?
           <EditWorkflow
             workflow={workflow}

frontend/src/views/AppCreator.jsx

@@ -300,7 +300,7 @@ const getJsonObject = (properties) => {
 
 				const jsonret = getJsonObject(property.items.properties);
 				if (property.type === "array") {
-					console.log("ARRAY!!")
+					//console.log("ARRAY!!")
 					jsonObject[key] = [jsonret];
 				} else {
 					jsonObject[key] = jsonret;
@@ -309,13 +309,13 @@ const getJsonObject = (properties) => {
 				if (property.hasOwnProperty("properties")) {
 					const jsonret = getJsonObject(property.properties);
 					if (property.type === "array") {
-						console.log("ARRAY2!!")
+						//console.log("ARRAY2!!")
 						jsonObject[key] = [jsonret];
 					} else {
 						jsonObject[key] = jsonret;
 					}
 				} else { 
-					console.log("No items or properties found: ", property);
+					//console.log("No items or properties found: ", property);
 				}
 			}
 
@@ -426,9 +426,7 @@ const AppCreator = (defaultprops) => {
   const [categories, setCategories] = useState(appCategories)
   
 
-  const isCloud =
+  const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io";
-    window.location.host === "localhost:3002" ||
-    window.location.host === "shuffler.io";
 
   useEffect(() => {
 		if (window.location.pathname.includes("apps/edit")) {
@@ -612,8 +610,16 @@ const AppCreator = (defaultprops) => {
 
     setBasedata(data);
 		console.log("Info: ", data)
+
+		try { 
 			if (data.info !== null && data.info !== undefined) {
 				if (data.info.title !== undefined && data.info.title !== null) {
+					if (data.info.title.endsWith(" API")) {
+						data.info.title = data.info.title.substring(0, data.info.title.length - 4)
+					} else if (data.info.title.endsWith("API")) {
+						data.info.title = data.info.title.substring(0, data.info.title.length - 3)
+					}
+
 					if (data.info.title.length > 29) {
 						setName(data.info.title.slice(0, 29));
 					} else {
@@ -649,7 +655,12 @@ const AppCreator = (defaultprops) => {
 					setNewWorkflowCategories(data.info["x-categories"]);
 				}
 			}
+		} catch (e) {
+			console.log("Failed setting info: ", e)
+		}
 
+		console.log("Tags: ", data.tags)
+		try {
 			if (data.tags !== undefined && data.tags.length > 0) {
 				var newtags = [];
 				for (let tagkey in data.tags) {
@@ -668,16 +679,34 @@ const AppCreator = (defaultprops) => {
 
 				setNewWorkflowTags(newtags);
 			}
+		} catch (e) {
+			console.log("Failed to parse tags: ", e)
+		}
 
     // This is annoying (:
-    var securitySchemes = data.components.securityDefinitions;
+		console.log("Security schemes 1: ", data.securitySchemes)
+
+		// Weird generator problems to be handle
+		var securitySchemes = undefined
+		try { 
+			if (data.securitySchemes !== undefined) {
+				securitySchemes = data.securitySchemes
 				if (securitySchemes === undefined) {
-      securitySchemes = data.securitySchemes;
+					securitySchemes = data.securityDefinitions;
+				}
 			}
 			
-    if (securitySchemes === undefined) {
+			if (securitySchemes === undefined && data.components !== undefined) { 
 				securitySchemes = data.components.securitySchemes;
+				if (securitySchemes === undefined) {
+					securitySchemes = data.components.securityDefinitions;
 				}
+			}
+		} catch (e) {
+			console.log("Failed to parse security schemes: ", e)
+		}
+
+		console.log("Security schemes 2: ", securitySchemes)
 
     const allowedfunctions = [
       "GET",
@@ -693,8 +722,9 @@ const AppCreator = (defaultprops) => {
     var newActions = [];
     var wordlist = {};
     var all_categories = [];
-		console.log("Paths: ", data.paths)
 		var parentUrl = ""
+
+		console.log("Paths: ", data.paths)
     if (data.paths !== null && data.paths !== undefined) {
       for (let [path, pathvalue] of Object.entries(data.paths)) {
 
@@ -708,7 +738,7 @@ const AppCreator = (defaultprops) => {
             // Typical YAML issue
             if (method !== "parameters") {
               console.log("Invalid method: ", method, "data: ", methodvalue);
-              alert.info("Skipped method (not allowed): " + method);
+              //alert.info("Skipped method (not allowed): " + method);
             }
             continue;
           }
@@ -830,7 +860,7 @@ const AppCreator = (defaultprops) => {
 										if (methodvalue["requestBody"]["content"]["application/json"]["schema"]["properties"] !== undefined) {
 											// Read out properties from a JSON object
 											const jsonObject = getJsonObject(methodvalue["requestBody"]["content"]["application/json"]["schema"]["properties"])
-											console.log("JSON OBJECT: ", jsonObject)
+											//console.log("JSON OBJECT: ", jsonObject)
 											if (jsonObject !== undefined && jsonObject !== null) {
 												try {
 													newaction["body"] = JSON.stringify(jsonObject, null, 2)
@@ -1128,7 +1158,6 @@ const AppCreator = (defaultprops) => {
 
 											if (selectedExample["content"]["application/json"]["schema"]["properties"] !== undefined && selectedExample["content"]["application/json"]["schema"]["properties"] !== null) {
 												const jsonObject = getJsonObject(selectedExample["content"]["application/json"]["schema"]["properties"]) 
-												console.log("ReSP Return: ", jsonObject)
 												if (jsonObject !== undefined && jsonObject !== null) {
 													try {
                           	newaction.example_response = JSON.stringify(jsonObject, null, 2)
@@ -1483,7 +1512,8 @@ const AppCreator = (defaultprops) => {
                 const searchactions = newActions.find(
                   (data) => data.name === newname
                 );
-                console.log("SEARCH: ", searchactions);
+
+                //console.log("SEARCH: ", searchactions);
                 if (searchactions !== undefined) {
                   newaction.errors.push("Missing name");
                 } else {
@@ -1532,13 +1562,13 @@ const AppCreator = (defaultprops) => {
     }
 
     if (securitySchemes !== undefined) {
-      // FIXME: Should add Oauth2 (Microsoft) and JWT (Wazuh)
+      
-      //console.log("SECURITY: ", securitySchemes)
+			console.log("SECURITY: ", securitySchemes)
       var newauth = [];
 			try {
 				var optionset = false 
       	for (const [key, value] of Object.entries(securitySchemes)) {
-      	  console.log(key, value);
+      	  console.log("AUTH: ", key, value);
 
       	  if (key === "jwt") {
       	    setAuthenticationOption("JWT");
@@ -1749,7 +1779,8 @@ const AppCreator = (defaultprops) => {
 			if (!found) {
 				newActions2.push(action)
 			} else {
-				console.log("Skipping duplicate action: ", action.url, " . Should merge contents")
+				console.log("NOT skipping duplicate action: ", action.url, ". Should merge contents")
+				newActions2.push(action)
 			}
 		}
 

frontend/src/views/Docs.jsx

@@ -112,7 +112,6 @@ const Docs = (defaultprops) => {
   const SideBar = {
     minWidth: 250,
     maxWidth: 300,
-		borderRight: "1px solid rgba(255,255,255,0.3)",
 		left: 0,
 		position: "sticky",
 		top: 50,
@@ -121,6 +120,7 @@ const Docs = (defaultprops) => {
 		overflowX: "hidden",
 		overflowY: "auto",
 		zIndex: 1000,
+		//borderRight: "1px solid rgba(255,255,255,0.3)",
   };
 
   const fetchDocList = () => {
@@ -361,6 +361,7 @@ const Docs = (defaultprops) => {
 		maxWidth: "100%",
 		minWidth: "100%",
 		overflow: "hidden",
+		fontSize: isMobile ? "1.3rem" : "1.0rem",
   };
 
   function OuterLink(props) {

frontend/src/views/Welcome.jsx

@@ -5,6 +5,7 @@ import Stepper from "@material-ui/core/Stepper";
 import Step from "@material-ui/core/Step";
 import StepLabel from "@material-ui/core/StepLabel";
 import AppFramework from "../components/AppFramework.jsx";
+import ArrowBackIosIcon from '@mui/icons-material/ArrowBackIos';
 import { 
 	Grid, 
 	Container,
@@ -303,19 +304,21 @@ const Welcome = (props) => {
 		}
 
 		const actionObject = {
-			padding: "50px 35px 50px 35px", 
+			padding: "35px", 
+			maxHeight: 300,
+			minHeight: 300,
 		}
 
 		const imageStyle = {
 			width: 150, 
-			height: 150, 
+			// height: 150, 
-			margin: "auto", 
+			// margin: "auto", 
-			marginTop: 30, 
+			// marginTop: 10, 
 			borderRadius: 75, 
 			objectFit: "scale-down",
 		}
-
+		const buttonStyle = { borderRadius: 8, height: 51, width: 464, fontSize: 16, background: "linear-gradient(89.83deg, #FF8444 0.13%, #F2643B 99.84%)", padding: "16px 24px", top: 75, }
-		const experienced_image = userdata !== undefined && userdata !== null && userdata.active_org !== undefined && userdata.active_org.image !== undefined && userdata.active_org.image !== null && userdata.active_org.image !== "" ? userdata.active_org.image : "/images/social/shuffle_logo_round.png" 
+		const experienced_image = userdata !== undefined && userdata !== null && userdata.active_org !== undefined && userdata.active_org.image !== undefined && userdata.active_org.image !== null && userdata.active_org.image !== "" ? userdata.active_org.image : "/images/experienced.png" 
     return (
 				<div style={{width: 1000, margin: "auto", backgroundColor: theme.palette.platformColor, paddingBottom: 150, minHeight: 1500, }}>
 					{/*
@@ -424,14 +427,26 @@ const Welcome = (props) => {
 						: 
 						<Fade in={true}>
 							<div style={{maxWidth: 700, margin: "auto", marginTop: 50, }}>
-								<Typography variant="h4" style={{color: "white", textAlign: "center"}}>
+								{/*
-									Welcome to Shuffle
+							<div style={{display:"flex"}}>
+								<ArrowBackIosIcon style={{color: "#9E9E9E",}} onClick={() => {
+										navigate("/login")
+									}}/>
+								<Typography variant="body1" style={{color: "#9E9E9E",textAlign: "center", marginBottom: 50, paddingRight: "366px"}} onClick={() => {
+										navigate("/login")
+									}}>
+								Back
 								</Typography>
-								<Typography variant="body1" style={{textAlign: "center", marginBottom: 50, }}>
+							</div>
-									Who do you identify with the most?
+								*/}
+								<Typography variant="h4" style={{color: "#F1F1F1", textAlign: "center", paddingRight: "300px"}}>
+									Help us get to know you.
+								</Typography>
+								<Typography variant="body1" style={{color: "#9E9E9E", textAlign: "center", marginBottom: 50, paddingRight: "366px"}}>
+								Let us help you create a smoother journey.
 								</Typography>
 								<div style={{display: "flex", marginTop: 70, width: 700, margin: "auto",}}>
-									<div style={{border: "1px solid #49A928",}}>
+									<div style={{border: "1px solid #806BFF",}}>
 										<Card style={paperObject} onClick={() => {
 											if (isCloud) {
 													ReactGA.event({
@@ -446,20 +461,20 @@ const Welcome = (props) => {
 											setShowWelcome(true)
 										}}>
 											<CardActionArea style={actionObject}>
-												<Typography variant="h4" style={{color: "#49A928"}}> 
+												<img src="/images/welcome-to-shuffle.png" style={imageStyle} />
+												<Typography variant="h4" style={{color: "#F1F1F1"}}> 
 													New to Shuffle 
 												</Typography>
-												<img src="/images/welcome_cog.png" style={imageStyle} />
+												<Typography variant="body1" style={{marginTop: 10, color: "rgba(255,255,255,0.8)"}}>
-												<Typography variant="body1" style={{marginTop: 30, color: "rgba(255,255,255,0.8)"}}>
+														Let us guide you for an easier experience.
-													Follow our short introduction and learn some tips and tricks
 												</Typography>
 											</CardActionArea>
 										</Card>
 									</div>
 									<div style={{marginLeft: 25, marginRight: 25, }}>
-										<Typography style={{marginTop: 200, }}>
+										{/* <Typography style={{marginTop: 200, }}>
 											OR
-										</Typography>
+										</Typography> */}
 									</div>
 									<Card style={paperObject} onClick={() => {
 										if (isCloud) {
@@ -473,17 +488,24 @@ const Welcome = (props) => {
 										navigate("/workflows?message=Skipped intro")
 									}}>
 										<CardActionArea style={actionObject}>
-											<Typography variant="h4" style={{color: "#f86a3e"}}> 
+											<img src={experienced_image} style={{padding: "38px", objectFit: "scale-down", minHeight: 40, maxHeight: 40, }} />
+											<Typography variant="h4" style={{color: "#F1F1F1"}}> 
 												Experienced 
 											</Typography>										
-											<img src={experienced_image} style={imageStyle} />
+											<Typography variant="body1" style={{marginTop: 10, color: "rgba(255,255,255,0.8)"}}>
-										
+											You will head to Shuffle right away.
-											<Typography variant="body1" style={{marginTop: 30, color: "rgba(255,255,255,0.8)"}}>
-												You know Shuffle well. Head to your organization right away!
 											</Typography>
 										</CardActionArea>
 									</Card>
 								</div>
+
+								<div style={{display: "flex", flexDirection: "row", }}>
+									<Button variant="contained" type="submit" fullWidth style={buttonStyle} onClick={() => {
+										navigate("/workflows?message=Skipped intro")
+									}}>
+										Continue
+									</Button>
+								</div>
 								{/*
 								<div style={{margin: "auto", border: "1px solid rgba(255,255,255,0.5)", borderRadius: theme.palette.borderRadius, marginTop: 50, width: 200, overflow: "wrap", padding: 25, cursor: "pointer", }} onClick={() => {
 									if (window.drift !== undefined) {

functions/onprem/orborus/go.mod

@@ -2,7 +2,7 @@ module orborus
 
 go 1.19
 
-//replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared
+replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared
 
 require (
 	github.com/docker/docker v23.0.0+incompatible
@@ -27,6 +27,7 @@ require (
 	github.com/frikky/go-elasticsearch/v8 v8.13.1 // indirect
 	github.com/frikky/kin-openapi v0.41.0 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/swag v0.19.5 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
@@ -44,7 +45,11 @@ require (
 	github.com/opencontainers/image-spec v1.0.2 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect
+	github.com/tklauser/go-sysconf v0.3.11 // indirect
+	github.com/tklauser/numcpus v0.6.0 // indirect
+	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	go.opencensus.io v0.22.5 // indirect
 	go4.org v0.0.0-20201209231011-d4a079459e60 // indirect
 	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
@@ -52,7 +57,7 @@ require (
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
 	golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
 	golang.org/x/oauth2 v0.0.0-20210113160501-8b1d76fa0423 // indirect
-	golang.org/x/sys v0.1.0 // indirect
+	golang.org/x/sys v0.2.0 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/tools v0.1.12 // indirect
 	google.golang.org/api v0.36.0 // indirect

functions/onprem/orborus/go.sum

@@ -86,6 +86,8 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
+github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
 github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
@@ -194,6 +196,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
 github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
+github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
+github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shuffle/shuffle-shared v0.3.51 h1:+JPEGw6R4a320who+SrGP/VqBxZdKPdcLw/07cO7d6c=
 github.com/shuffle/shuffle-shared v0.3.51/go.mod h1:jQrYySmvp/0De5ftrAaY6xwwr7TMfqBmBxQ2AX9yrjQ=
 github.com/shuffle/shuffle-shared v0.3.52 h1:d9OycFpuWxrcgHdP2vplKAkY8n+oK5vW02vRK9X+azs=
@@ -214,10 +218,16 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM=
+github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI=
+github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms=
+github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
+github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -330,6 +340,7 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -354,6 +365,8 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A=
+golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

functions/onprem/orborus/orborus.go

@@ -4,11 +4,6 @@ package main
 	Orborus exists to listen for new workflow executions whcih are deployed as workers.
 */
 
-// FIXME:
-// 2022/01/12 17:13:36 [WARNING] Swarm init: Error response from daemon: manager stopped: failed to listen on remote API address: listen tcp: address tcp/2377%!(EXTRA string=172.23.0.2): unknown port
-
-// frikky@debian:~/git/shuffle/functions/onprem/worker$ docker service create --replicas 5 --name shuffle-workers --env SHUFFLE_SWARM_CONFIG=run --publish published=33333,target=33333 ghcr.io/shuffle/shuffle-worker:nightly
-
 //  Potential issues:
 // Default network could be same as on the host
 // Ingress network may not exist (default)
@@ -43,15 +38,14 @@ import (
 	dockerclient "github.com/docker/docker/client"
 	uuid "github.com/satori/go.uuid"
 
-	//network "github.com/docker/docker/api/types/network"
+	//"github.com/mackerelio/go-osstat/disk"
-	//natting "github.com/docker/go-connections/nat"
-	"github.com/mackerelio/go-osstat/cpu"
 	"github.com/mackerelio/go-osstat/memory"
+	"github.com/shirou/gopsutil/cpu"
 )
 
 // Starts jobs in bulk, so this could be increased
 var sleepTime = 3
-var maxConcurrency = 10
+var maxConcurrency = 25
 
 // Timeout if something rashes
 var workerTimeoutEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_TIMEOUT")
@@ -90,6 +84,7 @@ var executionIds = []string{}
 
 var dockercli *dockerclient.Client
 var containerId string
+var executionCount = 0
 
 func init() {
 	var err error
@@ -714,18 +709,18 @@ func initializeImages() {
 	ctx := context.Background()
 
 	if appSdkVersion == "" {
-		appSdkVersion = "1.1.0"
+		appSdkVersion = "latest"
 		log.Printf("[WARNING] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %s", appSdkVersion)
 	}
 
 	if workerVersion == "" {
-		workerVersion = "1.1.0"
+		workerVersion = "latest"
 		log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %s", workerVersion)
 	}
 
 	if baseimageregistry == "" {
-		baseimageregistry = "docker.io"
+		baseimageregistry = "docker.io" // Dockerhub
-		baseimageregistry = "ghcr.io"
+		baseimageregistry = "ghcr.io"   // Github
 		log.Printf("[DEBUG] Setting baseimageregistry")
 	}
 
@@ -744,6 +739,7 @@ func initializeImages() {
 
 	// check whether they are the same first
 	images := []string{
+		fmt.Sprintf("frikky/shuffle:app_sdk"),
 		fmt.Sprintf("shuffle/shuffle:app_sdk"),
 		fmt.Sprintf("%s/%s/shuffle-app_sdk:%s", baseimageregistry, baseimagename, appSdkVersion),
 		newWorker,
@@ -763,40 +759,6 @@ func initializeImages() {
 	}
 }
 
-// Will be used for checking if there's enough to deploy based on a threshold
-// E.g. having maximum CPU and maxmimum RAM
-// Does this work containerized?
-func getStats() {
-	fmt.Printf("\n")
-
-	memory, err := memory.Get()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "%s\n", err)
-		return
-	}
-
-	before, err := cpu.Get()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "%s\n", err)
-		return
-	}
-	time.Sleep(time.Duration(250) * time.Millisecond)
-	after, err := cpu.Get()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "%s\n", err)
-		return
-	}
-	total := float64(after.Total - before.Total)
-
-	fmt.Printf("[INFO] memory total: %d bytes\n", memory.Total)
-	fmt.Printf("[INFO] memory used: %d bytes\n", memory.Used)
-	fmt.Printf("[INFO] cpu used  : %f%%\n", float64(after.User-before.User)/total*100)
-	fmt.Printf("[INFO] cpu system: %f%%\n", float64(after.System-before.System)/total*100)
-	fmt.Printf("[INFO] cpu idle  : %f%%\n", float64(after.Idle-before.Idle)/total*100)
-
-	fmt.Printf("\n")
-}
-
 func findActiveSwarmNodes() (int64, error) {
 	ctx := context.Background()
 	nodes, err := dockercli.NodeList(ctx, types.NodeListOptions{})
@@ -857,6 +819,76 @@ func checkSwarmService(ctx context.Context) {
 	log.Printf("[DEBUG] Swarm info: %s\n\n", ret)
 }
 
+func getOrborusStats() shuffle.OrborusStats {
+	newStats := shuffle.OrborusStats{
+		OrgId:        org,
+		Environment:  environment,
+		OrborusLabel: orborusLabel,
+		Timestamp:    time.Now().Unix(),
+	}
+
+	if swarmConfig == "run" || swarmConfig == "swarm" {
+		newStats.Swarm = true
+	}
+
+	if runningMode == "kubernetes" || runningMode == "k8s" {
+		newStats.Kubernetes = true
+	}
+
+	newStats.PollTime = sleepTime
+	newStats.MaxQueue = maxConcurrency
+	newStats.Queue = executionCount
+
+	// Get CPU usage and max CPU
+	/*
+		before, err := cpu.Get()
+		if err != nil {
+			log.Printf("[ERROR] Failed getting CPU stats: %s", err)
+		} else {
+			newStats.CPU = int(before.User)
+			newStats.MaxCPU = int(before.Total)
+		}
+	*/
+
+	cpuPercent, err := cpu.Percent(250*time.Millisecond, false)
+	if err == nil && len(cpuPercent) > 0 {
+		newStats.CPUPercent = cpuPercent[0]
+	}
+	//Percent(interval time.Duration, percpu bool) ([]float64, error)
+
+	// Get memory usage
+	memory, err := memory.Get()
+	if err != nil {
+		log.Printf("[ERROR] Failed getting memory stats: %s", err)
+	} else {
+		newStats.Memory = int(memory.Used)
+		newStats.MaxMemory = int(memory.Total)
+	}
+
+	// Get disk usage
+	/*
+		disk, err := disk.Get()
+		if err != nil {
+			log.Printf("[ERROR] Failed getting disk stats: %s", err)
+		} else {
+			newStats.Disk = int(disk.Used)
+			newStats.MaxDisk = int(disk.Total)
+		}
+	*/
+
+	/*
+			// General
+			Disk   int `json:"disk"`
+
+			// Docker
+			AppContainers    int `json:"app_containers"`
+			WorkerContainers int `json:"worker_containers"`
+			TotalContainers  int `json:"total_containers"`
+		}
+	*/
+	return newStats
+}
+
 // Initial loop etc
 func main() {
 	startupDelay := os.Getenv("SHUFFLE_ORBORUS_STARTUP_DELAY")
@@ -867,7 +899,7 @@ func main() {
 		if err == nil {
 			time.Sleep(time.Duration(tmpInt) * time.Second)
 		} else {
-			log.Printf("[WARNING] Env SHUFFLE_ORBORUS_STARTUP_DELAY must be a number, not %s", startupDelay)
+			log.Printf("[WARNING] Env SHUFFLE_ORBORUS_STARTUP_DELAY must be a number, not '%s'. Using default.", startupDelay)
 		}
 	}
 
@@ -892,6 +924,8 @@ func main() {
 		timezone = "Europe/Amsterdam"
 	}
 
+	log.Printf("[INFO] Using environment '%s' with timezone %s", environment, timezone)
+
 	if len(os.Getenv("SHUFFLE_ORBORUS_PULL_TIME")) > 0 {
 		log.Printf("[INFO] Trying to set Orborus sleep time between polls to %s", os.Getenv("SHUFFLE_ORBORUS_PULL_TIME"))
 
@@ -901,8 +935,6 @@ func main() {
 		}
 	}
 
-	log.Printf("[INFO] Running with timezone %s", timezone)
-
 	workerTimeout := 600
 	if workerTimeoutEnv != "" {
 		tmpInt, err := strconv.Atoi(workerTimeoutEnv)
@@ -941,7 +973,7 @@ func main() {
 
 	if environment == "" {
 		environment = "onprem"
-		log.Printf("[INFO] Defaulting to environment name %s. Set environment variable ENVIRONMENT_NAME to change. This should be the same as in the frontend action.", environment)
+		log.Printf("[WARNING] Defaulting to environment name %s. Set environment variable ENVIRONMENT_NAME to change. This should be the same as in the frontend action.", environment)
 	}
 
 	// FIXME - during init, BUILD and/or LOAD worker and app_sdk
@@ -970,14 +1002,17 @@ func main() {
 		//deployServiceWorkers(workerImage)
 	}
 
-	log.Printf("[INFO] Finished configuring docker environment")
-
 	client := shuffle.GetExternalClient(baseUrl)
 	fullUrl := fmt.Sprintf("%s/api/v1/workflows/queue", baseUrl)
+	log.Printf("[INFO] Finished configuring docker environment. Connecting to %s", fullUrl)
+
+	forwardData := bytes.NewBuffer([]byte{})
+	forwardMethod := "POST"
+
 	req, err := http.NewRequest(
-		"GET",
+		forwardMethod,
 		fullUrl,
-		nil,
+		forwardData,
 	)
 
 	if err != nil {
@@ -986,9 +1021,9 @@ func main() {
 	}
 
 	zombiecounter := 0
+
 	req.Header.Add("Content-Type", "application/json")
 	req.Header.Add("Org-Id", environment)
-
 	if len(auth) > 0 {
 		req.Header.Add("Authorization", auth)
 	}
@@ -998,7 +1033,7 @@ func main() {
 	}
 
 	if len(orborusLabel) > 0 {
-		log.Printf("[DEBUG] Sending with Label %s", orborusLabel)
+		log.Printf("[DEBUG] Sending with Label '%s'", orborusLabel)
 		req.Header.Add("X-Orborus-Label", orborusLabel)
 	}
 
@@ -1011,8 +1046,19 @@ func main() {
 	log.Printf("[INFO] Waiting for executions at %s with Environment %#v", fullUrl, environment)
 	hasStarted := false
 	for {
-		//go getStats()
+		if req.Method == "POST" {
-		//log.Printf("[DEBUG] Prerequest - queue")
+			// Should find data to send (memory etc.)
+
+			orborusStats := getOrborusStats()
+			// Marshal and set body
+			jsonData, err := json.Marshal(orborusStats)
+			if err == nil {
+				req.Body = ioutil.NopCloser(bytes.NewBuffer(jsonData))
+			} else {
+				log.Printf("[ERROR] Failed marshalling json: %s", err)
+			}
+		}
+
 		newresp, err := client.Do(req)
 		//log.Printf("[DEBUG] Postrequest - queue")
 		if err != nil {
@@ -1027,6 +1073,16 @@ func main() {
 			continue
 		}
 
+		if newresp.StatusCode == 405 {
+			log.Printf("[WARNING] Received 405 from %s. This is likely due to a misconfigured base URL. Automatically swapping to GET request (backwards compatibility)", fullUrl)
+
+			req.Method = "GET"
+			req.Body = nil
+
+			//time.Sleep(time.Duration(sleepTime) * time.Second)
+			continue
+		}
+
 		body, err := ioutil.ReadAll(newresp.Body)
 		if err != nil {
 			log.Printf("[ERROR] Failed reading body from Shuffle: %s", err)
@@ -1082,7 +1138,7 @@ func main() {
 			}
 
 			// Anything below here verifies concurrency
-			executionCount := getRunningWorkers(ctx, workerTimeout)
+			executionCount = getRunningWorkers(ctx, workerTimeout)
 			if executionCount >= maxConcurrency {
 				if zombiecounter*sleepTime > workerTimeout {
 					go zombiecheck(ctx, workerTimeout)