diff --git a/backend/app_sdk/Dockerfile b/backend/app_sdk/Dockerfile index ac322bb9..d3709032 100644 --- a/backend/app_sdk/Dockerfile +++ b/backend/app_sdk/Dockerfile @@ -1,6 +1,6 @@ #FROM python:3.9.1-alpine as base -#FROM python:3.10.0-alpine as base -FROM python:3.11.3-alpine as base +FROM python:3.10.0-alpine as base +#FROM python:3.11.3-alpine as base FROM base as builder RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev tzdata coreutils diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index b4ab4ff7..39073de7 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -2279,13 +2279,6 @@ class AppBase: # Can't handle self yet (?) ret = run.render(**globals()) - - # Load output as JSON - try: - ret = json.loads(ret) - except: - pass - return ret except jinja2.exceptions.TemplateNotFound as e: self.logger.info(f"[ERROR] Liquid Template error: {e}") @@ -3071,6 +3064,7 @@ class AppBase: #self.logger.info(action["parameters"]) # This seems redundant now + self.logger.info("[DEBUG] Pre parameters") for parameter in newparams: action["parameters"].append(parameter) @@ -3092,6 +3086,7 @@ class AppBase: # Multi_parameter has the data for each. variable minlength = 0 + self.logger.info("[DEBUG] Pre-loading parameters") multi_parameters = json.loads(json.dumps(params)) multiexecution = False multi_execution_lists = [] @@ -3518,11 +3513,8 @@ class AppBase: try: del params[field] self.logger.info("[WARNING] Removed field invalid field %s" % field) - except KeyError as e: - self.logger.info("[WARNING] Tried to remove field %s but it didn't exist" % field) + except KeyError: break - else: - self.logger.info("[ERROR] Couldn't find fieldsplit in error. Raw error: %s" % errorstring) else: newres = json.dumps({ "success": False, @@ -3899,11 +3891,7 @@ class AppBase: else: self.logger.info("ACTION TYPE (unhandled): %s" % type(action)) - #await app.execute_action(app.action) app.execute_action(app.action) - #app.run(host="0.0.0.0", port=33334) - if __name__ == "__main__": AppBase.run() - #asyncio.run(AppBase.run(), debug=True) diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 1ff45c75..dd540b94 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -2,7 +2,7 @@ ### DEFAULT NAME=shuffle-app_sdk -VERSION=1.1.0 +VERSION=1.2.0 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly -t shuffle/shuffle:app_sdk -t shuffle/$NAME:$VERSION -t docker.pkg.github.com/shuffle/shuffle/$NAME:$VERSION -t ghcr.io/shuffle/$NAME:$VERSION -t ghcr.io/shuffle/$NAME:nightly diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index dbf1958e..00202dbd 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -678,8 +678,11 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { alternativeName = strings.Join(alternativeNameSplit[1:3], "/") } + log.Printf("[INFO] Trying to download image: %s. Alt: %s", version.Name, alternativeName) + for _, image := range images { for _, tag := range image.RepoTags { + //log.Printf("[DEBUG] Tag: %s", tag) if strings.ToLower(tag) == strings.ToLower(version.Name) { img = image tagFound = tag @@ -693,6 +696,29 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { } } + pullOptions := types.ImagePullOptions{} + if len(img.ID) == 0 { + _, err := dockercli.ImagePull(context.Background(), version.Name, pullOptions) + if err == nil { + tagFound = version.Name + img.ID = version.Name + img2.ID = version.Name + + dockercli.ImageTag(ctx, version.Name, alternativeName) + } + } + + if len(img2.ID) == 0 { + _, err := dockercli.ImagePull(context.Background(), alternativeName, pullOptions) + if err == nil { + tagFound = alternativeName + img.ID = alternativeName + img2.ID = alternativeName + + dockercli.ImageTag(ctx, alternativeName, version.Name) + } + } + // REBUILDS THE APP if len(img.ID) == 0 { if len(img2.ID) == 0 { @@ -722,7 +748,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { foundApp := shuffle.WorkflowApp{} imageName = strings.ToLower(imageName) imageVersion = strings.ToLower(imageVersion) - log.Printf("[DEBUG] Looking for appname %s with version %s", imageName, imageVersion) + log.Printf("[DEBUG] Docker Looking for appname %s with version %s", imageName, imageVersion) for _, app := range workflowapps { if strings.ToLower(strings.Replace(app.Name, " ", "_", -1)) == imageName && app.AppVersion == imageVersion { diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 95ba41de..69662589 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -19,7 +19,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.4.2 + github.com/shuffle/shuffle-shared v0.4.9 golang.org/x/crypto v0.3.0 google.golang.org/api v0.103.0 google.golang.org/appengine v1.6.7 diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 08756aad..dc8059b3 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -81,18 +81,14 @@ import ( var gceProject = "shuffle" var bucketName = "shuffler.appspot.com" var baseAppPath = "/home/frikky/git/shaffuru/tmp/apps" + var baseDockerName = "frikky/shuffle" var registryName = "registry.hub.docker.com" var runningEnvironment = "onprem" var syncUrl = "https://shuffler.io" - -// var syncUrl = "http://localhost:5002" var syncSubUrl = "https://shuffler.io" -//var syncUrl = "http://localhost:5002" -//var syncSubUrl = "https://050196912a9d.ngrok.io" - var dbclient *datastore.Client type Userapi struct { @@ -5873,7 +5869,7 @@ func initHandlers() { r.HandleFunc("/api/v1/streams/results", handleGetStreamResults).Methods("POST", "OPTIONS") // Used by orborus - r.HandleFunc("/api/v1/workflows/queue", handleGetWorkflowqueue).Methods("GET") + r.HandleFunc("/api/v1/workflows/queue", handleGetWorkflowqueue).Methods("GET", "POST") r.HandleFunc("/api/v1/workflows/queue/confirm", handleGetWorkflowqueueConfirm).Methods("POST") // App specific @@ -6018,6 +6014,8 @@ func initHandlers() { r.HandleFunc("/api/v1/users/notifications/clear", shuffle.HandleClearNotifications).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/users/notifications/{notificationId}/markasread", shuffle.HandleMarkAsRead).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/conversation", shuffle.RunActionAI).Methods("POST", "OPTIONS") + //r.HandleFunc("/api/v1/users/notifications/{notificationId}/markasread", shuffle.HandleMarkAsRead).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/dashboards/{key}/widgets", shuffle.HandleNewWidget).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/dashboards/{key}/widgets/{widget_id}", shuffle.HandleGetWidget).Methods("GET", "OPTIONS") diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index d410a3ed..91bc6dbd 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -11,6 +11,7 @@ import ( "io" "io/ioutil" "log" + "math/rand" "net/http" "net/url" "os" @@ -251,16 +252,42 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { return } - id := request.Header.Get("Org-Id") - if len(id) == 0 { - log.Printf("[INFO] No org-id header set") + // This is really the environment's name - NOT org-id + orgId := request.Header.Get("Org-Id") + if len(orgId) == 0 { + log.Printf("[AUDIT] No org-id header set") resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Specify the org-id header."}`))) return } - ctx := context.Background() - env, err := shuffle.GetEnvironment(ctx, id, "") + environment := request.Header.Get("org") + if len(environment) == 0 { + log.Printf("[AUDIT] No 'org' header set (get workflow queue). Required for cloud.") + /* + resp.WriteHeader(403) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Specify the org header. This can be done by setting the 'ORG' environment variable for Orborus to your Org ID in Shuffle"}`))) + return + */ + } + + orborusLabel := request.Header.Get("x-orborus-label") + + // This section is cloud custom for now + auth := request.Header.Get("Authorization") + if len(auth) == 0 { + log.Printf("[AUDIT] No Authorization header set. Required for cloud. Env: %s, org: %s", orgId, environment) + /* + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Specify the auth header (only applicable for cloud for now)."}`))) + return + */ + } + + //log.Printf("[AUDIT] Get workflow queue for org %s, env %s, orborus label %s", orgId, environment, orborusLabel) + + ctx := shuffle.GetContext(request) + env, err := shuffle.GetEnvironment(ctx, orgId, "") timeNow := time.Now().Unix() if err == nil && len(env.Id) > 0 && len(env.Name) > 0 { if time.Now().Unix() > env.Edited+60 { @@ -273,7 +300,154 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { } } - executionRequests, err := shuffle.GetWorkflowQueue(ctx, id, 100) + //log.Printf("Found env: %#v", env) + if len(env.OrgId) > 0 { + environment = env.OrgId + } + + if request.Method == "POST" { + if rand.Intn(1) == 0 { + // Parse out body + body, err := ioutil.ReadAll(request.Body) + if err == nil { + + // Parse out CPU, memory and disk. + + var envData shuffle.OrborusStats + err = json.Unmarshal(body, &envData) + if err == nil && !envData.Swarm && !envData.Kubernetes && (envData.CPU > 0 || envData.Memory > 0 || envData.Disk > 0) { + + // Set the input in memory + envData.OrgId = orgId + envData.Environment = environment + envData.OrborusLabel = orborusLabel + envData.Timestamp = time.Now().Unix() + + if envData.CPU > 0 && envData.MaxCPU > 0 { + envData.CPUPercent = float64(envData.CPU) / float64(envData.MaxCPU) + } + + if envData.Memory > 0 && envData.MaxMemory > 0 { + envData.MemoryPercent = float64(envData.Memory) / float64(envData.MaxMemory) + } + + // Check if CPU percent constantly has stayed above X% for the last Y requests + percentageCheck := 90 + concurrentChecks := 0 + + //if int(envData.CPUPercent) > percentageCheck { + // Get cached data + percentages := []float64{} + cacheKey := fmt.Sprintf("%s_%s_percent", orgId, strings.ToLower(environment)) + + // Marshal float list into []byte + cacheData := []byte{} + cache, err := shuffle.GetCache(ctx, cacheKey) + if err == nil { + // Unmarshal into percentages + cacheData := []byte(cache.([]uint8)) + err = json.Unmarshal(cacheData, &percentages) + if err != nil { + log.Printf("[INFO] error in cache unmarshal for percentages: %s", err) + } + + if len(percentages) > concurrentChecks { + percentages = percentages[:concurrentChecks] + } + + percentages = append(percentages, envData.CPUPercent) + if len(percentages) > concurrentChecks { + //log.Printf("[INFO] Checking percentages: %v", percentages) + + // percentageCheck := 1 + sendAlert := true + for _, p := range percentages { + if int(p) < percentageCheck { + //log.Printf("[AUDIT] CPU percent is below %d: %d", percentageCheck, int(p)) + sendAlert = false + break + } + } + + if sendAlert { + log.Printf("[INFO] CPU percent has been above %d percent for the last 5 requests. Sending alert. Env: %s, org: %s", percentageCheck, environment, orgId) + + // Set notification + alert for organization + err = shuffle.CreateOrgNotification( + ctx, + fmt.Sprintf("CPU percent has been above %d percent", percentageCheck), + fmt.Sprintf("A environment %s has been using more than %d\\% CPU for the last 5 requests.", environment, percentageCheck), + fmt.Sprintf("/admin?tab=environments"), + environment, + true, + ) + + if err != nil { + log.Printf("[ERROR] error creating notification: %s", err) + } + + org, err := shuffle.GetOrg(ctx, environment) + if err == nil { + foundRecommendation := false + for _, recommendation := range org.Priorities { + if strings.Contains(recommendation.Name, "CPU") { + foundRecommendation = true + break + } + } + + if !foundRecommendation { + // Add to start of org.Priorities + org.Priorities = append(org.Priorities, shuffle.Priority{ + Name: fmt.Sprintf("High CPU in environment %s", orgId), + Description: fmt.Sprintf("The environment %s has been using more than %d percent CPU.", orgId, percentageCheck), + Type: "scale", + Active: true, + URL: fmt.Sprintf("/admin?tab=environments"), + }) + + //Make last item the first item + org.Priorities = append([]shuffle.Priority{org.Priorities[len(org.Priorities)-1]}, org.Priorities[:len(org.Priorities)-1]...) + err = shuffle.SetOrg(ctx, *org, org.Id) + if err != nil { + log.Printf("[ERROR] Problem setting org: %s", err) + } + } + } + } + + if len(percentages) > 1 { + percentages = percentages[1:] + } + } + + // Marshal float list into []byte + } else { + //log.Printf("[ERROR] Failed getting cache: %s", err) + percentages = append(percentages, envData.CPUPercent) + } + + if len(percentages) > 0 { + //log.Printf("[DEBUG] Setting cache for %s: %#v", cacheKey, percentages) + cacheData, err = json.Marshal(percentages) + if err != nil { + log.Printf("[INFO] error in cache marshal: %s", err) + } + + // Add the new data + go shuffle.SetCache(ctx, cacheKey, cacheData, 5) + } + } + + //log.Printf("CPU percent: %f", envData.CPUPercent) + //log.Printf("Memory percent: %f", envData.MemoryPercent*100) + + go shuffle.SetenvStats(ctx, envData) + } + } + } + + executionRequests, err := shuffle.GetWorkflowQueue(ctx, orgId, 100) if err != nil { // Skipping as this comes up over and over //log.Printf("(2) Failed reading body for workflowqueue: %s", err) @@ -290,21 +464,21 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { // Try again :) if len(env.Id) == 0 && len(env.Name) == 0 { - orgId := "" + foundId := "" for _, requestData := range executionRequests.Data { execution, err := shuffle.GetWorkflowExecution(ctx, requestData.ExecutionId) if err == nil { if len(execution.ExecutionOrg) > 0 { - orgId = execution.ExecutionOrg + foundId = execution.ExecutionOrg break } } } if len(orgId) > 0 { - env, err := shuffle.GetEnvironment(ctx, id, orgId) + env, err := shuffle.GetEnvironment(ctx, orgId, foundId) if err != nil { - log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", id, err) + log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", orgId, err) //resp.WriteHeader(401) //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No env found matching %s"}`, id))) //return @@ -361,9 +535,9 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { err = json.Unmarshal(body, &actionResult) if err != nil { log.Printf("[WARNING] Failed ActionResult unmarshaling (stream result): %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return + //resp.WriteHeader(401) + //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + //return } ctx := context.Background() @@ -476,9 +650,9 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { err = json.Unmarshal(body, &actionResult) if err != nil { log.Printf("[WARNING] Failed ActionResult unmarshaling (queue): %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return + //resp.WriteHeader(401) + //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + //return } //log.Printf("Received action: %#v", actionResult) diff --git a/docker-compose.yml b/docker-compose.yml index 1143f01c..b6878df7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,62 +1,62 @@ version: '3' services: - frontend: - image: ghcr.io/shuffle/shuffle-frontend:latest - container_name: shuffle-frontend - hostname: shuffle-frontend - ports: - - "${FRONTEND_PORT}:80" - - "${FRONTEND_PORT_HTTPS}:443" - networks: - - shuffle - environment: - - BACKEND_HOSTNAME=${BACKEND_HOSTNAME} - restart: unless-stopped - depends_on: - - backend - backend: - image: ghcr.io/shuffle/shuffle-backend:latest - container_name: shuffle-backend - hostname: ${BACKEND_HOSTNAME} - # Here for debugging: - ports: - - "${BACKEND_PORT}:5001" - networks: - - shuffle - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps:z - - ${SHUFFLE_FILE_LOCATION}:/shuffle-files:z - env_file: .env - environment: - #- DOCKER_HOST=tcp://docker-socket-proxy:2375 - - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps - - SHUFFLE_FILE_LOCATION=/shuffle-files - restart: unless-stopped - orborus: - image: ghcr.io/shuffle/shuffle-orborus:latest - container_name: shuffle-orborus - hostname: shuffle-orborus - networks: - - shuffle - volumes: - - /var/run/docker.sock:/var/run/docker.sock - environment: - #- DOCKER_HOST=tcp://docker-socket-proxy:2375 - - SHUFFLE_WORKER_VERSION=latest - - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - - BASE_URL=http://${OUTER_HOSTNAME}:5001 - - DOCKER_API_VERSION=1.40 - - SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME} - - SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY} - - SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX} - - HTTP_PROXY=${HTTP_PROXY} - - HTTPS_PROXY=${HTTPS_PROXY} - - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY} - - SHUFFLE_PASS_APP_PROXY=${SHUFFLE_PASS_APP_PROXY} - restart: unless-stopped - security_opt: - - seccomp:unconfined + #frontend: + # image: ghcr.io/shuffle/shuffle-frontend:latest + # container_name: shuffle-frontend + # hostname: shuffle-frontend + # ports: + # - "${FRONTEND_PORT}:80" + # - "${FRONTEND_PORT_HTTPS}:443" + # networks: + # - shuffle + # environment: + # - BACKEND_HOSTNAME=${BACKEND_HOSTNAME} + # restart: unless-stopped + # depends_on: + # - backend + #backend: + # image: ghcr.io/shuffle/shuffle-backend:latest + # container_name: shuffle-backend + # hostname: ${BACKEND_HOSTNAME} + # # Here for debugging: + # ports: + # - "${BACKEND_PORT}:5001" + # networks: + # - shuffle + # volumes: + # - /var/run/docker.sock:/var/run/docker.sock + # - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps:z + # - ${SHUFFLE_FILE_LOCATION}:/shuffle-files:z + # env_file: .env + # environment: + # #- DOCKER_HOST=tcp://docker-socket-proxy:2375 + # - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps + # - SHUFFLE_FILE_LOCATION=/shuffle-files + # restart: unless-stopped + #orborus: + # image: ghcr.io/shuffle/shuffle-orborus:latest + # container_name: shuffle-orborus + # hostname: shuffle-orborus + # networks: + # - shuffle + # volumes: + # - /var/run/docker.sock:/var/run/docker.sock + # environment: + # #- DOCKER_HOST=tcp://docker-socket-proxy:2375 + # - SHUFFLE_WORKER_VERSION=latest + # - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} + # - BASE_URL=http://${OUTER_HOSTNAME}:5001 + # - DOCKER_API_VERSION=1.40 + # - SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME} + # - SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY} + # - SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX} + # - HTTP_PROXY=${HTTP_PROXY} + # - HTTPS_PROXY=${HTTPS_PROXY} + # - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY} + # - SHUFFLE_PASS_APP_PROXY=${SHUFFLE_PASS_APP_PROXY} + # restart: unless-stopped + # security_opt: + # - seccomp:unconfined opensearch: image: opensearchproject/opensearch:2.5.0 hostname: shuffle-opensearch diff --git a/frontend/public/images/experienced.png b/frontend/public/images/experienced.png new file mode 100644 index 00000000..8e5f3e5a Binary files /dev/null and b/frontend/public/images/experienced.png differ diff --git a/frontend/public/images/logos/orange_logo.svg b/frontend/public/images/logos/orange_logo.svg new file mode 100644 index 00000000..1024dd6a --- /dev/null +++ b/frontend/public/images/logos/orange_logo.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/frontend/public/images/welcome-to-shuffle.png b/frontend/public/images/welcome-to-shuffle.png new file mode 100644 index 00000000..cb976e31 Binary files /dev/null and b/frontend/public/images/welcome-to-shuffle.png differ diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 06c91245..41922ff4 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -10,7 +10,7 @@ import GettingStarted from "./views/GettingStarted"; import EditWebhook from "./views/EditWebhook"; import AngularWorkflow from "./views/AngularWorkflow"; -import Header from "./components/Header"; +import Header from "./components/Header.jsx"; import theme from "./theme"; import Apps from "./views/Apps"; import AppCreator from "./views/AppCreator"; @@ -336,7 +336,9 @@ const App = (message, props) => { userdata={userdata} {...props} /> + {/*
+ */} { style: { backgroundColor: theme.palette.surfaceColor, color: "white", - minWidth: isMobile ? "90%" : newWorkflow === true ? 1000 : 550, - maxWidth: isMobile ? "90%" : newWorkflow === true ? 1000 : 550, + minWidth: isMobile ? "90%" : 550, + maxWidth: isMobile ? "90%" : 550, minHeight: 400, + //minWidth: isMobile ? "90%" : newWorkflow === true ? 1000 : 550, + //maxWidth: isMobile ? "90%" : newWorkflow === true ? 1000 : 550, }, }} > @@ -196,7 +198,7 @@ const EditWorkflow = (props) => {
: null} - {newWorkflow === true ? + {/*newWorkflow === true ?
Use a Template @@ -205,7 +207,7 @@ const EditWorkflow = (props) => { Start your workflow from our templating system. This uses publied workflows from our Creators to generate full Usecases or parts of your Workflow.
- : null} + : null*/} @@ -438,7 +440,7 @@ const EditWorkflow = (props) => { - {newWorkflow === true ? + {/*newWorkflow === true ?
{ userdata={userdata} />
- : null} + : null*/} " + + // Find mouse cursor position on screen + console.log("Suggestion html to be added at location: ", event) + /* + const position = { + "top": cy.pan().y, + "left": cy.pan().x, + } + */ + + + const position = event.target.renderedPosition(); + const container = cy.container(); + const offset = { + left: container.offsetLeft, + top: container.offsetTop + }; + + // Calculate the actual screen position for the box + const screenPosition = { + left: position.x + offset.left - 150, + top: position.y + offset.top, + }; + + // Log the position to the console + console.log('Node screen position:', screenPosition); + + const newbox = { + "position": screenPosition, + "node_position": event.target.position(), + "open": true, + "attachedTo": data.attachedTo, + } + + console.log("Rendered position: ", newbox.node_position) + + setSuggestionBox(newbox) + + // Unselect + event.target.unselect(); + + } else if (data.buttonType === "delete") { const parentNode = cy.getElementById(data.attachedTo); if (parentNode !== null && parentNode !== undefined) { removeNode(data.attachedTo) @@ -3187,6 +3320,14 @@ const AngularWorkflow = (defaultprops) => { console.log("DOne in the node update") + setSuggestionBox({ + "position": { + "top": 500, + "left": 500, + }, + "open": false, + "attachedTo": "", + }); sendStreamRequest({ "item": "node", "type": "select", @@ -3582,7 +3723,7 @@ const AngularWorkflow = (defaultprops) => { } const eventTarget = event.target.target() - console.log("BUTTON ADDED! Find parent from: ", eventTarget) + //console.log("BUTTON ADDED! Find parent from: ", eventTarget) if (eventTarget.data("isButton") === true) { const parentNode = cy.getElementById(eventTarget.data("attachedTo")) event.target.remove() @@ -4211,6 +4352,8 @@ const AngularWorkflow = (defaultprops) => { item = newitem item.type = "ACTION" item.isStartNode = false + item.data.type = "ACTION" + item.data.isStartNode = false } item.data.id = uuidv4() @@ -4473,7 +4616,46 @@ const AngularWorkflow = (defaultprops) => { }; const addSuggestionButtons = (nodedata, event) => { - console.log("In suggestion buttons: ", nodedata, ". This is not enabled yet. Backend needs further work.") + //console.log("In suggestion buttons: ", nodedata, ". This is not enabled yet. Backend needs further work.") + // Skipping add for now. Should Re-enable + + // Add a button for autocompletion based on input + console.log("Type: ", nodedata.type) + if (nodedata.type === "ACTION") { + const color = "#34a853" + + // Fix icon + const iconInfo = { + icon: "M7.5 5.6 10 7 8.6 4.5 10 2 7.5 3.4 5 2l1.4 2.5L5 7zm12 9.8L17 14l1.4 2.5L17 19l2.5-1.4L22 19l-1.4-2.5L22 14zM22 2l-2.5 1.4L17 2l1.4 2.5L17 7l2.5-1.4L22 7l-1.4-2.5zm-7.63 5.29a.9959.9959 0 0 0-1.41 0L1.29 18.96c-.39.39-.39 1.02 0 1.41l2.34 2.34c.39.39 1.02.39 1.41 0L16.7 11.05c.39-.39.39-1.02 0-1.41l-2.33-2.35zm-1.03 5.49-2.12-2.12 2.44-2.44 2.12 2.12-2.44 2.44z", + iconColor: buttonColor, + iconBackgroundColor: buttonBackgroundColor, + }; + + const svg_pin = ``; + const svgpin_Url = encodeURI("data:image/svg+xml;utf-8," + svg_pin); + + const decoratorNode = { + position: { + x: event.target.position().x + 0, + y: event.target.position().y + 65, + }, + locked: true, + data: { + isButton: true, + isValid: true, + is_valid: true, + //label: "+", + attachedTo: nodedata.id, + imageColor: color, + buttonType: "suggestion", + icon: svgpin_Url, + iconBackground: iconInfo.iconBackgroundColor, + }, + }; + + cy.add(decoratorNode); + } + return //setWorkflowRecommendations(responseJson.actions) @@ -7199,7 +7381,6 @@ const AngularWorkflow = (defaultprops) => { setSelectedAction(newSelectedAction); setUpdate(Math.random()); - // FIXME - should change icon-node (descriptor) as well const allNodes = cy.nodes().jsons(); if (allNodes !== undefined && allNodes !== null) { for (let nodekey in allNodes) { @@ -7221,7 +7402,12 @@ const AngularWorkflow = (defaultprops) => { } } } - }; + + // Send it in here, after all fields are filled + // Disabled for now :( + // const aiMsg = "Fill based on previous values" + // aiSubmit(aiMsg, undefined, undefined, newSelectedAction) + }; // APPSELECT at top // appname & version @@ -10905,7 +11091,6 @@ const AngularWorkflow = (defaultprops) => { fullWidth multiline rows="2" - defaultValue={trigger_header_auth} color="primary" disabled={selectedTrigger.status === "running"} placeholder={"OK"} @@ -12612,6 +12797,8 @@ const AngularWorkflow = (defaultprops) => { requiresAuthentication={requiresAuthentication} setLastSaved={setLastSaved} lastSaved={lastSaved} + + aiSubmit={aiSubmit} /> } else if (Object.getOwnPropertyNames(selectedComment).length > 0) { @@ -12929,6 +13116,9 @@ const AngularWorkflow = (defaultprops) => { getSettings(); getFiles() + // For loading datastore + // listOrgCache(workflow.org_id) + setUpdate(Math.random()); }} > @@ -14566,8 +14756,8 @@ const AngularWorkflow = (defaultprops) => {
{curapp === null ? null : ( {selectedResult.app_name} {
Status {selectedResult.status}
+ + +

{ + console.log("APP: ", curapp) + console.log("Data: ", selectedResult) + }}> + HIYA: {selectedResult.action.app_name} +

+ {validate.valid ? ( { /> ) : (
- Result  + Result +
{ console.log("IN HERE TO CLICK"); to_be_copied = selectedResult.result; @@ -15734,6 +15937,358 @@ const AngularWorkflow = (defaultprops) => {
) : null; + + // Should get AI autocompletes + const aiSubmit = (value, setResponseMsg, setSuggestionLoading, inputAction) => { + if (setResponseMsg !== undefined) { + setResponseMsg("") + } + + if (value === undefined || value === "") { + console.log("No value input!") + return + } + + if (setSuggestionLoading !== undefined) { + setSuggestionLoading(true) + } + + console.log("Submit conversation with value: ", value); + + // This is to find sample response and parse it as string + + var AppContext = [] + if (inputAction !== undefined && inputAction !== null) { + const parents = getParents(inputAction) + + console.log("Parents: ", parents) + var actionlist = [] + if (parents.length > 1) { + for (let [key,keyval] in Object.entries(parents)) { + const item = parents[key]; + if (item.label === "Execution Argument") { + continue; + } + + var exampledata = item.example === undefined || item.example === null ? "" : item.example; + // Find previous execution and their variables + //exampledata === "" && + if (workflowExecutions.length > 0) { + // Look for the ID + const found = false; + for (let [key,keyval] in Object.entries(workflowExecutions)) { + if (workflowExecutions[key].results === undefined || workflowExecutions[key].results === null) { + continue; + } + + var foundResult = workflowExecutions[key].results.find((result) => result.action.id === item.id); + if (foundResult === undefined || foundResult === null) { + continue; + } + + if (foundResult.result !== undefined && foundResult.result !== null) { + foundResult = foundResult.result + } + + const valid = validateJson(foundResult, true) + if (valid.valid) { + if (valid.result.success === false) { + //console.log("Skipping success false autocomplete") + } else { + exampledata = valid.result; + break; + } + } else { + exampledata = foundResult; + } + } + } + + // 1. Take + const itemlabelComplete = item.label === null || item.label === undefined ? "" : item.label.split(" ").join("_"); + + const actionvalue = { + app_name: item.app_name, + action_name: item.name, + label: item.label, + + type: "action", + id: item.id, + name: item.label, + autocomplete: itemlabelComplete, + example: exampledata, + }; + + actionlist.push(actionvalue); + } + } + + var fixedResults = [] + for (var i = 0; i < actionlist.length; i++) { + const item = actionlist[i]; + const responseFix = SetJsonDotnotation(item.example, "") + + // Check if json + const validated = validateJson(responseFix) + var exampledata = responseFix; + if (validated.valid) { + exampledata = JSON.stringify(validated.result) + } + + AppContext.push({ + "app_name": item.app_name, + "action_name": item.action_name, + "label": item.label, + "example": exampledata, + }) + } + } + + var conversationData = { + "query": value, + "output_format": "action", + "app_context": AppContext, + + "workflow_id": workflow.id, + } + + if (inputAction !== undefined) { + console.log("Add app context! This should them get parameters directly") + conversationData.output_format = "action_parameters" + + conversationData.app_id = inputAction.app_id + conversationData.app_name = inputAction.app_name + conversationData.action_name = inputAction.name + conversationData.parameters = inputAction.parameters + + if (!value.includes(inputAction.label)) { + conversationData.query = inputAction.label.replace("_", " ", -1) + } + } + + // Onprem not available yet (April 2023) + // Should: Make OpenAI work for them with their own key + //fetch("https://shuffler.io/api/v1/conversation", { + fetch(`${globalUrl}/api/v1/conversation`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(conversationData), + credentials: "include", + }) + .then((response) => { + if (setSuggestionLoading !== undefined) { + setSuggestionLoading(false) + } + + if (response.status !== 200) { + console.log("Status not 200 for stream results :O!"); + } + + return response.json(); + }) + .then((responseJson) => { + console.log("Conversation response: ", responseJson) + if (responseJson.success === false) { + if (responseJson.reason !== undefined) { + if (setResponseMsg !== undefined) { + setResponseMsg(responseJson.reason) + } + } + + return + } + + if (inputAction !== undefined) { + console.log("In input action! Should check params if they match, and add suggestions") + + if (responseJson.parameters === undefined || responseJson.parameters.length === 0) { + return + } + + var changed = false + + for (let paramkey in inputAction.parameters) { + const actionParam = inputAction.parameters[paramkey] + + if (actionParam.autocompleted === true) { + continue + } + + if (actionParam.configuration === true && actionParam.name !== "url") { + continue + } + + if (actionParam.value !== "" && actionParam.value !== actionParam.example) { + console.log("Skipping: ", actionParam) + continue + } + + for (let respParam of responseJson.parameters) { + if (respParam.name === actionParam.name) { + console.log("Found match for param: ", respParam) + + if (respParam.value === "") { + break + } + + changed = true + + inputAction.parameters[paramkey].autocompleted = true + inputAction.parameters[paramkey].value = respParam.value + break + } + } + } + + if (changed === true) { + console.log("Setting action! Force update pls :)") + setUpdate(Math.random()) + setSelectedAction(inputAction) + } + + return + } + + console.log("Suggestionbox location: ", suggestionBox) + + // Add action + if (responseJson.app_name !== undefined && responseJson.app_name !== null) { + // Always added to 0, 0 + // Should use suggestionBox.position.x, suggestionBox.position.y + var newitem = { + "data": responseJson, + "position": { + "x": suggestionBox.node_position.x !== undefined ? suggestionBox.node_position.x : 0, + "y": suggestionBox.node_position.y !== undefined ? suggestionBox.node_position.y + 100 : 0, + }, + "group": "nodes", + } + + newitem.type = "ACTION" + newitem.isStartNode = false + newitem.data.id = uuidv4() + newitem.data.type = "ACTION" + newitem.data.isStartNode = false + + newitem.data.is_valid = true + newitem.data.isValid = true + + cy.add({ + group: newitem.group, + data: newitem.data, + position: newitem.position, + }); + + // Add edge + const newId = uuidv4() + cy.add({ + group: "edges", + data: { + id: newId, + _id: newId, + source: suggestionBox.attachedTo, + target: newitem.data.id, + } + }) + //label: "Generated", + + setSuggestionBox({ + "position": { + "top": 500, + "left": 500, + }, + "open": false, + "attachedTo": "", + }); + } + }) + .catch((error) => { + if (setSuggestionLoading !== undefined) { + setSuggestionLoading(false) + } + + console.log("Conv response error: ", error); + }); + } + + const SuggestionBoxUi = () => { + const [suggestionValue, setSuggestionValue] = useState(""); + const [suggestionLoading, setSuggestionLoading] = useState(false); + const [responseMsg, setResponseMsg] = useState(""); + + if (suggestionBox === undefined || suggestionBox.open === false) { + return false + } + + return ( +
+ {/* + + */} + + { + e.preventDefault(); + setSuggestionBox({ + "position": { + "top": 500, + "left": 500, + }, + "open": false, + "value": "", + "loading": false, + }); + }} + > + + + + { + e.preventDefault(); + aiSubmit(suggestionValue, setResponseMsg, setSuggestionLoading) + }}> + { + setSuggestionValue(e.target.value) + }} + InputProps={{ + endAdornment: ( + + + { + e.preventDefault(); + aiSubmit(suggestionValue, setResponseMsg, setSuggestionLoading) + }} /> + + + ), + }} + /> + + {suggestionLoading === true ? + + : null} + {responseMsg.length > 0 ? + + {responseMsg} + + : null} +
+ ) + } + const loadedCheck = isLoaded && workflowDone ? (
@@ -15746,6 +16301,8 @@ const AngularWorkflow = (defaultprops) => { {configureWorkflowModal} {editWorkflowModal} + + {editWorkflowModalOpen === true ? { const jsonret = getJsonObject(property.items.properties); if (property.type === "array") { - console.log("ARRAY!!") + //console.log("ARRAY!!") jsonObject[key] = [jsonret]; } else { jsonObject[key] = jsonret; @@ -309,13 +309,13 @@ const getJsonObject = (properties) => { if (property.hasOwnProperty("properties")) { const jsonret = getJsonObject(property.properties); if (property.type === "array") { - console.log("ARRAY2!!") + //console.log("ARRAY2!!") jsonObject[key] = [jsonret]; } else { jsonObject[key] = jsonret; } } else { - console.log("No items or properties found: ", property); + //console.log("No items or properties found: ", property); } } @@ -426,9 +426,7 @@ const AppCreator = (defaultprops) => { const [categories, setCategories] = useState(appCategories) - const isCloud = - window.location.host === "localhost:3002" || - window.location.host === "shuffler.io"; + const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; useEffect(() => { if (window.location.pathname.includes("apps/edit")) { @@ -612,72 +610,103 @@ const AppCreator = (defaultprops) => { setBasedata(data); console.log("Info: ", data) - if (data.info !== null && data.info !== undefined) { - if (data.info.title !== undefined && data.info.title !== null) { - if (data.info.title.length > 29) { - setName(data.info.title.slice(0, 29)); - } else { - setName(data.info.title); - } - } - setDescription(data.info.description); - document.title = "Apps - " + data.info.title; + try { + if (data.info !== null && data.info !== undefined) { + if (data.info.title !== undefined && data.info.title !== null) { + if (data.info.title.endsWith(" API")) { + data.info.title = data.info.title.substring(0, data.info.title.length - 4) + } else if (data.info.title.endsWith("API")) { + data.info.title = data.info.title.substring(0, data.info.title.length - 3) + } - if (data.info["x-logo"] !== undefined) { - if (data.info["x-logo"].url !== undefined) { - //console.log("PARSED LOGO: ", data.info["x-logo"].url); - setFileBase64(data.info["x-logo"].url); - } else { - setFileBase64(data.info["x-logo"]); - } - //console.log(""); - //console.log(""); - //console.log("LOGO: ", data.info["x-logo"]); - //console.log(""); - //console.log(""); - } + if (data.info.title.length > 29) { + setName(data.info.title.slice(0, 29)); + } else { + setName(data.info.title); + } + } - if (data.info.contact !== undefined) { - setContact(data.info.contact); - } + setDescription(data.info.description); + document.title = "Apps - " + data.info.title; - if (data.info["x-categories"] !== undefined && data.info["x-categories"].length > 0) { - if (typeof data.info["x-categories"] === "array") { - } else { - } - setNewWorkflowCategories(data.info["x-categories"]); - } - } + if (data.info["x-logo"] !== undefined) { + if (data.info["x-logo"].url !== undefined) { + //console.log("PARSED LOGO: ", data.info["x-logo"].url); + setFileBase64(data.info["x-logo"].url); + } else { + setFileBase64(data.info["x-logo"]); + } + //console.log(""); + //console.log(""); + //console.log("LOGO: ", data.info["x-logo"]); + //console.log(""); + //console.log(""); + } - if (data.tags !== undefined && data.tags.length > 0) { - var newtags = []; - for (let tagkey in data.tags) { - if (data.tags[tagkey].name.length > 50) { - console.log("Skipping tag because it's too long: ",data.tags[tagkey].name.length); + if (data.info.contact !== undefined) { + setContact(data.info.contact); + } - continue; - } + if (data.info["x-categories"] !== undefined && data.info["x-categories"].length > 0) { + if (typeof data.info["x-categories"] === "array") { + } else { + } + setNewWorkflowCategories(data.info["x-categories"]); + } + } + } catch (e) { + console.log("Failed setting info: ", e) + } - newtags.push(data.tags[tagkey].name); - } + console.log("Tags: ", data.tags) + try { + if (data.tags !== undefined && data.tags.length > 0) { + var newtags = []; + for (let tagkey in data.tags) { + if (data.tags[tagkey].name.length > 50) { + console.log("Skipping tag because it's too long: ",data.tags[tagkey].name.length); - if (newtags.length > 10) { - newtags = newtags.slice(0, 9); - } + continue; + } - setNewWorkflowTags(newtags); - } + newtags.push(data.tags[tagkey].name); + } + + if (newtags.length > 10) { + newtags = newtags.slice(0, 9); + } + + setNewWorkflowTags(newtags); + } + } catch (e) { + console.log("Failed to parse tags: ", e) + } // This is annoying (: - var securitySchemes = data.components.securityDefinitions; - if (securitySchemes === undefined) { - securitySchemes = data.securitySchemes; - } + console.log("Security schemes 1: ", data.securitySchemes) - if (securitySchemes === undefined) { - securitySchemes = data.components.securitySchemes; - } + // Weird generator problems to be handle + var securitySchemes = undefined + try { + if (data.securitySchemes !== undefined) { + securitySchemes = data.securitySchemes + if (securitySchemes === undefined) { + securitySchemes = data.securityDefinitions; + } + } + + if (securitySchemes === undefined && data.components !== undefined) { + securitySchemes = data.components.securitySchemes; + if (securitySchemes === undefined) { + securitySchemes = data.components.securityDefinitions; + } + } + } catch (e) { + console.log("Failed to parse security schemes: ", e) + } + + console.log("Security schemes 2: ", securitySchemes) const allowedfunctions = [ "GET", @@ -693,8 +722,9 @@ const AppCreator = (defaultprops) => { var newActions = []; var wordlist = {}; var all_categories = []; - console.log("Paths: ", data.paths) var parentUrl = "" + + console.log("Paths: ", data.paths) if (data.paths !== null && data.paths !== undefined) { for (let [path, pathvalue] of Object.entries(data.paths)) { @@ -708,7 +738,7 @@ const AppCreator = (defaultprops) => { // Typical YAML issue if (method !== "parameters") { console.log("Invalid method: ", method, "data: ", methodvalue); - alert.info("Skipped method (not allowed): " + method); + //alert.info("Skipped method (not allowed): " + method); } continue; } @@ -830,7 +860,7 @@ const AppCreator = (defaultprops) => { if (methodvalue["requestBody"]["content"]["application/json"]["schema"]["properties"] !== undefined) { // Read out properties from a JSON object const jsonObject = getJsonObject(methodvalue["requestBody"]["content"]["application/json"]["schema"]["properties"]) - console.log("JSON OBJECT: ", jsonObject) + //console.log("JSON OBJECT: ", jsonObject) if (jsonObject !== undefined && jsonObject !== null) { try { newaction["body"] = JSON.stringify(jsonObject, null, 2) @@ -1128,7 +1158,6 @@ const AppCreator = (defaultprops) => { if (selectedExample["content"]["application/json"]["schema"]["properties"] !== undefined && selectedExample["content"]["application/json"]["schema"]["properties"] !== null) { const jsonObject = getJsonObject(selectedExample["content"]["application/json"]["schema"]["properties"]) - console.log("ReSP Return: ", jsonObject) if (jsonObject !== undefined && jsonObject !== null) { try { newaction.example_response = JSON.stringify(jsonObject, null, 2) @@ -1483,7 +1512,8 @@ const AppCreator = (defaultprops) => { const searchactions = newActions.find( (data) => data.name === newname ); - console.log("SEARCH: ", searchactions); + + //console.log("SEARCH: ", searchactions); if (searchactions !== undefined) { newaction.errors.push("Missing name"); } else { @@ -1532,13 +1562,13 @@ const AppCreator = (defaultprops) => { } if (securitySchemes !== undefined) { - // FIXME: Should add Oauth2 (Microsoft) and JWT (Wazuh) - //console.log("SECURITY: ", securitySchemes) + + console.log("SECURITY: ", securitySchemes) var newauth = []; try { var optionset = false for (const [key, value] of Object.entries(securitySchemes)) { - console.log(key, value); + console.log("AUTH: ", key, value); if (key === "jwt") { setAuthenticationOption("JWT"); @@ -1749,7 +1779,8 @@ const AppCreator = (defaultprops) => { if (!found) { newActions2.push(action) } else { - console.log("Skipping duplicate action: ", action.url, " . Should merge contents") + console.log("NOT skipping duplicate action: ", action.url, ". Should merge contents") + newActions2.push(action) } } diff --git a/frontend/src/views/Docs.jsx b/frontend/src/views/Docs.jsx index 557ee384..e0d974f5 100644 --- a/frontend/src/views/Docs.jsx +++ b/frontend/src/views/Docs.jsx @@ -112,7 +112,6 @@ const Docs = (defaultprops) => { const SideBar = { minWidth: 250, maxWidth: 300, - borderRight: "1px solid rgba(255,255,255,0.3)", left: 0, position: "sticky", top: 50, @@ -121,6 +120,7 @@ const Docs = (defaultprops) => { overflowX: "hidden", overflowY: "auto", zIndex: 1000, + //borderRight: "1px solid rgba(255,255,255,0.3)", }; const fetchDocList = () => { @@ -361,6 +361,7 @@ const Docs = (defaultprops) => { maxWidth: "100%", minWidth: "100%", overflow: "hidden", + fontSize: isMobile ? "1.3rem" : "1.0rem", }; function OuterLink(props) { diff --git a/frontend/src/views/Welcome.jsx b/frontend/src/views/Welcome.jsx index 1305d21d..300b753e 100644 --- a/frontend/src/views/Welcome.jsx +++ b/frontend/src/views/Welcome.jsx @@ -5,6 +5,7 @@ import Stepper from "@material-ui/core/Stepper"; import Step from "@material-ui/core/Step"; import StepLabel from "@material-ui/core/StepLabel"; import AppFramework from "../components/AppFramework.jsx"; +import ArrowBackIosIcon from '@mui/icons-material/ArrowBackIos'; import { Grid, Container, @@ -303,19 +304,21 @@ const Welcome = (props) => { } const actionObject = { - padding: "50px 35px 50px 35px", + padding: "35px", + maxHeight: 300, + minHeight: 300, } const imageStyle = { width: 150, - height: 150, - margin: "auto", - marginTop: 30, + // height: 150, + // margin: "auto", + // marginTop: 10, borderRadius: 75, objectFit: "scale-down", } - - const experienced_image = userdata !== undefined && userdata !== null && userdata.active_org !== undefined && userdata.active_org.image !== undefined && userdata.active_org.image !== null && userdata.active_org.image !== "" ? userdata.active_org.image : "/images/social/shuffle_logo_round.png" + const buttonStyle = { borderRadius: 8, height: 51, width: 464, fontSize: 16, background: "linear-gradient(89.83deg, #FF8444 0.13%, #F2643B 99.84%)", padding: "16px 24px", top: 75, } + const experienced_image = userdata !== undefined && userdata !== null && userdata.active_org !== undefined && userdata.active_org.image !== undefined && userdata.active_org.image !== null && userdata.active_org.image !== "" ? userdata.active_org.image : "/images/experienced.png" return (
{/* @@ -424,14 +427,26 @@ const Welcome = (props) => { :
- - Welcome to Shuffle + {/* +
+ { + navigate("/login") + }}/> + { + navigate("/login") + }}> + Back - - Who do you identify with the most? +
+ */} + + Help us get to know you. + + + Let us help you create a smoother journey.
-
+
{ if (isCloud) { ReactGA.event({ @@ -446,20 +461,20 @@ const Welcome = (props) => { setShowWelcome(true) }}> - + + New to Shuffle - - - Follow our short introduction and learn some tips and tricks + + Let us guide you for an easier experience.
- + {/* OR - + */}
{ if (isCloud) { @@ -473,17 +488,24 @@ const Welcome = (props) => { navigate("/workflows?message=Skipped intro") }}> - + + Experienced - - - - - You know Shuffle well. Head to your organization right away! + + + You will head to Shuffle right away.
+ +
+ +
{/*
{ if (window.drift !== undefined) { diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index c6a0f659..83ebaf7d 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -2,7 +2,7 @@ module orborus go 1.19 -//replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared +replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared require ( github.com/docker/docker v23.0.0+incompatible @@ -27,6 +27,7 @@ require ( github.com/frikky/go-elasticsearch/v8 v8.13.1 // indirect github.com/frikky/kin-openapi v0.41.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/swag v0.19.5 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -44,7 +45,11 @@ require ( github.com/opencontainers/image-spec v1.0.2 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect + github.com/tklauser/go-sysconf v0.3.11 // indirect + github.com/tklauser/numcpus v0.6.0 // indirect + github.com/yusufpapurcu/wmi v1.2.2 // indirect go.opencensus.io v0.22.5 // indirect go4.org v0.0.0-20201209231011-d4a079459e60 // indirect golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect @@ -52,7 +57,7 @@ require ( golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect golang.org/x/oauth2 v0.0.0-20210113160501-8b1d76fa0423 // indirect - golang.org/x/sys v0.1.0 // indirect + golang.org/x/sys v0.2.0 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/tools v0.1.12 // indirect google.golang.org/api v0.36.0 // indirect diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index 93ce991d..490fabe4 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -86,6 +86,8 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= @@ -194,6 +196,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shuffle/shuffle-shared v0.3.51 h1:+JPEGw6R4a320who+SrGP/VqBxZdKPdcLw/07cO7d6c= github.com/shuffle/shuffle-shared v0.3.51/go.mod h1:jQrYySmvp/0De5ftrAaY6xwwr7TMfqBmBxQ2AX9yrjQ= github.com/shuffle/shuffle-shared v0.3.52 h1:d9OycFpuWxrcgHdP2vplKAkY8n+oK5vW02vRK9X+azs= @@ -214,10 +218,16 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= +github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= +github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= +github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= +github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -330,6 +340,7 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -354,6 +365,8 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 940503b3..abec33ee 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -4,11 +4,6 @@ package main Orborus exists to listen for new workflow executions whcih are deployed as workers. */ -// FIXME: -// 2022/01/12 17:13:36 [WARNING] Swarm init: Error response from daemon: manager stopped: failed to listen on remote API address: listen tcp: address tcp/2377%!(EXTRA string=172.23.0.2): unknown port - -// frikky@debian:~/git/shuffle/functions/onprem/worker$ docker service create --replicas 5 --name shuffle-workers --env SHUFFLE_SWARM_CONFIG=run --publish published=33333,target=33333 ghcr.io/shuffle/shuffle-worker:nightly - // Potential issues: // Default network could be same as on the host // Ingress network may not exist (default) @@ -43,15 +38,14 @@ import ( dockerclient "github.com/docker/docker/client" uuid "github.com/satori/go.uuid" - //network "github.com/docker/docker/api/types/network" - //natting "github.com/docker/go-connections/nat" - "github.com/mackerelio/go-osstat/cpu" + //"github.com/mackerelio/go-osstat/disk" "github.com/mackerelio/go-osstat/memory" + "github.com/shirou/gopsutil/cpu" ) // Starts jobs in bulk, so this could be increased var sleepTime = 3 -var maxConcurrency = 10 +var maxConcurrency = 25 // Timeout if something rashes var workerTimeoutEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_TIMEOUT") @@ -90,6 +84,7 @@ var executionIds = []string{} var dockercli *dockerclient.Client var containerId string +var executionCount = 0 func init() { var err error @@ -714,18 +709,18 @@ func initializeImages() { ctx := context.Background() if appSdkVersion == "" { - appSdkVersion = "1.1.0" + appSdkVersion = "latest" log.Printf("[WARNING] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %s", appSdkVersion) } if workerVersion == "" { - workerVersion = "1.1.0" + workerVersion = "latest" log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %s", workerVersion) } if baseimageregistry == "" { - baseimageregistry = "docker.io" - baseimageregistry = "ghcr.io" + baseimageregistry = "docker.io" // Dockerhub + baseimageregistry = "ghcr.io" // Github log.Printf("[DEBUG] Setting baseimageregistry") } @@ -744,6 +739,7 @@ func initializeImages() { // check whether they are the same first images := []string{ + fmt.Sprintf("frikky/shuffle:app_sdk"), fmt.Sprintf("shuffle/shuffle:app_sdk"), fmt.Sprintf("%s/%s/shuffle-app_sdk:%s", baseimageregistry, baseimagename, appSdkVersion), newWorker, @@ -763,40 +759,6 @@ func initializeImages() { } } -// Will be used for checking if there's enough to deploy based on a threshold -// E.g. having maximum CPU and maxmimum RAM -// Does this work containerized? -func getStats() { - fmt.Printf("\n") - - memory, err := memory.Get() - if err != nil { - fmt.Fprintf(os.Stderr, "%s\n", err) - return - } - - before, err := cpu.Get() - if err != nil { - fmt.Fprintf(os.Stderr, "%s\n", err) - return - } - time.Sleep(time.Duration(250) * time.Millisecond) - after, err := cpu.Get() - if err != nil { - fmt.Fprintf(os.Stderr, "%s\n", err) - return - } - total := float64(after.Total - before.Total) - - fmt.Printf("[INFO] memory total: %d bytes\n", memory.Total) - fmt.Printf("[INFO] memory used: %d bytes\n", memory.Used) - fmt.Printf("[INFO] cpu used : %f%%\n", float64(after.User-before.User)/total*100) - fmt.Printf("[INFO] cpu system: %f%%\n", float64(after.System-before.System)/total*100) - fmt.Printf("[INFO] cpu idle : %f%%\n", float64(after.Idle-before.Idle)/total*100) - - fmt.Printf("\n") -} - func findActiveSwarmNodes() (int64, error) { ctx := context.Background() nodes, err := dockercli.NodeList(ctx, types.NodeListOptions{}) @@ -857,6 +819,76 @@ func checkSwarmService(ctx context.Context) { log.Printf("[DEBUG] Swarm info: %s\n\n", ret) } +func getOrborusStats() shuffle.OrborusStats { + newStats := shuffle.OrborusStats{ + OrgId: org, + Environment: environment, + OrborusLabel: orborusLabel, + Timestamp: time.Now().Unix(), + } + + if swarmConfig == "run" || swarmConfig == "swarm" { + newStats.Swarm = true + } + + if runningMode == "kubernetes" || runningMode == "k8s" { + newStats.Kubernetes = true + } + + newStats.PollTime = sleepTime + newStats.MaxQueue = maxConcurrency + newStats.Queue = executionCount + + // Get CPU usage and max CPU + /* + before, err := cpu.Get() + if err != nil { + log.Printf("[ERROR] Failed getting CPU stats: %s", err) + } else { + newStats.CPU = int(before.User) + newStats.MaxCPU = int(before.Total) + } + */ + + cpuPercent, err := cpu.Percent(250*time.Millisecond, false) + if err == nil && len(cpuPercent) > 0 { + newStats.CPUPercent = cpuPercent[0] + } + //Percent(interval time.Duration, percpu bool) ([]float64, error) + + // Get memory usage + memory, err := memory.Get() + if err != nil { + log.Printf("[ERROR] Failed getting memory stats: %s", err) + } else { + newStats.Memory = int(memory.Used) + newStats.MaxMemory = int(memory.Total) + } + + // Get disk usage + /* + disk, err := disk.Get() + if err != nil { + log.Printf("[ERROR] Failed getting disk stats: %s", err) + } else { + newStats.Disk = int(disk.Used) + newStats.MaxDisk = int(disk.Total) + } + */ + + /* + // General + Disk int `json:"disk"` + + // Docker + AppContainers int `json:"app_containers"` + WorkerContainers int `json:"worker_containers"` + TotalContainers int `json:"total_containers"` + } + */ + return newStats +} + // Initial loop etc func main() { startupDelay := os.Getenv("SHUFFLE_ORBORUS_STARTUP_DELAY") @@ -867,7 +899,7 @@ func main() { if err == nil { time.Sleep(time.Duration(tmpInt) * time.Second) } else { - log.Printf("[WARNING] Env SHUFFLE_ORBORUS_STARTUP_DELAY must be a number, not %s", startupDelay) + log.Printf("[WARNING] Env SHUFFLE_ORBORUS_STARTUP_DELAY must be a number, not '%s'. Using default.", startupDelay) } } @@ -892,6 +924,8 @@ func main() { timezone = "Europe/Amsterdam" } + log.Printf("[INFO] Using environment '%s' with timezone %s", environment, timezone) + if len(os.Getenv("SHUFFLE_ORBORUS_PULL_TIME")) > 0 { log.Printf("[INFO] Trying to set Orborus sleep time between polls to %s", os.Getenv("SHUFFLE_ORBORUS_PULL_TIME")) @@ -901,8 +935,6 @@ func main() { } } - log.Printf("[INFO] Running with timezone %s", timezone) - workerTimeout := 600 if workerTimeoutEnv != "" { tmpInt, err := strconv.Atoi(workerTimeoutEnv) @@ -941,7 +973,7 @@ func main() { if environment == "" { environment = "onprem" - log.Printf("[INFO] Defaulting to environment name %s. Set environment variable ENVIRONMENT_NAME to change. This should be the same as in the frontend action.", environment) + log.Printf("[WARNING] Defaulting to environment name %s. Set environment variable ENVIRONMENT_NAME to change. This should be the same as in the frontend action.", environment) } // FIXME - during init, BUILD and/or LOAD worker and app_sdk @@ -970,14 +1002,17 @@ func main() { //deployServiceWorkers(workerImage) } - log.Printf("[INFO] Finished configuring docker environment") - client := shuffle.GetExternalClient(baseUrl) fullUrl := fmt.Sprintf("%s/api/v1/workflows/queue", baseUrl) + log.Printf("[INFO] Finished configuring docker environment. Connecting to %s", fullUrl) + + forwardData := bytes.NewBuffer([]byte{}) + forwardMethod := "POST" + req, err := http.NewRequest( - "GET", + forwardMethod, fullUrl, - nil, + forwardData, ) if err != nil { @@ -986,9 +1021,9 @@ func main() { } zombiecounter := 0 + req.Header.Add("Content-Type", "application/json") req.Header.Add("Org-Id", environment) - if len(auth) > 0 { req.Header.Add("Authorization", auth) } @@ -998,7 +1033,7 @@ func main() { } if len(orborusLabel) > 0 { - log.Printf("[DEBUG] Sending with Label %s", orborusLabel) + log.Printf("[DEBUG] Sending with Label '%s'", orborusLabel) req.Header.Add("X-Orborus-Label", orborusLabel) } @@ -1011,8 +1046,19 @@ func main() { log.Printf("[INFO] Waiting for executions at %s with Environment %#v", fullUrl, environment) hasStarted := false for { - //go getStats() - //log.Printf("[DEBUG] Prerequest - queue") + if req.Method == "POST" { + // Should find data to send (memory etc.) + + orborusStats := getOrborusStats() + // Marshal and set body + jsonData, err := json.Marshal(orborusStats) + if err == nil { + req.Body = ioutil.NopCloser(bytes.NewBuffer(jsonData)) + } else { + log.Printf("[ERROR] Failed marshalling json: %s", err) + } + } + newresp, err := client.Do(req) //log.Printf("[DEBUG] Postrequest - queue") if err != nil { @@ -1027,6 +1073,16 @@ func main() { continue } + if newresp.StatusCode == 405 { + log.Printf("[WARNING] Received 405 from %s. This is likely due to a misconfigured base URL. Automatically swapping to GET request (backwards compatibility)", fullUrl) + + req.Method = "GET" + req.Body = nil + + //time.Sleep(time.Duration(sleepTime) * time.Second) + continue + } + body, err := ioutil.ReadAll(newresp.Body) if err != nil { log.Printf("[ERROR] Failed reading body from Shuffle: %s", err) @@ -1082,7 +1138,7 @@ func main() { } // Anything below here verifies concurrency - executionCount := getRunningWorkers(ctx, workerTimeout) + executionCount = getRunningWorkers(ctx, workerTimeout) if executionCount >= maxConcurrency { if zombiecounter*sleepTime > workerTimeout { go zombiecheck(ctx, workerTimeout)