From 0c9931a70fe51682fb4db9e94377a4dc164097c6 Mon Sep 17 00:00:00 2001 From: Frikky Date: Wed, 25 Jun 2025 22:05:42 +0200 Subject: [PATCH 01/59] Minor environment / org_id name changes --- .env | 1 - backend/go-app/main.go | 4 ++-- backend/go-app/walkoff.go | 2 ++ 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.env b/.env index 53204db2..0b1e8cee 100755 --- a/.env +++ b/.env @@ -1,5 +1,4 @@ # Default execution environment for workers -ORG_ID=Shuffle ENVIRONMENT_NAME=Shuffle # Sanitize liquid.py input diff --git a/backend/go-app/main.go b/backend/go-app/main.go index cfbc768a..bd939374 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -638,7 +638,7 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) { } else { log.Printf("[DEBUG] Successfully created the default org!") - defaultEnv := os.Getenv("ORG_ID") + defaultEnv := os.Getenv("ENVIRONMENT_NAME") if len(defaultEnv) == 0 { defaultEnv = "Shuffle" log.Printf("[DEBUG] Setting default environment for org to %s", defaultEnv) @@ -3925,7 +3925,7 @@ func runInitEs(ctx context.Context) { log.Printf("[INFO] Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy) } - defaultEnv := os.Getenv("ORG_ID") + defaultEnv := os.Getenv("ENVIRONMENT_NAME") if len(defaultEnv) == 0 { defaultEnv = "Shuffle" log.Printf("[DEBUG] Setting default environment for org to %s", defaultEnv) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 2331a273..c194d091 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -292,6 +292,8 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { log.Printf("[WARNING] Failed handling Orborus failover: %s", err) } + log.Printf("[DEBUG] Issue with environment ID: %s", orgId) + return } From 4f48d712521b867dac2dc4e8f80fb2e501b0ec62 Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 26 Jun 2025 00:24:50 +0200 Subject: [PATCH 02/59] Removed more old files --- functions/extensions/aws-s3-lambda/README.md | 2 - .../extensions/aws-s3-lambda/s3_function.py | 22 - .../Shuffle/requirements.txt | 2 - .../cortex-responders/Shuffle/shuffle.json | 43 -- .../cortex-responders/Shuffle/shuffle.py | 27 - .../Shuffle_Webhook/requirements.txt | 2 - .../Shuffle_Webhook/shuffle_webhook.json | 28 - .../Shuffle_Webhook/shuffle_webhook.py | 28 - functions/extensions/elasticsearch/run.sh | 19 - functions/extensions/k8s/shuffle/.helmignore | 23 - functions/extensions/k8s/shuffle/Chart.yaml | 24 - .../k8s/shuffle/templates/NOTES.txt | 22 - .../k8s/shuffle/templates/_helpers.tpl | 62 -- .../k8s/shuffle/templates/deployment.yaml | 253 -------- .../extensions/k8s/shuffle/templates/hpa.yaml | 28 - .../k8s/shuffle/templates/ingress.yaml | 61 -- .../shuffle/templates/persistent-volume.yaml | 24 - .../templates/persistentvolumeclaim.yaml | 17 - .../k8s/shuffle/templates/service.yaml | 52 -- .../k8s/shuffle/templates/serviceaccount.yaml | 12 - .../templates/tests/test-connection.yaml | 15 - functions/extensions/k8s/shuffle/values.yaml | 82 --- functions/extensions/misp/Dockerfile | 23 - functions/extensions/misp/docker-compose.yml | 9 - functions/extensions/misp/requirements.txt | 2 - functions/extensions/misp/sub.py | 57 -- .../extensions/scripts/disable_disk_check.sh | 11 - functions/extensions/swarm/.gitignore | 1 - functions/extensions/swarm/docker-compose.yml | 131 ----- functions/extensions/swarm/network.sh | 1 - functions/extensions/swarm/orborus.yml | 39 -- functions/extensions/swarm/run.sh | 4 - functions/extensions/swarm/run_orborus.sh | 4 - functions/extensions/swarm/shuffle-apps/tmp | 0 .../extensions/swarm/shuffle-database/tmp | 0 functions/extensions/swarm/shuffle-files/tmp | 0 functions/extensions/swarm/stop.sh | 3 - functions/extensions/wazuh/custom-shuffle | 36 -- functions/extensions/wazuh/custom-shuffle.py | 206 ------- functions/extensions/wazuh/ossec.conf | 6 - functions/usecases/README.md | 16 - functions/usecases/categories.json | 260 -------- functions/usecases/categories.rtf | 555 ------------------ functions/usecases/read_categories.py | 66 --- 44 files changed, 2278 deletions(-) delete mode 100644 functions/extensions/aws-s3-lambda/README.md delete mode 100644 functions/extensions/aws-s3-lambda/s3_function.py delete mode 100755 functions/extensions/cortex-responders/Shuffle/requirements.txt delete mode 100755 functions/extensions/cortex-responders/Shuffle/shuffle.json delete mode 100755 functions/extensions/cortex-responders/Shuffle/shuffle.py delete mode 100755 functions/extensions/cortex-responders/Shuffle_Webhook/requirements.txt delete mode 100755 functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.json delete mode 100755 functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.py delete mode 100755 functions/extensions/elasticsearch/run.sh delete mode 100755 functions/extensions/k8s/shuffle/.helmignore delete mode 100755 functions/extensions/k8s/shuffle/Chart.yaml delete mode 100755 functions/extensions/k8s/shuffle/templates/NOTES.txt delete mode 100755 functions/extensions/k8s/shuffle/templates/_helpers.tpl delete mode 100755 functions/extensions/k8s/shuffle/templates/deployment.yaml delete mode 100755 functions/extensions/k8s/shuffle/templates/hpa.yaml delete mode 100755 functions/extensions/k8s/shuffle/templates/ingress.yaml delete mode 100755 functions/extensions/k8s/shuffle/templates/persistent-volume.yaml delete mode 100755 functions/extensions/k8s/shuffle/templates/persistentvolumeclaim.yaml delete mode 100755 functions/extensions/k8s/shuffle/templates/service.yaml delete mode 100755 functions/extensions/k8s/shuffle/templates/serviceaccount.yaml delete mode 100755 functions/extensions/k8s/shuffle/templates/tests/test-connection.yaml delete mode 100755 functions/extensions/k8s/shuffle/values.yaml delete mode 100755 functions/extensions/misp/Dockerfile delete mode 100755 functions/extensions/misp/docker-compose.yml delete mode 100755 functions/extensions/misp/requirements.txt delete mode 100755 functions/extensions/misp/sub.py delete mode 100755 functions/extensions/scripts/disable_disk_check.sh delete mode 100644 functions/extensions/swarm/.gitignore delete mode 100644 functions/extensions/swarm/docker-compose.yml delete mode 100644 functions/extensions/swarm/network.sh delete mode 100644 functions/extensions/swarm/orborus.yml delete mode 100644 functions/extensions/swarm/run.sh delete mode 100644 functions/extensions/swarm/run_orborus.sh delete mode 100644 functions/extensions/swarm/shuffle-apps/tmp delete mode 100644 functions/extensions/swarm/shuffle-database/tmp delete mode 100644 functions/extensions/swarm/shuffle-files/tmp delete mode 100644 functions/extensions/swarm/stop.sh delete mode 100755 functions/extensions/wazuh/custom-shuffle delete mode 100755 functions/extensions/wazuh/custom-shuffle.py delete mode 100755 functions/extensions/wazuh/ossec.conf delete mode 100644 functions/usecases/README.md delete mode 100644 functions/usecases/categories.json delete mode 100644 functions/usecases/categories.rtf delete mode 100644 functions/usecases/read_categories.py diff --git a/functions/extensions/aws-s3-lambda/README.md b/functions/extensions/aws-s3-lambda/README.md deleted file mode 100644 index f665851a..00000000 --- a/functions/extensions/aws-s3-lambda/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# AWS Lambda forwarder to Shuffle -This function is made to forward S3 notifications to Shuffle to run a workflow when an object is made or updated. diff --git a/functions/extensions/aws-s3-lambda/s3_function.py b/functions/extensions/aws-s3-lambda/s3_function.py deleted file mode 100644 index d83d6154..00000000 --- a/functions/extensions/aws-s3-lambda/s3_function.py +++ /dev/null @@ -1,22 +0,0 @@ -import json -import urllib.parse -import urllib3 -import os - -print('Loading function') - -def lambda_handler(event, context): - # Get the object from the event and show its content type - bucket = event['Records'][0]['s3']['bucket']['name'] - - webhook = os.environ.get("SHUFFLE_WEBHOOK") - if not webhook: - return "No webhook environment defined: SHUFFLE_WEBHOOK" - - http = urllib3.PoolManager() - ret = http.request('POST', webhook, body=json.dumps(event["Records"][0]).encode("utf-8")) - if ret.status != 200: - return "Bad status code for webhook: %d" % ret.status - - print("Status code: %d\nData: %s" % (ret.status, ret.data)) - return "Successfully started with data %s" % ret.data diff --git a/functions/extensions/cortex-responders/Shuffle/requirements.txt b/functions/extensions/cortex-responders/Shuffle/requirements.txt deleted file mode 100755 index 6aabc3cf..00000000 --- a/functions/extensions/cortex-responders/Shuffle/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -cortexutils -requests diff --git a/functions/extensions/cortex-responders/Shuffle/shuffle.json b/functions/extensions/cortex-responders/Shuffle/shuffle.json deleted file mode 100755 index db755b49..00000000 --- a/functions/extensions/cortex-responders/Shuffle/shuffle.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "name": "Shuffle", - "version": "1.0", - "author": "@frikkylikeme", - "url": "https://github.com/frikky/shuffle", - "license": "AGPL-V3", - "description": "Execute a workflow in Shuffle", - "dataTypeList": ["thehive:case", "thehive:alert", "thehive:case_artifact"], - "command": "Shuffle/shuffle.py", - "baseConfig": "Shuffle", - "configurationItems": [ - { - "name": "url", - "description": "The URL to your shuffle instance", - "type": "string", - "multi": false, - "required": true, - "defaultValue": "https://shuffler.io" - }, - { - "name": "api_key", - "description": "The API key to your Shuffle user", - "type": "string", - "multi": false, - "required": true - }, - { - "name": "verifyssl", - "description": "Verify SSL certificate", - "type": "boolean", - "multi": false, - "required": true, - "defaultValue": true - }, - { - "name": "workflow_id", - "description": "The ID of the workflow to execute", - "type": "string", - "multi": false, - "required": true - } - ] -} diff --git a/functions/extensions/cortex-responders/Shuffle/shuffle.py b/functions/extensions/cortex-responders/Shuffle/shuffle.py deleted file mode 100755 index db343230..00000000 --- a/functions/extensions/cortex-responders/Shuffle/shuffle.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python3 -#encoding: utf-8 - -from cortexutils.responder import Responder -import requests - -class Shuffle(Responder): - def __init__(self): - Responder.__init__(self) - self.api_key = self.get_param("config.api_key", "") - self.url = self.get_param("config.url", "") - self.workflow_id = self.get_param("config.workflow_id", "") - self.verify = self.get_param('config.verifyssl', True, None) - - def run(self): - Responder.run(self) - parsed_url = "%s/api/v1/workflows/%s/execute" % (self.url, self.workflow_id) - headers = { - "Authorization": "Bearer %s" % self.api_key, - "User-Agent": "Cortex-Analyzer" - } - requests.post(parsed_url, headers=headers,verify=self.verify) - - self.report({'message': 'message sent'}) - -if __name__ == '__main__': - Shuffle().run() diff --git a/functions/extensions/cortex-responders/Shuffle_Webhook/requirements.txt b/functions/extensions/cortex-responders/Shuffle_Webhook/requirements.txt deleted file mode 100755 index 6aabc3cf..00000000 --- a/functions/extensions/cortex-responders/Shuffle_Webhook/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -cortexutils -requests diff --git a/functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.json b/functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.json deleted file mode 100755 index a6d3e9d8..00000000 --- a/functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "Shuffle_webhook", - "version": "1.0", - "author": "@azgaviperr", - "url": "https://github.com/frikky/shuffle", - "license": "AGPL-V3", - "description": "Execute a webhook in Shuffle", - "dataTypeList": ["thehive:case", "thehive:alert", "thehive:case_artifact"], - "command": "Shuffle_Webhook/shuffle_webhook.py", - "baseConfig": "Shuffle_Webhook", - "configurationItems": [ - { - "name": "webhook_url", - "description": "The URL to your shuffle instance", - "type": "string", - "multi": false, - "required": true - }, - { - "name": "verifyssl", - "description": "Verify SSL certificate", - "type": "boolean", - "multi": false, - "required": true, - "defaultValue": true - } - ] -} diff --git a/functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.py b/functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.py deleted file mode 100755 index daed501c..00000000 --- a/functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python3 -#encoding: utf-8 - -from cortexutils.responder import Responder -import requests - -class Shuffle(Responder): - def __init__(self): - Responder.__init__(self) - self.api_key = self.get_param("config.api_key", "") - self.webhook_url = self.get_param("config.webhook_url", "") - self.webhook_id = self.get_param("config.webhook_id", "") - self.verify = self.get_param('config.verifyssl', True, None) - self.data = self.get_param('data') - - def run(self): - Responder.run(self) - headers = { - "Content-Type": "application/json", - "Accept": "application/json", - "User-Agent": "Cortex-Analyzer" - } - requests.post(self.webhook_url, headers=headers,verify=self.verify, json=self.data) - - self.report({'message': 'message sent'}) - -if __name__ == '__main__': - Shuffle().run() diff --git a/functions/extensions/elasticsearch/run.sh b/functions/extensions/elasticsearch/run.sh deleted file mode 100755 index 666b1d95..00000000 --- a/functions/extensions/elasticsearch/run.sh +++ /dev/null @@ -1,19 +0,0 @@ -#docker run -d -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -e ELASTICSEARCH_USERNAME=frikky -e ELASTICSEARCH_PASSWORD=likeme -e xpack.security.enabled=true docker.elastic.co/elasticsearch/elasticsearch:7.12.1 -#docker run -d -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:7.12.1 -# -# -# -#echo "\nWaiting for 1.5 minute, then adding data" -#sleep 90 -#echo "\nSlept 90 seconds: ADDING DATA" -#curl -XPOST http://localhost:9200/_security/user/frikky -H "Content-Type: application/json" -d '{"enabled": true, "email": "frikky@shuffler.io"}' - -#curl -XPOST -u frikky:likeme http://localhost:9200/samples/_doc -H "Content-Type: application/json" -d '{"src": "122.14.137.67", "dst": "103.35.191.16", "message": "alert", "md5": "CAEF973033E593C625FB2AA34F7026DC", "sha256": "DB1AEC5222075800EDA75D7205267569679B424E5C58A28102417F46D3B5790D", "hits": 0}' -#echo -#curl -XPOST -u frikky:likeme http://localhost:9200/samples/_doc -H "Content-Type: application/json" -d '{"src": "134.119.219.71", "dst": "103.35.191.41", "message": "alert", "md5": "9498FF82A64FF445398C8426ED63EA5B", "sha256": "8B2E701E91101955C73865589A4C72999AEABC11043F712E05FDB1C17C4AB19A", "hits": 0}' -# -#echo -#curl -XPOST -u frikky:likeme http://localhost:9200/samples2/_doc -H "Content-Type: application/json" -d '{"src": "122.14.137.67", "dst": "103.35.191.16", "message": "alert", "md5": "CAEF973033E593C625FB2AA34F7026DC", "sha256": "DB1AEC5222075800EDA75D7205267569679B424E5C58A28102417F46D3B5790D"}' -#echo -#curl -XPOST -u frikky:likeme http://localhost:9200/samples2/_doc -H "Content-Type: application/json" -d '{"src": "134.119.219.71", "dst": "103.35.191.41", "message": "alert", "md5": "9498FF82A64FF445398C8426ED63EA5B", "sha256": "8B2E701E91101955C73865589A4C72999AEABC11043F712E05FDB1C17C4AB19A"}' -#echo diff --git a/functions/extensions/k8s/shuffle/.helmignore b/functions/extensions/k8s/shuffle/.helmignore deleted file mode 100755 index 0e8a0eb3..00000000 --- a/functions/extensions/k8s/shuffle/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/functions/extensions/k8s/shuffle/Chart.yaml b/functions/extensions/k8s/shuffle/Chart.yaml deleted file mode 100755 index eb53a0b5..00000000 --- a/functions/extensions/k8s/shuffle/Chart.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: v2 -name: shuffle -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.2.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "2.0.0" diff --git a/functions/extensions/k8s/shuffle/templates/NOTES.txt b/functions/extensions/k8s/shuffle/templates/NOTES.txt deleted file mode 100755 index eaba4460..00000000 --- a/functions/extensions/k8s/shuffle/templates/NOTES.txt +++ /dev/null @@ -1,22 +0,0 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "shuffle.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "shuffle.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "shuffle.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "shuffle.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} diff --git a/functions/extensions/k8s/shuffle/templates/_helpers.tpl b/functions/extensions/k8s/shuffle/templates/_helpers.tpl deleted file mode 100755 index 66efef07..00000000 --- a/functions/extensions/k8s/shuffle/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "shuffle.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "shuffle.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "shuffle.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "shuffle.labels" -}} -helm.sh/chart: {{ include "shuffle.chart" . }} -{{ include "shuffle.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "shuffle.selectorLabels" -}} -app.kubernetes.io/name: {{ include "shuffle.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "shuffle.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "shuffle.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/functions/extensions/k8s/shuffle/templates/deployment.yaml b/functions/extensions/k8s/shuffle/templates/deployment.yaml deleted file mode 100755 index f2050d82..00000000 --- a/functions/extensions/k8s/shuffle/templates/deployment.yaml +++ /dev/null @@ -1,253 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Values.name }}frontend - namespace: {{ .Values.namespace | quote }} -spec: - replicas: 1 - selector: - matchLabels: - service: shuffle - app: shuffle-frontend - template: - metadata: - labels: - service: shuffle - app: shuffle-frontend - spec: - containers: - - name: shuffle-frontend - image: ghcr.io/frikky/shuffle-frontend:nightly - imagePullPolicy: {{ .Values.image.pullPolicy | quote }} - env: - - name: BACKEND_HOSTNAME - value: backend-service - - name: TZ - value: Asia/Shanghai - ports: - - name: http - containerPort: 80 - hostPort: 3001 - - name: https - containerPort: 443 - hostname: shuffle-frontend - restartPolicy: Always - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Values.name }}backend - namespace: {{ .Values.namespace | quote }} -spec: - replicas: 1 - selector: - matchLabels: - service: shuffle - app: shuffle-backend - template: - metadata: - labels: - service: shuffle - app: shuffle-backend - spec: - containers: - - name: shuffle-backend - image: ghcr.io/frikky/shuffle-backend:nightly - env: - - name: BACKEND_HOSTNAME - value: "backend-service" - - name: BACKEND_PORT - value: "5001" - - name: ENVIRONMENT_NAME - value: "Shuffle" - - name: HTTPS_PROXY - - name: HTTP_PROXY - - name: ORG_ID - value: "Shuffle" - - name: OUTER_HOSTNAME - value: "backend-service" - - name: SHUFFLE_APP_FORCE_UPDATE - value: "false" - - name: SHUFFLE_APP_HOTLOAD_FOLDER - value: "/shuffle-apps" - - name: SHUFFLE_APP_HOTLOAD_LOCATION - value: "/shuffle-apps" - - name: SHUFFLE_CONTAINER_AUTO_CLEANUP - value: "false" - - name: SHUFFLE_DEFAULT_APIKEY - - name: SHUFFLE_DEFAULT_PASSWORD - - name: SHUFFLE_DEFAULT_USERNAME - - name: SHUFFLE_DOWNLOAD_AUTH_BRANCH - - name: SHUFFLE_DOWNLOAD_AUTH_PASSWORD - - name: SHUFFLE_DOWNLOAD_AUTH_USERNAME - - name: SHUFFLE_DOWNLOAD_WORKFLOW_BRANCH - - name: SHUFFLE_DOWNLOAD_WORKFLOW_LOCATION - - name: SHUFFLE_DOWNLOAD_WORKFLOW_PASSWORD - - name: SHUFFLE_DOWNLOAD_WORKFLOW_USERNAME - - name: SHUFFLE_ELASTIC - value: "true" - - name: SHUFFLE_OPENSEARCH_APIKEY - - name: SHUFFLE_OPENSEARCH_CERTIFICATE_FILE - - name: SHUFFLE_OPENSEARCH_CLOUDID - - name: SHUFFLE_OPENSEARCH_PASSWORD - - name: SHUFFLE_OPENSEARCH_PROXY - - name: SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY - value: "true" - - name: SHUFFLE_OPENSEARCH_URL - value: http://opensearch-service:9200 - - name: SHUFFLE_OPENSEARCH_USERNAME - value: "" - - name: SHUFFLE_PASS_APP_PROXY - value: "FALSE" - - name: SHUFFLE_PASS_WORKER_PROXY - value: "FALSE" - volumeMounts: - - mountPath: /var/run/docker.sock - name: docker-sock - - mountPath: /shuffle-apps - name: shuffle-app-hotload-location - - mountPath: /shuffle-files - name: shuffle-file-location - hostname: shuffle-backend - volumes: - - name: docker-sock - hostPath: - path: /var/run/docker.sock - - name: shuffle-app-hotload-location - hostPath: - path: /data/kubernetes/shuffle-apps - type: DirectoryOrCreate - - name: shuffle-file-location - hostPath: - path: /data/kubernetes/shuffle-files - type: DirectoryOrCreate - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Values.name }}orborus - namespace: {{ .Values.namespace | quote }} -spec: - replicas: 1 - selector: - matchLabels: - service: shuffle - app: shuffle-orborus - template: - metadata: - labels: - service: shuffle - app: shuffle-orborus - spec: - containers: - - name: shuffle-orborus - image: ghcr.io/frikky/shuffle-orborus:nightly - env: - - name: RUNNING_MODE - value: kubernetes - - name: BASE_URL - value: http://backend-service:5001 - - name: CLEANUP - value: "false" - - name: DOCKER_API_VERSION - value: "1.40" - - name: ENVIRONMENT_NAME - value: Shuffle - - name: HTTPS_PROXY - - name: HTTP_PROXY - - name: ORG_ID - value: Shuffle - - name: SHUFFLE_APP_SDK_VERSION - value: 0.8.97 - - name: SHUFFLE_BASE_IMAGE_NAME - value: frikky - - name: SHUFFLE_BASE_IMAGE_REGISTRY - value: ghcr.io - - name: SHUFFLE_BASE_IMAGE_TAG_SUFFIX - value: "-0.8.80" - - name: SHUFFLE_ORBORUS_EXECUTION_TIMEOUT - value: "600" - - name: SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY - value: "50" - - name: SHUFFLE_PASS_WORKER_PROXY - value: "TRUE" - - name: SHUFFLE_WORKER_VERSION - value: nightly - - name: TZ - value: Asia/Shanghai - volumeMounts: - - mountPath: /var/run/docker.sock - name: docker-sock - hostname: shuffle-orborus - volumes: - - name: docker-sock - hostPath: - path: /var/run/docker.sock -#--- -#apiVersion: apps/v1 -#kind: Deployment -#metadata: -# name: {{ .Values.name }}opensearch -# namespace: {{ .Values.namespace | quote }} -#spec: -# replicas: 1 -# selector: -# matchLabels: -# service: shuffle -# app: shuffle-opensearch -# template: -# metadata: -# labels: -# service: shuffle -# app: shuffle-opensearch -# spec: -# nodeSelector: -# node.bdlab-venus.com/opensearch: available -# initContainers: -# - name: permissions-fix -# image: frikky/busybox -# #volumeMounts: -# # - name: opensearch-claim0 -# # mountPath: /usr/share/elasticsearch/data -# command: [ 'chown' ] -# args: [ '1000:1000', '/usr/share/elasticsearch/data' ] -# containers: -# - name: shuffle-opensearch -# image: opensearchproject/opensearch:1.0.1 -# env: -# - name: TZ -# value: Asia/Shanghai -# - name: bootstrap.memory_lock -# value: "false" -# - name: OPENSEARCH_JAVA_OPTS -# value: "-Xms1024m -Xmx1024m" -# - name: opendistro_security.disabled -# value: "true" -# - name: cluster.routing.allocation.disk.threshold_enabled -# value: "false" -# - name: cluster.name -# value: shuffle-cluster -# - name: node.name -# value: opensearch-service -# - name: discovery.seed_hosts -# value: opensearch-service -# - name: cluster.initial_master_nodes -# value: opensearch-service - # volumeMounts: - # - mountPath: /usr/share/opensearch/data - # name: opensearch-claim0 - #volumes: - # - name: opensearch-claim0 - # persistentVolumeClaim: - # claimName: opensearch-claim0 -# volumeMounts: -# - mountPath: /usr/share/opensearch/data -# readOnly: true -# name: db-location -# volumes: -# - name: db-location -# hostPath: -# path: /data/kubernetes/shuffle-opensearch -# type: DirectoryOrCreate diff --git a/functions/extensions/k8s/shuffle/templates/hpa.yaml b/functions/extensions/k8s/shuffle/templates/hpa.yaml deleted file mode 100755 index 49f84ca2..00000000 --- a/functions/extensions/k8s/shuffle/templates/hpa.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "shuffle.fullname" . }} - labels: - {{- include "shuffle.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "shuffle.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/functions/extensions/k8s/shuffle/templates/ingress.yaml b/functions/extensions/k8s/shuffle/templates/ingress.yaml deleted file mode 100755 index e35d2ae0..00000000 --- a/functions/extensions/k8s/shuffle/templates/ingress.yaml +++ /dev/null @@ -1,61 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "shuffle.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "shuffle.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/functions/extensions/k8s/shuffle/templates/persistent-volume.yaml b/functions/extensions/k8s/shuffle/templates/persistent-volume.yaml deleted file mode 100755 index 934c68e2..00000000 --- a/functions/extensions/k8s/shuffle/templates/persistent-volume.yaml +++ /dev/null @@ -1,24 +0,0 @@ -#apiVersion: v1 -#kind: PersistentVolume -#metadata: -# name: opensearch-claim0 -# labels: -# app: opensearch-claim0 -#spec: -# capacity: -# storage: "10G" -# volumeMode: Filesystem -# persistentVolumeReclaimPolicy: Retain -# storageClassName: local-storage -# accessModes: -# - "ReadWriteOnce" -# local: -# path: "/data/kubernetes/shuffle-opensearch" -# nodeAffinity: -# required: -# nodeSelectorTerms: -# - matchExpressions: -# - key: node.dollar.com/opensearch -# operator: In -# values: -# - available diff --git a/functions/extensions/k8s/shuffle/templates/persistentvolumeclaim.yaml b/functions/extensions/k8s/shuffle/templates/persistentvolumeclaim.yaml deleted file mode 100755 index 7d1736a4..00000000 --- a/functions/extensions/k8s/shuffle/templates/persistentvolumeclaim.yaml +++ /dev/null @@ -1,17 +0,0 @@ -#apiVersion: v1 -#kind: PersistentVolumeClaim -#metadata: -# name: opensearch-claim0 -# namespace: {{ .Values.namespace }} -# labels: -# app: opensearch-claim0 -#spec: -# selector: -# matchLabels: -# app: opensearch-claim0 -# accessModes: -# - ReadWriteOnce -# storageClassName: local-storage -# resources: -# requests: -# storage: 5Gi diff --git a/functions/extensions/k8s/shuffle/templates/service.yaml b/functions/extensions/k8s/shuffle/templates/service.yaml deleted file mode 100755 index 8fc78dfa..00000000 --- a/functions/extensions/k8s/shuffle/templates/service.yaml +++ /dev/null @@ -1,52 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: backend-service - namespace: {{ .Values.namespace | quote }} -spec: - ports: - - name: "5001" - port: 5001 - targetPort: 5001 - selector: - app: shuffle-backend - ---- -apiVersion: v1 -kind: Service -metadata: - name: frontend-service - namespace: {{ .Values.namespace | quote }} -spec: - type: NodePort - externalTrafficPolicy: Local - ports: - - name: "3001" - port: 3001 - nodePort: 3001 - targetPort: 80 - - name: "3443" - port: 3443 - nodePort: 3443 - targetPort: 443 - protocol: TCP - selector: - app: shuffle-frontend - -#--- -#apiVersion: v1 -#kind: Service -#metadata: -# name: opensearch-service -# namespace: {{ .Values.namespace | quote }} -#spec: -# type: NodePort -# externalTrafficPolicy: Local -# ports: -# - name: "9200" -# port: 9200 -# targetPort: 9200 -# nodePort: 9200 -# protocol: TCP -# selector: -# app: shuffle-opensearch diff --git a/functions/extensions/k8s/shuffle/templates/serviceaccount.yaml b/functions/extensions/k8s/shuffle/templates/serviceaccount.yaml deleted file mode 100755 index 11e9bdb7..00000000 --- a/functions/extensions/k8s/shuffle/templates/serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "shuffle.serviceAccountName" . }} - labels: - {{- include "shuffle.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/functions/extensions/k8s/shuffle/templates/tests/test-connection.yaml b/functions/extensions/k8s/shuffle/templates/tests/test-connection.yaml deleted file mode 100755 index e68f3142..00000000 --- a/functions/extensions/k8s/shuffle/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "shuffle.fullname" . }}-test-connection" - labels: - {{- include "shuffle.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['{{ include "shuffle.fullname" . }}:{{ .Values.service.port }}'] - restartPolicy: Never diff --git a/functions/extensions/k8s/shuffle/values.yaml b/functions/extensions/k8s/shuffle/values.yaml deleted file mode 100755 index 07e935c6..00000000 --- a/functions/extensions/k8s/shuffle/values.yaml +++ /dev/null @@ -1,82 +0,0 @@ -# Default values for shuffle. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 1 - -image: - repository: nginx - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "" - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -podAnnotations: {} - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -service: - type: ClusterIP - port: 80 - -ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -nodeSelector: {} - -tolerations: [] - -affinity: {} diff --git a/functions/extensions/misp/Dockerfile b/functions/extensions/misp/Dockerfile deleted file mode 100755 index f2b8ae3f..00000000 --- a/functions/extensions/misp/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -FROM python:3.9.4-alpine as base - -FROM base as builder - -RUN mkdir /install -WORKDIR /install - -FROM base -RUN apk add g++ - -COPY --from=builder /install /usr/local -COPY requirements.txt /requirements.txt -RUN pip3 install -r /requirements.txt - - -RUN mkdir /app -WORKDIR /app -COPY requirements.txt /app/requirements.txt -RUN python3 -m pip install -r /app/requirements.txt - -COPY sub.py /app/sub.py - -CMD ["python3", "sub.py"] diff --git a/functions/extensions/misp/docker-compose.yml b/functions/extensions/misp/docker-compose.yml deleted file mode 100755 index 5b0130c0..00000000 --- a/functions/extensions/misp/docker-compose.yml +++ /dev/null @@ -1,9 +0,0 @@ -version: '3' -services: - zmq: - image: ghcr.io/frikky/shuffle-zmq:latest - environment: - - ZMQ_HOSTNAME=localhost - - ZMQ_PORT=50000 - - ZMQ_FORWARD_URL=https://shuffler.io/api/v1/hooks/webhook_e09bea36-9976-1421-82bc-b8764ca83c1e - restart: unless-stopped diff --git a/functions/extensions/misp/requirements.txt b/functions/extensions/misp/requirements.txt deleted file mode 100755 index a1cdf7dd..00000000 --- a/functions/extensions/misp/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -pyzmq -requests diff --git a/functions/extensions/misp/sub.py b/functions/extensions/misp/sub.py deleted file mode 100755 index feb78c08..00000000 --- a/functions/extensions/misp/sub.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -print("Running imports") -import sys -import zmq -import json -import time -import pprint -import os -import sys -import requests - -forward_url = os.getenv("ZMQ_FORWARD_URL", "") -print("Checking forward url (ZMQ_FORWARD_URL): %s" % forward_url) -def handle_hook(data): - ret = requests.post(forward_url, json=data) - print(ret.text) - print(ret.status_code) - -def main(): - host = os.getenv("ZMQ_HOST", "localhost") - port = os.getenv("ZMQ_PORT", "50000") - - if len(forward_url) == 0: - print("Failed to start - define ZMQ_FORWARD_URL for webhook forwarder") - exit(0) - - print("Starting connection setup to %s:%s" % (host, port)) - context = zmq.Context() - socket = context.socket(zmq.SUB) - socket.connect ("tcp://%s:%s" % (host, port)) - socket.setsockopt(zmq.SUBSCRIBE, b'') - - poller = zmq.Poller() - poller.register(socket, zmq.POLLIN) - - print("Starting zmq check for %s:%s" % (host, port)) - while True: - socks = dict(poller.poll(timeout=None)) - if socket in socks and socks[socket] == zmq.POLLIN: - message = socket.recv() - #print(message) - topic, s, m = message.decode('utf-8').partition(" ") - - d = json.loads(m) - try: - # print test if you want status (heartbeat) - test = d["status"] - except KeyError: - handle_hook(d) - - time.sleep(1) - -if __name__ == "__main__": - print("In init ") - main() diff --git a/functions/extensions/scripts/disable_disk_check.sh b/functions/extensions/scripts/disable_disk_check.sh deleted file mode 100755 index cadda899..00000000 --- a/functions/extensions/scripts/disable_disk_check.sh +++ /dev/null @@ -1,11 +0,0 @@ -curl -XPUT -u admin:admin https://localhost:9200/_cluster/settings -H "Content-Type:application/json" -k -d \ -'{ - "transient": { - "cluster.routing.allocation.disk.threshold_enabled": false - } -}' - -curl -XPUT -u admin:admin https://localhost:9200/_all/_settings -H "Content-Type: application/json" -k -d \ -'{ - "index.blocks.read_only_allow_delete": null -}' diff --git a/functions/extensions/swarm/.gitignore b/functions/extensions/swarm/.gitignore deleted file mode 100644 index 9ce4a537..00000000 --- a/functions/extensions/swarm/.gitignore +++ /dev/null @@ -1 +0,0 @@ -shuffle-database/nodes diff --git a/functions/extensions/swarm/docker-compose.yml b/functions/extensions/swarm/docker-compose.yml deleted file mode 100644 index 57ea9b49..00000000 --- a/functions/extensions/swarm/docker-compose.yml +++ /dev/null @@ -1,131 +0,0 @@ -version: '3.4' -services: - backend: - image: ghcr.io/frikky/shuffle-backend:nightly - #hostname: shuffle-backend - environment: - BACKEND_HOSTNAME: backend - OUTER_HOSTNAME: backend - BACKEND_PORT: '5001' - HTTPS_PROXY: '' - HTTP_PROXY: '' - SHUFFLE_APP_DOWNLOAD_LOCATION: https://github.com/frikky/shuffle-apps - SHUFFLE_APP_FORCE_UPDATE: 'false' - SHUFFLE_APP_HOTLOAD_FOLDER: /shuffle-apps - SHUFFLE_APP_HOTLOAD_LOCATION: ./shuffle-apps - DATASTORE_EMULATOR_HOST: "shuffle-database:8000" - DOCKER_API_VERSION: '1.40' - SHUFFLE_BASE_IMAGE_NAME: frikky - SHUFFLE_BASE_IMAGE_REGISTRY: ghcr.io - SHUFFLE_BASE_IMAGE_TAG_SUFFIX: '-0.9.30' - SHUFFLE_CONTAINER_AUTO_CLEANUP: 'true' - SHUFFLE_DEFAULT_APIKEY: '' - SHUFFLE_FILE_LOCATION: /shuffle-files - SHUFFLE_OPENSEARCH_APIKEY: '' - SHUFFLE_OPENSEARCH_CERTIFICATE_FILE: '' - SHUFFLE_OPENSEARCH_CLOUDID: '' - SHUFFLE_OPENSEARCH_PROXY: '' - SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY: 'true' - SHUFFLE_OPENSEARCH_URL: http://opensearch:9200 - SHUFFLE_PASS_APP_PROXY: 'FALSE' - SHUFFLE_PASS_WORKER_PROXY: 'TRUE' - SHUFFLE_ELASTIC: 'true' - #SHUFFLE_ENCRYPTION_MODIFIER: - ports: - - "5001:5001" - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - ./shuffle-apps:/shuffle-apps - - ./shuffle-files:/shuffle-files - networks: - - shuffle_prod - #- reverseproxy - depends_on: - - opensearch - logging: - driver: json-file - frontend: - image: ghcr.io/frikky/shuffle-frontend:nightly - healthcheck: - test: curl -fs http://localhost:80 || exit 1 - interval: 30s - timeout: 5s - retries: 3 - ports: - - "3001:80" - - "3443:443" - networks: - - shuffle_prod - #- reverseproxy - environment: - - "BACKEND_HOSTNAME=backend" - depends_on: - - backend - deploy: - update_config: - order: start-first - opensearch: - image: opensearchproject/opensearch:1.1.0 - healthcheck: - test: curl -fs http://localhost:9200/_cat/health || exit 1 - interval: 30s - timeout: 5s - retries: 3 - environment: - - bootstrap.memory_lock=false - - "OPENSEARCH_JAVA_OPTS=-Xms1024m -Xmx1024m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM - - plugins.security.disabled=true - - cluster.routing.allocation.disk.threshold_enabled=false - - cluster.name=shuffle-cluster - - node.name=opensearch - - discovery.seed_hosts=opensearch - - cluster.initial_master_nodes=opensearch - - node.store.allow_mmap=false - volumes: - - ./shuffle-database:/usr/share/opensearch/data:rw - networks: - - shuffle_prod - #- reverseproxy - logging: - driver: json-file - - orborus: - image: ghcr.io/frikky/shuffle-orborus:nightly - #hostname: shuffle-orborus - environment: - #SHUFFLE_WORKER_VERSION: nightly - SHUFFLE_APP_SDK_VERSION: 0.8.97 - SHUFFLE_WORKER_VERSION: nightly - BASE_URL: http://backend:5001 - #BASE_URL: http://192.168.86.37:5001 - CLEANUP: 'true' - DOCKER_API_VERSION: '1.40' - ENVIRONMENT_NAME: Shuffle - HTTPS_PROXY: '' - HTTP_PROXY: '' - ORG_ID: Shuffle - SHUFFLE_BASE_IMAGE_NAME: frikky - SHUFFLE_BASE_IMAGE_REGISTRY: ghcr.io - SHUFFLE_BASE_IMAGE_TAG_SUFFIX: -0.8.80 - SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY: '50' - SHUFFLE_ORBORUS_EXECUTION_TIMEOUT: '800' - SHUFFLE_PASS_APP_PROXY: 'FALSE' - SHUFFLE_PASS_WORKER_PROXY: 'TRUE' - SHUFFLE_SCALE_REPLICAS: 5 - SHUFFLE_SWARM_NETWORK_NAME: shuffle_prod - SHUFFLE_SWARM_CONFIG: "run" - volumes: - - /var/run/docker.sock:/var/run/docker.sock - networks: - - shuffle_prod - #- reverseproxy - logging: - driver: json-file - -networks: - shuffle_prod: - driver: overlay - external: true - #reverseproxy: - # driver: overlay - # #external: true diff --git a/functions/extensions/swarm/network.sh b/functions/extensions/swarm/network.sh deleted file mode 100644 index f3a323ed..00000000 --- a/functions/extensions/swarm/network.sh +++ /dev/null @@ -1 +0,0 @@ -docker network create -d overlay shuffle_prod diff --git a/functions/extensions/swarm/orborus.yml b/functions/extensions/swarm/orborus.yml deleted file mode 100644 index fb90b10f..00000000 --- a/functions/extensions/swarm/orborus.yml +++ /dev/null @@ -1,39 +0,0 @@ -version: '3.4' -services: - orborus: - image: ghcr.io/frikky/shuffle-orborus:nightly - #hostname: shuffle-orborus - environment: - #SHUFFLE_WORKER_VERSION: nightly - SHUFFLE_APP_SDK_VERSION: 0.8.97 - SHUFFLE_WORKER_VERSION: nightly - BASE_URL: http://:5001 - #BASE_URL: http://192.168.86.37:5001 - CLEANUP: 'true' - DOCKER_API_VERSION: '1.40' - ENVIRONMENT_NAME: Shuffle - HTTPS_PROXY: '' - HTTP_PROXY: '' - ORG_ID: Shuffle - SHUFFLE_BASE_IMAGE_NAME: frikky - SHUFFLE_BASE_IMAGE_REGISTRY: ghcr.io - SHUFFLE_BASE_IMAGE_TAG_SUFFIX: -0.8.80 - SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY: '50' - SHUFFLE_ORBORUS_EXECUTION_TIMEOUT: '800' - SHUFFLE_PASS_APP_PROXY: 'FALSE' - SHUFFLE_PASS_WORKER_PROXY: 'TRUE' - SHUFFLE_SCALE_REPLICAS: 5 - SHUFFLE_SWARM_NETWORK_NAME: shuffle_prod - SHUFFLE_SWARM_CONFIG: "run" - volumes: - - /var/run/docker.sock:/var/run/docker.sock - networks: - - shuffle_prod - #- reverseproxy - logging: - driver: json-file - -networks: - shuffle_prod: - driver: overlay - external: true diff --git a/functions/extensions/swarm/run.sh b/functions/extensions/swarm/run.sh deleted file mode 100644 index faeadab8..00000000 --- a/functions/extensions/swarm/run.sh +++ /dev/null @@ -1,4 +0,0 @@ -docker swarm init -chown 1000:1000 -R shuffle-database/ -docker network create -d overlay shuffle_prod -docker stack deploy --compose-file=docker-compose.yml shuffle_swarm diff --git a/functions/extensions/swarm/run_orborus.sh b/functions/extensions/swarm/run_orborus.sh deleted file mode 100644 index bcbd626f..00000000 --- a/functions/extensions/swarm/run_orborus.sh +++ /dev/null @@ -1,4 +0,0 @@ -docker swarm init -chown 1000:1000 -R shuffle-database/ -docker network create -d overlay shuffle_prod -docker stack deploy --compose-file=orborus.yml shuffle_orborus diff --git a/functions/extensions/swarm/shuffle-apps/tmp b/functions/extensions/swarm/shuffle-apps/tmp deleted file mode 100644 index e69de29b..00000000 diff --git a/functions/extensions/swarm/shuffle-database/tmp b/functions/extensions/swarm/shuffle-database/tmp deleted file mode 100644 index e69de29b..00000000 diff --git a/functions/extensions/swarm/shuffle-files/tmp b/functions/extensions/swarm/shuffle-files/tmp deleted file mode 100644 index e69de29b..00000000 diff --git a/functions/extensions/swarm/stop.sh b/functions/extensions/swarm/stop.sh deleted file mode 100644 index c516f348..00000000 --- a/functions/extensions/swarm/stop.sh +++ /dev/null @@ -1,3 +0,0 @@ -docker stack rm shuffle_swarm - -# diff --git a/functions/extensions/wazuh/custom-shuffle b/functions/extensions/wazuh/custom-shuffle deleted file mode 100755 index bd540414..00000000 --- a/functions/extensions/wazuh/custom-shuffle +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/sh -# Created by Shuffle, AS. . - -WPYTHON_BIN="framework/python/bin/python3" - -SCRIPT_PATH_NAME="$0" - -DIR_NAME="$(cd $(dirname ${SCRIPT_PATH_NAME}); pwd -P)" -SCRIPT_NAME="$(basename ${SCRIPT_PATH_NAME})" - -case ${DIR_NAME} in - */active-response/bin | */wodles*) - if [ -z "${WAZUH_PATH}" ]; then - WAZUH_PATH="$(cd ${DIR_NAME}/../..; pwd)" - fi - - PYTHON_SCRIPT="${DIR_NAME}/${SCRIPT_NAME}.py" - ;; - */bin) - if [ -z "${WAZUH_PATH}" ]; then - WAZUH_PATH="$(cd ${DIR_NAME}/..; pwd)" - fi - - PYTHON_SCRIPT="${WAZUH_PATH}/framework/scripts/${SCRIPT_NAME}.py" - ;; - */integrations) - if [ -z "${WAZUH_PATH}" ]; then - WAZUH_PATH="$(cd ${DIR_NAME}/..; pwd)" - fi - - PYTHON_SCRIPT="${DIR_NAME}/${SCRIPT_NAME}.py" - ;; -esac - - -${WAZUH_PATH}/${WPYTHON_BIN} ${PYTHON_SCRIPT} "$@" diff --git a/functions/extensions/wazuh/custom-shuffle.py b/functions/extensions/wazuh/custom-shuffle.py deleted file mode 100755 index 8e555ea9..00000000 --- a/functions/extensions/wazuh/custom-shuffle.py +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env python3 -# Created by Shuffle, AS. . -# Based on the Slack integration using Webhooks - -import json -import sys -import time -import os - -try: - import requests - from requests.auth import HTTPBasicAuth -except Exception as e: - print("No module 'requests' found. Install: pip install requests") - sys.exit(1) - -# ADD THIS TO ossec.conf configuration: -# -# custom-shuffle -# http://:3001/api/v1/hooks/ -# 3 -# json -# - -# Global vars -debug_enabled = False -pwd = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -json_alert = {} -now = time.strftime("%a %b %d %H:%M:%S %Z %Y") - -# Set paths -log_file = '{0}/logs/integrations.log'.format(pwd) - -try: - with open("/tmp/shuffle_start.txt", "w+") as tmp: - tmp.write("Script started") -except: - pass - - -def main(args): - debug("# Starting") - - # Read args - alert_file_location = args[1] - webhook = args[3] - - debug("# Webhook") - debug(webhook) - - debug("# File location") - debug(alert_file_location) - - # Load alert. Parse JSON object. - try: - with open(alert_file_location) as alert_file: - json_alert = json.load(alert_file) - except: - debug("# Alert file %s doesn't exist" % alert_file_location) - - debug("# Processing alert") - try: - debug(json_alert) - except Exception as e: - debug("Failed getting json_alert %s" % e) - sys.exit(1) - - debug("# Generating message") - msg = generate_msg(json_alert) - if isinstance(msg, str): - if len(msg) == 0: - return - debug(msg) - - debug("# Sending message") - - try: - with open("/tmp/shuffle_end.txt", "w+") as tmp: - tmp.write("Script done pre-msg sending") - except: - pass - - - send_msg(msg, webhook) - - -def debug(msg): - if debug_enabled: - msg = "{0}: {1}\n".format(now, msg) - print(msg) - f = open(log_file, "a") - f.write(msg) - f.close() - -# Skips container kills to stop self-recursion -def filter_msg(alert): - # These are things that recursively happen because Shuffle starts Docker containers - skip = ["87924", "87900", "87901", "87902", "87903", "87904", "86001", "86002", "86003", "87932", "80710", "87929", "87928", "5710"] - if alert["rule"]["id"] in skip: - return False - - #try: - # if "docker" in alert["rule"]["description"].lower() and " - #msg['text'] = alert.get('full_log') - #except: - # pass - #msg['title'] = alert['rule']['description'] if 'description' in alert['rule'] else "N/A" - - return True - -def generate_msg(alert): - if not filter_msg(alert): - print("Skipping rule %s" % alert["rule"]["id"]) - return "" - - level = alert['rule']['level'] - - if (level <= 4): - severity = 1 - elif (level >= 5 and level <= 7): - severity = 2 - else: - severity = 3 - - msg = {} - msg['severity'] = severity - msg['pretext'] = "WAZUH Alert" - msg['title'] = alert['rule']['description'] if 'description' in alert['rule'] else "N/A" - msg['text'] = alert.get('full_log') - msg['rule_id'] = alert["rule"]["id"] - msg['timestamp'] = alert["timestamp"] - msg['id'] = alert['id'] - msg["all_fields"] = alert - - #msg['fields'] = [] - # msg['fields'].append({ - # "title": "Agent", - # "value": "({0}) - {1}".format( - # alert['agent']['id'], - # alert['agent']['name'] - # ), - # }) - #if 'agentless' in alert: - # msg['fields'].append({ - # "title": "Agentless Host", - # "value": alert['agentless']['host'], - # }) - - #msg['fields'].append({"title": "Location", "value": alert['location']}) - #msg['fields'].append({ - # "title": "Rule ID", - # "value": "{0} _(Level {1})_".format(alert['rule']['id'], level), - #}) - - #attach = {'attachments': [msg]} - - return json.dumps(msg) - - -def send_msg(msg, url): - debug("# In send msg") - headers = {'content-type': 'application/json', 'Accept-Charset': 'UTF-8'} - res = requests.post(url, data=msg, headers=headers, verify=False) - debug("# After send msg: %s" % res) - - -if __name__ == "__main__": - try: - # Read arguments - bad_arguments = False - if len(sys.argv) >= 4: - msg = '{0} {1} {2} {3} {4}'.format( - now, - sys.argv[1], - sys.argv[2], - sys.argv[3], - sys.argv[4] if len(sys.argv) > 4 else '', - ) - #debug_enabled = (len(sys.argv) > 4 and sys.argv[4] == 'debug') - debug_enabled = True - else: - msg = '{0} Wrong arguments'.format(now) - bad_arguments = True - - # Logging the call - try: - f = open(log_file, 'a') - except: - f = open(log_file, 'w+') - f.write("") - f.close() - - f = open(log_file, 'a') - f.write(msg + '\n') - f.close() - - if bad_arguments: - debug("# Exiting: Bad arguments. Inputted: %s" % sys.argv) - sys.exit(1) - - # Main function - main(sys.argv) - - except Exception as e: - debug(str(e)) - raise diff --git a/functions/extensions/wazuh/ossec.conf b/functions/extensions/wazuh/ossec.conf deleted file mode 100755 index e87bc53d..00000000 --- a/functions/extensions/wazuh/ossec.conf +++ /dev/null @@ -1,6 +0,0 @@ - - custom-shuffle - 9 - http://:/api/v1/hooks/webhook_hookid - json - diff --git a/functions/usecases/README.md b/functions/usecases/README.md deleted file mode 100644 index 553034c2..00000000 --- a/functions/usecases/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# Mindmap exporter -Shuffle has a mindmap for Workflow use-cases. These can be changed and exported, with the most important piece being that they're explorable and editable. This has and will come in handy for us as we build it into the product. - -https://www.mindmeister.com/map/2172644474 - -## Editing the Mindmap -There are a few categories. To edit them, click the small plus next to the branch you want to change. - -## Exporting the Mindmap -Click "Export as RTF" in the top left corner of the URL. Download it there. - -## Generating the Shuffle-comaptible mindmap -1. Move the rtf file here -2. Rename it categories.rtf -3. Run the read_categories.py file (python3 read_categories.py) -4. You now have a file called categories.json locally with all the categories in JSON format, ready to be used in graphs. diff --git a/functions/usecases/categories.json b/functions/usecases/categories.json deleted file mode 100644 index e438a380..00000000 --- a/functions/usecases/categories.json +++ /dev/null @@ -1,260 +0,0 @@ -[ - { - "name": "1. Collect & Distribute", - "color": "#c51152", - "list": [ - { - "name": "2-way Ticket synchronization", - "items": {} - }, - { - "name": "Email management", - "items": { - "name": "Release a quarantined message", - "items": {} - } - }, - { - "name": "EDR to ticket", - "items": { - "name": "Get host information", - "items": {} - } - }, - { - "name": "SIEM to ticket", - "items": {} - }, - { - "name": "ChatOps", - "items": {} - }, - { - "name": "Threat Intel received", - "items": {} - }, - { - "name": "Domain investigation with LetsEncrypt", - "items": {} - }, - { - "name": "Botnet tracker", - "items": {} - }, - { - "name": "Get running containers", - "items": {} - }, - { - "name": "Assign tickets", - "items": {} - }, - { - "name": "Firewall alerts", - "items": { - "name": "URL filtering", - "items": {} - } - }, - { - "name": "IDS/IPS alerts", - "items": { - "name": "Manage policies", - "items": {} - } - }, - { - "name": "Deduplicate information", - "items": {} - }, - { - "name": "Correlate information", - "items": {} - } - ] - }, - { - "name": "2. Enrich", - "color": "#f4c20d", - "list": [ - { - "name": "Internal Enrichment", - "items": { - "name": "...", - "items": {} - } - }, - { - "name": "External historical Enrichment", - "items": { - "name": "...", - "items": {} - } - }, - { - "name": "Realtime", - "items": { - "name": "Analyze screenshots", - "items": {} - } - }, - { - "name": "Ticketing webhook verification", - "items": {} - } - ] - }, - { - "name": "3. Detect", - "color": "#3cba54", - "list": [ - { - "name": "Search SIEM (Sigma)", - "items": { - "name": "Endpoint", - "items": {} - } - }, - { - "name": "Search EDR (OSQuery)", - "items": {} - }, - { - "name": "Search emails (Phish)", - "items": { - "name": "Check headers and IOCs", - "items": {} - } - }, - { - "name": "Search IOCs (ioc-finder)", - "items": {} - }, - { - "name": "Search files (Yara)", - "items": {} - }, - { - "name": "Correlate tickets", - "items": {} - }, - { - "name": "Honeypot access", - "items": { - "name": "...", - "items": {} - } - } - ] - }, - { - "name": "4. Respond", - "color": "#4a148c", - "list": [ - { - "name": "Eradicate malware", - "items": {} - }, - { - "name": "Quarantine host(s)", - "items": {} - }, - { - "name": "Trigger scans", - "items": {} - }, - { - "name": "Update indicators (FW, EDR, SIEM...)", - "items": {} - }, - { - "name": "Autoblock activity when threat intel is received", - "items": {} - }, - { - "name": "Lock/Delete/Reset account", - "items": {} - }, - { - "name": "Lock vault", - "items": {} - }, - { - "name": "Increase authentication", - "items": {} - }, - { - "name": "Get policies from assets", - "items": {} - } - ] - }, - { - "name": "5. Verify", - "color": "#4885ed", - "list": [ - { - "name": "Discover vulnerabilities", - "items": {} - }, - { - "name": "Discover assets", - "items": {} - }, - { - "name": "Ensure policies are followed", - "items": {} - }, - { - "name": "Find Inactive users", - "items": {} - }, - { - "name": "Ensure access rights match HR systems", - "items": {} - }, - { - "name": "Ensure onboarding is followed", - "items": {} - }, - { - "name": "Third party apps in SaaS", - "items": {} - }, - { - "name": "Devices used for your cloud account", - "items": {} - }, - { - "name": "Too much access in GCP/Azure/AWS/ other clouds", - "items": {} - }, - { - "name": "Certificate validation", - "items": {} - }, - { - "name": "Monitor new DNS entries for domain with passive DNS", - "items": {} - }, - { - "name": "Monitor and track password dumps", - "items": {} - }, - { - "name": "Monitor for mentions of domain on darknet sites", - "items": {} - }, - { - "name": "Reporting", - "items": { - "name": "Monthly reports", - "items": { - "name": "...", - "items": {} - } - } - } - ] - } -] \ No newline at end of file diff --git a/functions/usecases/categories.rtf b/functions/usecases/categories.rtf deleted file mode 100644 index e19cf26a..00000000 --- a/functions/usecases/categories.rtf +++ /dev/null @@ -1,555 +0,0 @@ -{\rtf1\ansi\deff0\deflang2057\plain\fs24\fet1 -{\fonttbl -{\f0\froman Arial;} -} -{\info -{\createim\yr2022\mo2\dy20\hr1\min15} -} - -\paperw11907\paperh16840\margl1800\margr1800\margt1440\margb1440 -\slmult0\ltrpar\li0 -{\b\fs28 -Shuffle categories -} -\par\pard\plain -\slmult0\ltrpar\li200 -{\fs24 -1. Collect & Distribute -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -2-way Ticket synchronization -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Email management -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Attachments -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Manage senders -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Manage URLs -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Encode & Decode URLs -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Release a quarantined message -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -EDR to ticket -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Fetch incidents & events -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Quarantine files -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Quarantine host (respond) -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Get host information -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -SIEM to ticket -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -ChatOps -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Threat Intel received -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Domain investigation with LetsEncrypt -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Botnet tracker -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Get running containers -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Assign tickets -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Firewall alerts -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Block/accept policies -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Add addresses and ports to groups -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Support custom URL categories -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Fetch logs for specific address -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -URL filtering -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -IDS/IPS alerts -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Get/Fetch alerts -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Receive alerts real-time -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Get PCAP files -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Get network logs -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Manage policies -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Deduplicate information -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Correlate information -} -\par\pard\plain -\slmult0\ltrpar\li200 -{\fs24 -3. Detect -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Search SIEM (Sigma) -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Network -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Endpoint -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Search EDR (OSQuery) -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Search emails (Phish) -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Check malware -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Check targeted -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Check headers and IOCs -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Search IOCs (ioc-finder) -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Search files (Yara) -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Correlate tickets -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Honeypot access -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -S3 Honeypot -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -SSH Honeypot -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -FTP honeypot -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Network honeypot -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -... -} -\par\pard\plain -\slmult0\ltrpar\li200 -{\fs24 -rich -} -\par\pard\plain -\slmult0\ltrpar\li200 -{\fs24 -5. Verify -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Discover vulnerabilities -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Discover assets -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Ensure policies are followed -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Find Inactive users -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Ensure access rights match HR systems -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Ensure onboarding is followed -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Third party apps in SaaS -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Devices used for your cloud account -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Too much access in GCP/Azure/AWS/ other clouds -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Certificate validation -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Monitor new DNS entries for domain with passive DNS -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Monitor and track password dumps -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Monitor for mentions of domain on darknet sites -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Reporting -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Automation time saved -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Automation money saved -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Incident response report -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Department cost -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Monthly reports -} -\par\pard\plain -\slmult0\ltrpar\li800 -{\fs24 -EDR alerts -} -\par\pard\plain -\slmult0\ltrpar\li800 -{\fs24 -SIEM alerts -} -\par\pard\plain -\slmult0\ltrpar\li800 -{\fs24 -Emails quarantined -} -\par\pard\plain -\slmult0\ltrpar\li800 -{\fs24 -... -} -\par\pard\plain -\slmult0\ltrpar\li200 -{\fs24 -4. Respond -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Eradicate malware -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Quarantine host(s) -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Trigger scans -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Update indicators (FW, EDR, SIEM...) -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Autoblock activity when threat intel is received -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Lock/Delete/Reset account -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Lock vault -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Increase authentication -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Get policies from assets -} -\par\pard\plain -\slmult0\ltrpar\li200 -{\fs24 -2. Enrich -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Internal Enrichment -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Users -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Hostnames -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -IPs -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Departments -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Role -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Software -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -... -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -External historical Enrichment -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -IPs -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -URLs -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Hashes -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Files -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -... -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Realtime -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -File detonation -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -URL detonation -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -PCAP analysis -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Analyze screenshots -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Ticketing webhook verification -} -\par\pard\plain -} \ No newline at end of file diff --git a/functions/usecases/read_categories.py b/functions/usecases/read_categories.py deleted file mode 100644 index 65df43c9..00000000 --- a/functions/usecases/read_categories.py +++ /dev/null @@ -1,66 +0,0 @@ -data = "" -with open("categories.rtf", "r") as tmp: - data = tmp.read() - -fixed_json = [] -linearity = 0 -heading = "" -subheading = "" -subsubheading = "" - -cnt = -1 -subcnt = -1 - -colors = ["#c51152", "#3cba54", "#4885ed", "#4a148c", "#f4c20d"] -for line in data.split("\n"): - if line == "rich": - continue - - if "li" in line: - lisplit = line.split("\\") - try: - linearity = int(lisplit[-1][2]) - except: - pass - - #print("Linearity: %s" % linearity) - - if line.startswith("{") or line.startswith("}"): - continue - - if line.startswith("\\"): - continue - - if linearity == 0: - continue - - if linearity == 2: - #if cnt >= 0: - # for key, value in fixed_json[cnt].items(): - # print(key, value) - - - cnt += 1 - subcnt = -1 - fixed_json.append({"name": line, "color": colors[cnt], "list": []}) - heading = line - elif linearity == 4: - subheading = line - fixed_json[cnt]["list"].append({"name": line, "items": {}}) - subcnt += 1 - elif linearity == 6: - fixed_json[cnt]["list"][subcnt]["items"] = {"name": line, "items": {}} - elif linearity == 8: - fixed_json[cnt]["list"][subcnt]["items"]["items"] = {"name": line, "items": {}} - else: - print("No handler for %s" % line) - -#print(line) -#print(data) -import json -filename = "categories.json" -fixed_json.sort(key=lambda x: x["name"]) -with open(filename, "w+") as tmp: - tmp.write(json.dumps(fixed_json, indent=4)) - -print("Wrote to file %s" % filename) From 074709c7bb6cccd8f570b6505a8d69d20a08db05 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Thu, 26 Jun 2025 20:02:50 +0530 Subject: [PATCH 03/59] fix orborus issues --- backend/go-app/walkoff.go | 24 ++++++++++++++-------- functions/onprem/orborus/orborus.go | 31 +++++++++++++++-------------- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index c194d091..0aff66c8 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -246,16 +246,16 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { } // This is really the environment's name - NOT org-id - orgId := request.Header.Get("Org-Id") - if len(orgId) == 0 { + environment := request.Header.Get("Org-Id") + if len(environment) == 0 { log.Printf("[AUDIT] No org-id header set") resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Specify the org-id header."}`))) return } - environment := request.Header.Get("org") - if len(environment) == 0 { + orgId := request.Header.Get("org") + if len(orgId) == 0 { //log.Printf("[AUDIT] No 'org' header set (get workflow queue). ") /* resp.WriteHeader(403) @@ -280,9 +280,17 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { //log.Printf("[AUDIT] Get workflow queue for org %s, env %s, orborus label %s", orgId, environment, orborusLabel) ctx := shuffle.GetContext(request) - env, err := shuffle.GetEnvironment(ctx, orgId, "") - if err != nil { - log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", orgId, err) + envs, err := shuffle.GetEnvironments(ctx, orgId) + if err != nil || len(envs) == 0 { + log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", environment, err) + } + + var env *shuffle.Environment + for _, e := range envs { + if e.Name == environment { + env = &e + break + } } timeNow := time.Now().Unix() @@ -450,7 +458,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { } } - executionRequests, err := shuffle.GetWorkflowQueue(ctx, orgId, 100) + executionRequests, err := shuffle.GetWorkflowQueue(ctx, environment, 100) if err != nil { // Skipping as this comes up over and over //log.Printf("(2) Failed reading body for workflowqueue: %s", err) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 9b5c87da..e350082b 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -813,21 +813,21 @@ func handleBackendImageDownload(ctx context.Context, images string) error { newImages = append(newImages, curimage) // Force remove the current image to avoid cached layers - if swarmConfig == "run" || swarmConfig == "swarm" { - _, err := dockercli.ImageRemove(ctx, curimage, image.RemoveOptions{ - Force: true, - PruneChildren: true, - }) - - if err != nil { - log.Printf("[ERROR] Failed removing image for re-download: %s", err) - } else { - log.Printf("[DEBUG] Removed image: %s", curimage) - } - } else { - //log.Printf("[DEBUG] Skipping image removal for %s as swarmConfig is not set to run or swarm. Value: %#v", curimage, swarmConfig) - } - +// if swarmConfig == "run" || swarmConfig == "swarm" { +// _, err := dockercli.ImageRemove(ctx, curimage, image.RemoveOptions{ +// Force: true, +// PruneChildren: true, +// }) +// +// if err != nil { +// log.Printf("[ERROR] Failed removing image for re-download: %s", err) +// } else { +// log.Printf("[DEBUG] Removed image: %s", curimage) +// } +// } else { +// //log.Printf("[DEBUG] Skipping image removal for %s as swarmConfig is not set to run or swarm. Value: %#v", curimage, swarmConfig) +// } +// err := shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, curimage) if err != nil { //log.Printf("[ERROR] Failed downloading image: %s", err) @@ -874,6 +874,7 @@ func handleBackendImageDownload(ctx context.Context, images string) error { // Update the service to run with the new image //docker service update --image username/imagename:latest servicename --force serviceUpdateOptions := types.ServiceUpdateOptions{} + service.Spec.TaskTemplate.ForceUpdate++ resp, err := dockercli.ServiceUpdate( ctx, service.ID, From 9bb578616fd4407ddb159d8ffa4b98a0d63c4093 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Sat, 28 Jun 2025 17:26:48 +0530 Subject: [PATCH 04/59] checks for nil pointer --- backend/go-app/walkoff.go | 178 +++++--------------------------------- 1 file changed, 20 insertions(+), 158 deletions(-) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 0aff66c8..08825a52 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -11,7 +11,7 @@ import ( "io" "io/ioutil" "log" - "math/rand" + //"math/rand" "net/http" "net/url" "os" @@ -265,7 +265,8 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { } orborusLabel := request.Header.Get("x-orborus-label") - + _ = orborusLabel + // This section is cloud custom for now auth := request.Header.Get("Authorization") if len(auth) == 0 { @@ -286,13 +287,22 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { } var env *shuffle.Environment - for _, e := range envs { - if e.Name == environment { - env = &e + found := false + for i := range envs { + if envs[i].Name == environment { + env = &envs[i] + found = true break } } + if !found { + log.Printf("[ERROR] Failed to find environment(%s) for org(%s)", environment, orgId) + resp.WriteHeader(400) + resp.Write([]byte(`{"success":false,"reason":"environment not found"}`)) + return + } + timeNow := time.Now().Unix() err = shuffle.HandleOrborusFailover(ctx, request, resp, env) if err != nil { @@ -300,162 +310,14 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { log.Printf("[WARNING] Failed handling Orborus failover: %s", err) } - log.Printf("[DEBUG] Issue with environment ID: %s", orgId) + log.Printf("[DEBUG] Issue with environment ID: %s", environment) return } //log.Printf("Found env: %#v", env) if len(env.OrgId) > 0 { - environment = env.OrgId - } - - // FIXME: Workflow stats disabled for now - // as it caused too many problems - // goal: track docker stuff once a minute and graph it - // For now: Disable this as it caused too many problems - if request.Method == "POST" && true == false { - //log.Printf("[DEBUG] POST to workflowqueue") - if rand.Intn(10) == 0 { - // Parse out body - body, err := ioutil.ReadAll(request.Body) - if err == nil { - - // Parse out CPU, memory and disk. - - var envData shuffle.OrborusStats - err = json.Unmarshal(body, &envData) - if err == nil && !envData.Swarm && !envData.Kubernetes && (envData.CPU > 0 || envData.Memory > 0 || envData.Disk > 0) { - - // Set the input in memory - envData.OrgId = orgId - envData.Environment = environment - envData.OrborusLabel = orborusLabel - envData.Timestamp = time.Now().Unix() - - if envData.CPU > 0 && envData.MaxCPU > 0 { - envData.CPUPercent = float64(envData.CPU) / float64(envData.MaxCPU) - } - - if envData.Memory > 0 && envData.MaxMemory > 0 { - envData.MemoryPercent = float64(envData.Memory) / float64(envData.MaxMemory) - } - - // Check if CPU percent constantly has stayed above X% for the last Y requests - percentageCheck := 90 - concurrentChecks := 2 - - //if int(envData.CPUPercent) > percentageCheck { - // Get cached data - percentages := []float64{} - cacheKey := fmt.Sprintf("%s_%s_percent", orgId, strings.ToLower(environment)) - - // Marshal float list into []byte - cacheData := []byte{} - cache, err := shuffle.GetCache(ctx, cacheKey) - if err == nil { - // Unmarshal into percentages - cacheData := []byte(cache.([]uint8)) - err = json.Unmarshal(cacheData, &percentages) - if err != nil { - log.Printf("[INFO] error in cache unmarshal for percentages: %s", err) - } - - if len(percentages) > concurrentChecks { - percentages = percentages[:concurrentChecks] - } - - percentages = append(percentages, envData.CPUPercent) - if len(percentages) > concurrentChecks { - //log.Printf("[INFO] Checking percentages: %v", percentages) - - // percentageCheck := 1 - sendAlert := true - for _, p := range percentages { - if int(p) < percentageCheck { - //log.Printf("[AUDIT] CPU percent is below %d: %d", percentageCheck, int(p)) - sendAlert = false - break - } - } - - if sendAlert { - log.Printf("[INFO] CPU percent has been above %d percent for the last 5 requests. Sending alert. Env: %s, org: %s", percentageCheck, environment, orgId) - - // Set notification + alert for organization - err = shuffle.CreateOrgNotification( - ctx, - fmt.Sprintf("CPU percent has been above %d percent", percentageCheck), - fmt.Sprintf("A environment %s has been using more than %d\\% CPU for the last 5 requests.", environment, percentageCheck), - fmt.Sprintf("/admin?tab=environments"), - environment, - true, - ) - - if err != nil { - log.Printf("[ERROR] error creating notification: %s", err) - } - - org, err := shuffle.GetOrg(ctx, environment) - if err == nil { - foundRecommendation := false - for _, recommendation := range org.Priorities { - if strings.Contains(recommendation.Name, "CPU") { - foundRecommendation = true - break - } - } - - if !foundRecommendation { - // Add to start of org.Priorities - org, _ = shuffle.AddPriority(*org, shuffle.Priority{ - Name: fmt.Sprintf("High CPU in environment %s", orgId), - Description: fmt.Sprintf("The environment %s has been using more than %d percent CPU. This indicates you may need to look at scaling.", orgId, percentageCheck), - Type: "scale", - Active: true, - URL: fmt.Sprintf("/admin?tab=environments"), - Severity: 1, - }, false) - - //Make last item the first item - org.Priorities = append([]shuffle.Priority{org.Priorities[len(org.Priorities)-1]}, org.Priorities[:len(org.Priorities)-1]...) - err = shuffle.SetOrg(ctx, *org, org.Id) - if err != nil { - log.Printf("[ERROR] Problem setting org: %s", err) - } - } - } - } - - if len(percentages) > 1 { - percentages = percentages[1:] - } - } - - // Marshal float list into []byte - } else { - //log.Printf("[ERROR] Failed getting cache: %s", err) - percentages = append(percentages, envData.CPUPercent) - } - - if len(percentages) > 0 { - //log.Printf("[DEBUG] Setting cache for %s: %#v", cacheKey, percentages) - cacheData, err = json.Marshal(percentages) - if err != nil { - log.Printf("[INFO] error in cache marshal: %s", err) - } - - // Add the new data - go shuffle.SetCache(ctx, cacheKey, cacheData, 5) - } - } - - //log.Printf("CPU percent: %f", envData.CPUPercent) - //log.Printf("Memory percent: %f", envData.MemoryPercent*100) - - go shuffle.SetenvStats(ctx, envData) - } - } + orgId = env.OrgId } executionRequests, err := shuffle.GetWorkflowQueue(ctx, environment, 100) @@ -486,10 +348,10 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { } } - if len(orgId) > 0 { - env, err := shuffle.GetEnvironment(ctx, orgId, foundId) + if len(environment) > 0 { + env, err := shuffle.GetEnvironment(ctx, environment, foundId) if err != nil { - log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", orgId, err) + log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", environment, err) //resp.WriteHeader(401) //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No env found matching %s"}`, id))) //return From 0be2b9fe93cb9d4e8277dc7771bc3c55a9fe7092 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Sat, 28 Jun 2025 19:18:52 +0530 Subject: [PATCH 05/59] log org from the header --- backend/go-app/walkoff.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 08825a52..0f078dca 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -254,9 +254,9 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { return } - orgId := request.Header.Get("org") + orgId := request.Header.Get("Org") if len(orgId) == 0 { - //log.Printf("[AUDIT] No 'org' header set (get workflow queue). ") + log.Printf("[AUDIT] No 'org' header set (get workflow queue). ") /* resp.WriteHeader(403) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Specify the org header. This can be done by setting the 'ORG' environment variable for Orborus to your Org ID in Shuffle"}`))) From 232b0263c661d164566acf4a480091cd239995cf Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Wed, 2 Jul 2025 20:19:56 +0530 Subject: [PATCH 06/59] fix: enabling /api/v1/orgs for prod --- backend/go-app/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index e1f0defa..06412fb4 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5221,8 +5221,8 @@ func initHandlers() { // EVERYTHING below here is NEW for 0.8.0 (written 25.05.2021) r.HandleFunc("/api/v1/workflows/{key}/publish", makeWorkflowPublic).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/cloud/setup", handleCloudSetup).Methods("POST", "OPTIONS") - //r.HandleFunc("/api/v1/orgs", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS") - //r.HandleFunc("/api/v1/orgs/", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/orgs", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/orgs/", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}", shuffle.HandleGetOrg).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}", shuffle.HandleEditOrg).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgid}/forms", shuffle.HandleGetOrgForms).Methods("GET", "OPTIONS") From aa1c98240bdae0d0030e0caf78158f7e676bb0bf Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Wed, 2 Jul 2025 20:59:49 +0530 Subject: [PATCH 07/59] ci: adding tagged nightly release back --- .github/workflows/tagged-nightly-release.yaml | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 .github/workflows/tagged-nightly-release.yaml diff --git a/.github/workflows/tagged-nightly-release.yaml b/.github/workflows/tagged-nightly-release.yaml new file mode 100644 index 00000000..c24b7e1d --- /dev/null +++ b/.github/workflows/tagged-nightly-release.yaml @@ -0,0 +1,92 @@ +name: Tagged Nightly Release +on: + release: + types: [published] + branches: + - nightly + +# there is a very clear point to this existing. +# we want to also release versions that look like this: +# v2.1.0-nightly-date, v2.1.0-nightly-date-1, v2.1.0-nightly-date-2 +# we NEVER want to send customers a "nightly" tag. We always want to send them +# a tagged nightly tag. So that when something breaks, They can always +# point to it. + +jobs: + main: + runs-on: ubuntu-latest + continue-on-error: ${{ matrix.experimental }} + strategy: + fail-fast: false + matrix: + include: + - app: frontend + path: frontend + experimental: true + - app: backend + path: backend + experimental: true + - app: app_sdk + path: backend/app_sdk + experimental: true + - app: orborus + path: functions/onprem/orborus + experimental: true + - app: worker + path: functions/onprem/worker + experimental: true + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set version + id: set_version + run: | + if [[ ${{ github.event_name }} == 'release' ]]; then + echo "VERSION=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT + else + echo "VERSION=nightly-untagged-latest" >> $GITHUB_OUTPUT + fi + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: "amd64,arm64,arm" + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to Ghcr + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Ghcr Build and push + id: docker_build + uses: docker/build-push-action@v4 + env: + BUILDX_NO_DEFAULT_LOAD: true + with: + logout: false + context: ${{ matrix.path }}/ + file: ${{ matrix.path }}/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + tags: | + ghcr.io/shuffle/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }} + ${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }} + frikky/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }} + frikky/shuffle:${{ matrix.app }} + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} \ No newline at end of file From 48b18635d2d1a4b8d25083de435791879b599616 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Wed, 9 Jul 2025 14:03:16 +0530 Subject: [PATCH 08/59] removed return onprem --- backend/go-app/walkoff.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 0f078dca..172cc0b4 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -306,13 +306,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { timeNow := time.Now().Unix() err = shuffle.HandleOrborusFailover(ctx, request, resp, env) if err != nil { - if !strings.Contains(err.Error(), "mismatch") { - log.Printf("[WARNING] Failed handling Orborus failover: %s", err) - } - - log.Printf("[DEBUG] Issue with environment ID: %s", environment) - - return + log.Printf("[WARNING] Failed handling Orborus failover: %s", err) } //log.Printf("Found env: %#v", env) From 98f5eabaaaafaa68ccaa0a7380445ada20daff21 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Wed, 9 Jul 2025 14:04:39 +0530 Subject: [PATCH 09/59] removed a spam log message --- backend/go-app/walkoff.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 172cc0b4..f2d8a118 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -256,7 +256,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { orgId := request.Header.Get("Org") if len(orgId) == 0 { - log.Printf("[AUDIT] No 'org' header set (get workflow queue). ") + //log.Printf("[AUDIT] No 'org' header set (get workflow queue). ") /* resp.WriteHeader(403) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Specify the org header. This can be done by setting the 'ORG' environment variable for Orborus to your Org ID in Shuffle"}`))) From ff4a63daea677fd480c45096dc4fa3eef32712cd Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Wed, 9 Jul 2025 14:46:39 +0530 Subject: [PATCH 10/59] shuffle-shared sync up --- backend/go-app/go.mod | 4 ++-- functions/onprem/orborus/go.mod | 2 +- functions/onprem/worker/go.mod | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index aeb3e7e0..3c21ac4c 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -23,7 +23,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.84 + github.com/shuffle/shuffle-shared v0.8.88 golang.org/x/crypto v0.38.0 google.golang.org/api v0.236.0 google.golang.org/grpc v1.72.2 @@ -72,7 +72,7 @@ require ( github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/frikky/schemaless v0.0.16 // indirect + github.com/frikky/schemaless v0.0.17 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-jose/go-jose/v4 v4.0.5 // indirect diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 5d3ee295..3fc74c7e 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -10,7 +10,7 @@ require ( github.com/docker/docker v28.2.2+incompatible github.com/docker/go-connections v0.5.0 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.84 + github.com/shuffle/shuffle-shared v0.8.88 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 ) diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 1df6cb59..f22f6fdf 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -10,7 +10,7 @@ require ( github.com/docker/docker v28.2.2+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.84 + github.com/shuffle/shuffle-shared v0.8.88 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 k8s.io/client-go v0.33.1 From 7e9cdce4b63a4ed640efde2035ba12a9ea1703b3 Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 11 Jul 2025 16:04:14 +0200 Subject: [PATCH 11/59] Tons of minor fixes from cloud sync --- frontend/src/components/AdminNavBar.jsx | 16 +- frontend/src/components/AnalyticsTab.jsx | 1 - frontend/src/components/Billing.jsx | 99 +- frontend/src/components/BillingStats.jsx | 49 +- frontend/src/components/CacheView.jsx | 308 +++-- .../src/components/CollectIngestModal.jsx | 207 +++- frontend/src/components/DashboardBarchart.jsx | 84 +- frontend/src/components/EditOrgTab.jsx | 39 +- frontend/src/components/Files.jsx | 1041 ++++++++--------- frontend/src/components/Navbar.jsx | 23 +- .../src/components/OrgHeaderexpandedNew.jsx | 41 +- frontend/src/components/OrganizationTab.jsx | 22 +- frontend/src/components/ParsedAction.jsx | 5 +- frontend/src/components/PartnerDetails.jsx | 77 +- frontend/src/components/PartnerSettings.jsx | 6 +- frontend/src/components/PartnerTab.jsx | 27 +- .../src/components/PartnersUsecasesTab.jsx | 38 +- frontend/src/components/Priorities.jsx | 9 +- frontend/src/components/RuntimeDebugger.jsx | 87 +- .../src/components/ShuffleCodeEditor1.jsx | 9 +- frontend/src/components/TenantsTab.jsx | 7 +- frontend/src/components/UserManagmentTab.jsx | 6 +- frontend/src/defaultCytoscapeStyle.jsx | 20 +- frontend/src/views/Admin2.jsx | 4 + frontend/src/views/AgentUI.jsx | 252 +++- frontend/src/views/AngularWorkflow.jsx | 66 +- frontend/src/views/AppExplorer.jsx | 232 +++- frontend/src/views/DashboardViews.jsx | 26 +- frontend/src/views/Workflows.jsx | 2 +- 29 files changed, 1788 insertions(+), 1015 deletions(-) diff --git a/frontend/src/components/AdminNavBar.jsx b/frontend/src/components/AdminNavBar.jsx index 2a480c95..fbbc391d 100644 --- a/frontend/src/components/AdminNavBar.jsx +++ b/frontend/src/components/AdminNavBar.jsx @@ -92,9 +92,14 @@ const AdminNavBar = (props) => { const HandleVisibleTabs = () => { if (userdata?.id?.length > 0) { if (userdata?.active_org?.role === "admin" || userdata?.support) { - setVisibleItems(items); + if (isChildOrg) { + const filteredItems = items.filter(item => item.text !== "Partner"); + setVisibleItems(filteredItems); + }else { + setVisibleItems(items); + } }else { - const filteredItems = items.filter(item => item.text !== "Users" && item.text !== "Files" && item.text !== "Datastore" && item.text !== "Triggers" && item.text !== "Locations"); + const filteredItems = items.filter(item => item.text !== "Users" && item.text !== "Files" && item.text !== "Datastore" && item.text !== "Triggers" && item.text !== "Locations" && item.text !== "Partner"); setVisibleItems(filteredItems); } } @@ -105,12 +110,12 @@ const AdminNavBar = (props) => { // Filter out Users and Tenants tabs if (userdata?.active_org?.role === "admin" || userdata?.support) { const filteredItems = items.filter(item => - item.text !== "Users" && item.text !== "Tenants" + item.text !== "Users" && item.text !== "Tenants" && item.text !== "Partner" ); setVisibleItems(filteredItems); }else { const filteredItems = items.filter(item => - item.text !== "Users" && item.text !== "Tenants" && item.text !== "Files" && item.text !== "Datastore" && item.text !== "Triggers" && item.text !== "Locations" + item.text !== "Users" && item.text !== "Partner" && item.text !== "Tenants" && item.text !== "Files" && item.text !== "Datastore" && item.text !== "Triggers" && item.text !== "Locations" ); setVisibleItems(filteredItems); } @@ -191,7 +196,7 @@ const AdminNavBar = (props) => { const params = new URLSearchParams(location.search); const tab = params?.get('tab')?.toLowerCase(); - if (tab === "users" || tab === "tenants") { + if (tab === "users" || tab === "tenants" || tab === "partner") { toast.info("You are not allowed to access this tab. Please contact your admin for more information. Redirecting to Organization Configuration tab."); setTimeout(() => { setSelectedItem("Organization"); @@ -200,6 +205,7 @@ const AdminNavBar = (props) => { } , 3000); } + } else if (userdata && isOrgLoaded && isUserDataLoaded && userdata?.active_org?.role !== "admin" && !userdata?.support) { const queryParams = new URLSearchParams(location.search); const tabName = queryParams?.get('admin_tab')?.toLowerCase(); diff --git a/frontend/src/components/AnalyticsTab.jsx b/frontend/src/components/AnalyticsTab.jsx index c97560ed..c64dee56 100644 --- a/frontend/src/components/AnalyticsTab.jsx +++ b/frontend/src/components/AnalyticsTab.jsx @@ -2,7 +2,6 @@ // import Switch from '@mui/material/Switch'; // import { Typography, Button } from '@mui/material'; // import { useNavigate, Link, useParams } from "react-router-dom"; -// import { Bar } from 'react-chartjs-2'; // import Grid from '@mui/material/Grid'; // import SearchIcon from '@mui/icons-material/Search'; // import NewReleasesIcon from '@mui/icons-material/NewReleases'; diff --git a/frontend/src/components/Billing.jsx b/frontend/src/components/Billing.jsx index 0b9daa6f..19698a13 100644 --- a/frontend/src/components/Billing.jsx +++ b/frontend/src/components/Billing.jsx @@ -94,9 +94,9 @@ const Billing = memo((props) => { useEffect(() => { if (userdata.app_execution_limit !== undefined && userdata.app_execution_usage !== undefined) { - const percentage = (userdata.app_execution_usage / userdata.app_execution_limit) * 100; + const percentage = ((userdata.app_execution_usage + userdata.app_executions_suborgs) / userdata.app_execution_limit) * 100; setCurrentAppRunsInPercentage(Math.round(percentage)); - setCurrentAppRunsInNumber(userdata.app_execution_limit - userdata.app_execution_usage); + setCurrentAppRunsInNumber(userdata.app_execution_limit - userdata.app_execution_usage - userdata.app_executions_suborgs); } if (userdata?.id?.length > 0 && isLoggedIn === false){ @@ -1860,19 +1860,6 @@ const Billing = memo((props) => { const updateAlertThreshold = (index, field, value) => { - if (field === 'percentage') { - if (value > 100 || value < 0) { - value = 0 - toast("The percentage value should be between 0 and 100") - } - } else if (field === 'count') { - if (value < 0 || value >= userdata.app_execution_limit) { - value = 0 - toast("The count value should be greater than 0 and less than the total app execution limit") - } - } - - const totalValue = userdata.app_execution_limit; const newAlertThresholds = alertThresholds.map((threshold, i) => { if (i === index) { @@ -2423,9 +2410,21 @@ const Billing = memo((props) => { }} /> - You have used {currentAppRunsInPercentage}% of total app execution limit or {userdata.app_execution_usage} app runs out of {userdata.app_execution_limit} app runs. + You have used {currentAppRunsInPercentage}% of total app execution limit or {userdata.app_execution_usage + userdata.app_executions_suborgs} app runs out of {userdata.app_execution_limit} app runs. - + + {userdata?.active_org?.creator_org?.length > 0 ? null : + ( + <> + + Parent Organization App Executions: {userdata.app_execution_usage} + + + Sub-Organization App Executions: {userdata.app_executions_suborgs || "N/A"} + + + )} +
Set email alert thresholds for app runs @@ -2442,8 +2441,8 @@ const Billing = memo((props) => { : " " + 0 + " "} app runs. - - Please note: Once your app runs reach the set alert threshold, all admins in the organization will receive an email notification. + + Please note: Once your app runs reach the set alert threshold, all admins in the organization will receive an email notification. For Parent organizations, the alert will be sent base on the total app runs from both parent and sub-organizations. For Sub-organizations, the alert will be sent based on the app runs of the sub-organization only.
{alertThresholds.map((threshold, index) => ( @@ -2706,6 +2705,7 @@ const BillingStatsChildOrg = memo(({ userdata, globalUrl, selectedOrganization, const { themeMode, brandColor, supportEmail } = useContext(Context); const theme = getTheme(themeMode, brandColor); + // Handle page change const handleChangePage = (event, newPage) => { setPage(newPage); @@ -2830,6 +2830,7 @@ const BillingStatsChildOrg = memo(({ userdata, globalUrl, selectedOrganization, usage: stat?.monthly_app_executions || "N/A", workflows_usage: stat?.total_workflow_executions || "N/A", workflow_usage_limit: subOrg?.sync_features?.workflow_executions?.limit || "N/A", + app_runs_hard_limit: subOrg?.Billing?.app_runs_hard_limit || 0, } }) @@ -2890,6 +2891,33 @@ const BillingStatsChildOrg = memo(({ userdata, globalUrl, selectedOrganization, )} }, + { + field: "app_runs_hard_limit", headerName: "App Executions Hard Limit", width: 200, renderCell: (params) => { + console.log("params.row: ", params.row) + return ( + <> + + {params.row.app_runs_hard_limit} + + { + setOpen(true) + setEditingOrgId(params.row.orgId) + setEditing("app_executions_hard_limit") + if (params.row.app_runs_hard_limit === "N/A") { + setLimit("") + } else { + setLimit(params.row.app_runs_hard_limit) + } + }} + > + + + + ) + } + } ] setSubOrgStatsColumns(columns) @@ -2913,7 +2941,7 @@ const BillingStatsChildOrg = memo(({ userdata, globalUrl, selectedOrganization, return } - if (selectedOrganization.sync_features.app_executions.limit <= 10000) { + if (selectedOrganization.sync_features.app_executions.limit <= 10000 && editing === "app_executions") { toast.error("Insufficient app execution limit to increase child org limit") return } @@ -2939,7 +2967,10 @@ const BillingStatsChildOrg = memo(({ userdata, globalUrl, selectedOrganization, const org = subOrgs[orgIndex] - org.sync_features[editing].limit = limit + if (editing !== "app_executions_hard_limit") { + org.sync_features[editing].limit = limit + } + org.sync_features.editing = true const sync_features = org.sync_features const data = { @@ -2947,6 +2978,13 @@ const BillingStatsChildOrg = memo(({ userdata, globalUrl, selectedOrganization, sync_features: sync_features, } + if (editing === "app_executions_hard_limit") { + data.editing = "app_runs_hard_limit"; + data.billing = { + app_runs_hard_limit: limit || 0 + }; + } + const url = `${globalUrl}/api/v1/orgs/${orgId}`; fetch(url, { method: "POST", @@ -2974,6 +3012,12 @@ const BillingStatsChildOrg = memo(({ userdata, globalUrl, selectedOrganization, newRows[orgIndex].workflow_usage_limit = limit; return newRows; }); + }else if (editing === "app_executions_hard_limit") { + setSubOrgStatsRows((prevRows) => { + const newRows = [...prevRows]; + newRows[orgIndex].app_runs_hard_limit = limit; + return newRows; + }); } } }) @@ -3087,10 +3131,21 @@ const IncreaseLimitPopUp = memo(({ open, onClose, limit, setLimit, HandleEditLim } }} > - + {editing === "app_executions_hard_limit" ? ( + + Add {editing.replaceAll("_", " ").replace(/\b\w/g, c => c.toUpperCase())} + + ) : ( + Increase {editing.replaceAll("_", " ").replace(/\b\w/g, c => c.toUpperCase())} Limit + )} + { editing === "app_executions_hard_limit" ? ( + + Please note that once you set a hard limit for app runs workflows will not be able to run if the limit is reached. You will be notified by email when you reach the limit. + + ) : null} setCurrentLimit(e.target.value)} diff --git a/frontend/src/components/BillingStats.jsx b/frontend/src/components/BillingStats.jsx index 5bb09ddb..7bd33476 100644 --- a/frontend/src/components/BillingStats.jsx +++ b/frontend/src/components/BillingStats.jsx @@ -31,57 +31,10 @@ import { Box, } from "@mui/material"; -import { - BarChart, - BarSeries, - Bar, - BarLabel, - - GridlineSeries, - Gridline, - TooltipArea, - ChartTooltip, - TooltipTemplate, -} from 'reaviz'; - import { typecost, typecost_single, } from "../views/HandlePaymentNew.jsx"; +import LineChartWrapper from '../components/LineChartWrapper.jsx'; import { Context } from '../context/ContextApi.jsx'; -const LineChartWrapper = ({keys, inputname, height, width}) => { - const [hovered, setHovered] = useState(""); - const inputdata = keys.data === undefined ? keys : keys.data - const {themeMode} = useContext(Context) - const theme = getTheme(themeMode) - - - return ( -
- - {inputname} - - - - } - /> - } - gridlines={ - } /> - } - /> - -
- ) -} - const AppStats = (defaultprops) => { const { diff --git a/frontend/src/components/CacheView.jsx b/frontend/src/components/CacheView.jsx index 1e94d131..e9670baf 100644 --- a/frontend/src/components/CacheView.jsx +++ b/frontend/src/components/CacheView.jsx @@ -4,7 +4,8 @@ import { getTheme } from "../theme.jsx"; import { toast } from 'react-toastify'; import ReactJson from "react-json-view-ssr"; -import { GetIconInfo } from "../views/Workflows2.jsx"; +import { GetIconInfo, } from "../views/Workflows2.jsx"; +import { validateJson, handleReactJsonClipboard, } from "../views/Workflows.jsx"; import { red } from "../views/AngularWorkflow.jsx"; import CollectIngestModal from "../components/CollectIngestModal.jsx"; import { @@ -76,8 +77,8 @@ import { SmartToy as SmartToyIcon, Settings as SettingsIcon, FilterAlt as FilterAltIcon, + CompareArrows as CompareArrowsIcon, } from "@mui/icons-material"; -import { validateJson, } from "../views/Workflows.jsx"; import { Context } from "../context/ContextApi.jsx"; const scrollStyle1 = { @@ -130,7 +131,6 @@ const CacheView = memo((props) => { }) const [_, setUpdate] = useState(Math.random()) const [selectedRows, setSelectedRows] = useState([]); - // Direct category migration from ../components/Files.jsx const [selectAllChecked, setSelectAllChecked] = React.useState(false) const [renderTextBox, setRenderTextBox] = React.useState(false); @@ -139,12 +139,14 @@ const CacheView = memo((props) => { const [selectedFileId, setSelectedFileId] = React.useState(""); const [updateToThisCategory, setUpdateToThisCategory] = useState("") const [workflows, setWorkflows] = useState([]); + const [apps, setApps] = useState([]); const [selectedFiles, setSelectedFiles] = useState([]); const [showAutomationMenu, setShowAutomationMenu] = useState(false); const [showSettingsMenu, setShowSettingsMenu] = useState(false); const [showCollectIngestMenu, setShowCollectIngestMenu] = useState(false); + var to_be_copied = ""; const defaultAutomation = [ { "name": "Run workflow", @@ -156,6 +158,42 @@ const CacheView = memo((props) => { "icon": , "enabled": false, }, + { + "name": "Correlate Categories", + "description": "", + "type": "singul", + "options": [{ + "key": "datastore_categories", + "value": "", + }], + "icon": , + "enabled": false, + "disabled": false, + }, + { + "name": "Run AI Agent", + "description": "", + "options": [{ + "key": "", + "value": "", + }], + "icon": , + "enabled": false, + "disabled": true, + }, + { + "name": "Send webhook", + "description": "Sends the updated value to a specified webhook URL.", + "options": [{ + "key": "webhook_url", + "value": "", + }], + "icon": , + "enabled": false, + }, + + + { "name": "Send message", "description": "", @@ -180,27 +218,6 @@ const CacheView = memo((props) => { "enabled": false, "disabled": true, }, - { - "name": "Run AI Agent", - "description": "", - "options": [{ - "key": "", - "value": "", - }], - "icon": , - "enabled": false, - "disabled": true, - }, - { - "name": "Send webhook", - "description": "Sends the updated value to a specified webhook URL.", - "options": [{ - "key": "webhook_url", - "value": "", - }], - "icon": , - "enabled": false, - }, ] const [categoryAutomations, setCategoryAutomations] = useState(defaultAutomation) @@ -210,6 +227,35 @@ const CacheView = memo((props) => { const theme = getTheme(themeMode, brandColor); const classes = useStyles(); + const getApps = () => { + const url = `${globalUrl}/api/v1/apps` + fetch(url, { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for workflows :O!"); + return; + } + + return response.json(); + }) + .then((responseJson) => { + if (responseJson?.success === false) { + toast.warn("Failed to load apps. Please try again or contact support@shuffler if this persists.") + } else { + setApps(responseJson) + } + }) + .catch((error) => { + toast(error.toString()); + }); + } const getWorkflows = () => { const url = `${globalUrl}/api/v1/workflows` @@ -251,6 +297,7 @@ const CacheView = memo((props) => { useEffect(() => { getWorkflows() + getApps() listOrgCache(orgId, selectedCategory, 0, pageSize, page) }, []) @@ -322,7 +369,7 @@ const CacheView = memo((props) => { .then((responseJson) => { setCachedLoaded(true); if (responseJson?.success === true) { - setListCache(responseJson.keys) + setListCache(responseJson.keys); if (responseJson.total_amount !== undefined && responseJson.total_amount !== null && responseJson.total_amount > 0) { setTotalAmount(responseJson.total_amount) @@ -563,34 +610,6 @@ const CacheView = memo((props) => { } } - const handleReactJsonClipboard = (copy) => { - const elementName = "copy_element_shuffle"; - let copyText = document.getElementById(elementName); - - if (copyText) { - if (copy.namespace && copy.name && copy.src) { - copy = copy.src; - } - - const clipboard = navigator.clipboard; - if (!clipboard) { - toast("Can only copy over HTTPS (port 3443)"); - return; - } - - let stringified = JSON.stringify(copy); - if (stringified.startsWith('"') && stringified.endsWith('"')) { - stringified = stringified.slice(1, -1); - } - - navigator.clipboard.writeText(stringified); - toast("Copied value to clipboard, NOT json path."); - } else { - console.log("Failed to copy from " + elementName + ": ", copyText); - } - }; - - const timestamp = (timestamp) => { if (timestamp === undefined || timestamp === null || timestamp === "") { return null @@ -1096,7 +1115,153 @@ const CacheView = memo((props) => { {showOptions && ( updatedAutomation.options.map((option, optionIndex) => { - if (option?.key === "workflow_id") { + if (option?.key === "datastore_categories") { + if (datastoreCategories === undefined || datastoreCategories === null || datastoreCategories.length <= 1) { + return ( + + No categories available. Please add categories in the settings. + + ) + } + + return ( + option?.value.includes(c)) || []} + classes={{ inputRoot: classes.inputRoot }} + ListboxProps={{ + style: { + backgroundColor: theme.palette.surfaceColor, + color: theme.palette.text.primary, + borderRadius: theme.palette.borderRadius, + }, + }} + onChange={(event, newValue) => { + console.log("New Value: ", newValue) + + option.value = "" + for (var i = 0; i < newValue.length; i++) { + option.value += newValue[i] + "," + } + + if (newValue.length > 0) { + updatedAutomation.enabled = true + } else { + updatedAutomation.enabled = false + } + + updatedAutomation.options[optionIndex] = option + setUpdatedAutomation(updatedAutomation) + setUpdated(true) + + setUpdate(Math.random()) // Force re-render + }} + + getOptionLabel={(option) => { + if (option === undefined || option === null) { + return "No Categories Selected"; + } + + return option + }} + options={datastoreCategories} + fullWidth + style={{ + backgroundColor: theme.palette.textFieldStyle.backgroundColor, + borderRadius: theme.palette.textFieldStyle.borderRadius, + color: theme.palette.textFieldStyle.color, + height: 35, + marginBottom: 40, + }} + renderOption={(props, data, state) => { + const fixedname = data?.charAt(0)?.toUpperCase() + data?.slice(1)?.replaceAll("_", " ") + const iconDetails = GetIconInfo({ + "app_name": fixedname, + "name": fixedname, + }) + + const keyfound = option?.value.includes(data) + + return ( + + {data.image !== undefined && data.image !== null && data.image.length > 0 ? + {data.name} + : null} + + Choose {data} + + + } > + + +
+ {iconDetails?.originalIcon && ( + iconDetails?.originalIcon + )} +
+ + {fixedname} + + + + ) + }} + renderInput={(params) => { + return ( + + ) + }} + /> + ) + } else if (option?.key === "workflow_id") { return ( { {...props} style={{ backgroundColor: theme.palette.surfaceColor, - color: data.id === option?.value ? "red" : theme.palette.text.primary, - borderBottom: data.id === "parent" ? "2px solid rgba(255,255,255,0.5)" : null + color: data.id === option?.value ? red : theme.palette.text.primary, }} value={data} > @@ -1327,16 +1491,11 @@ const CacheView = memo((props) => { }} collapsed={true} enableClipboard={(copy) => { - // handleReactJsonClipboard(copy); + handleReactJsonClipboard(copy) }} collapseStringsAfterLength={theme.palette.jsonCollapseStringsAfterLength} iconStyle={theme.palette.jsonIconStyle} displayDataTypes={false} - onSelect={(select) => { - - // HandleJsonCopy(showResult, select, data.action.label); - console.log("SELECTED!: ", select); - }} name={null} /> : @@ -1419,7 +1578,9 @@ const CacheView = memo((props) => { { + onClick={(e) => { + e.preventDefault() + e.stopPropagation() // Try to make the value JSON indented const valid = validateJson(data.value) var newvalue = data.value @@ -1469,7 +1630,9 @@ const CacheView = memo((props) => { { + onClick={(e) => { + e.preventDefault() + e.stopPropagation() window.open(`${globalUrl}/api/v1/orgs/${orgId}/cache/${data.key}?type=text&authorization=${data.public_authorization}`, "_blank"); }} > @@ -1486,7 +1649,9 @@ const CacheView = memo((props) => { { + onClick={(e) => { + e.preventDefault() + e.stopPropagation() deleteEntry(orgId, data.key, data.category) }} > @@ -1609,6 +1774,8 @@ const CacheView = memo((props) => { workflows={workflows} getWorkflows={getWorkflows} + + apps={apps} /> {cacheDistributionModal} @@ -2067,12 +2234,14 @@ const CacheView = memo((props) => { { + setSelectedRows(newSelection); + }} onRowSelectionModelChange={(newSelection) => { - setSelectedRows(newSelection) + setSelectedRows(newSelection); }} keepNonExistentRowsSelected={false} getRowId={(row) => row.key} @@ -2245,6 +2414,11 @@ const CacheView = memo((props) => {
+ ); }) diff --git a/frontend/src/components/CollectIngestModal.jsx b/frontend/src/components/CollectIngestModal.jsx index afba4afa..499b2fc3 100644 --- a/frontend/src/components/CollectIngestModal.jsx +++ b/frontend/src/components/CollectIngestModal.jsx @@ -10,10 +10,14 @@ import { DialogTitle, DialogContent, Typography, + IconButton, Paper, LinearProgress, Grid, Button, + Tooltip, + Autocomplete, + TextField, } from '@mui/material'; import { @@ -22,7 +26,7 @@ import { } from '@mui/icons-material'; const CollectIngestModal = (props) => { - const { globalUrl, open, setOpen } = props; + const { globalUrl, open, setOpen, workflows, getWorkflows, apps, } = props; const { themeMode, brandColor } = useContext(Context); const theme = getTheme(themeMode, brandColor); @@ -37,12 +41,17 @@ const CollectIngestModal = (props) => { return null } - const startIngestion = (appname, index) => { - console.log("APPNAME:", appname, "INDEX:", index) + const startIngestion = (bundleName, appnames, category, index) => { - const body = { - "app_name": appname, - "label": appname, + var body = { + "label": bundleName, + "app_name": appnames, + + "category": "", + } + + if (category !== undefined && category !== null) { + body.category = category } const url = `${globalUrl}/api/v2/workflows/generate` @@ -61,21 +70,27 @@ const CollectIngestModal = (props) => { return response.json(); }) .then((data) => { + + if (getWorkflows !== undefined) { + getWorkflows() + } + console.log("Ingestion started successfully:", data); - toast.success(`Ingestion for ${appname} started successfully!`); + toast.success(`Ingestion for ${bundleName} started successfully!`); }) .catch((error) => { console.error("Error starting ingestion:", error); - toast.error(`Failed to start ingestion for ${appname}. Please try again.`); + toast.error(`Failed to start ingestion for ${bundleName}. Please try again or contact support@shuffler.io if this persists.`); }); } const IngestItem = (props) => { - const { type, index } = props + const { type, appCategory, index } = props const [hovering, setHovering] = useState(false); - const [isFinished, setIsFinished] = useState(false); + const [selectedApps, setSelectedApps] = useState([]); + //const [isFinished, setIsFinished] = useState(false); const appname = type const ingestedAmount = 20 @@ -85,24 +100,93 @@ const CollectIngestModal = (props) => { "name": appname, }) + var foundMatchingWorkflow = null + if (workflows !== undefined && workflows !== null && workflows.length > 0) { + const parsedName = type.toLowerCase().replaceAll(" ", "_"); + const foundWorkflow = workflows.find((workflow) => { + return workflow?.name?.toLowerCase().replaceAll(" ", "_") === parsedName + }) + + if (foundWorkflow !== undefined && foundWorkflow !== null) { + foundMatchingWorkflow = foundWorkflow + + + // Find relevant apps and maps them + if (apps.length > 0 && selectedApps.length === 0 && foundWorkflow?.actions !== undefined && foundWorkflow?.actions !== null && foundWorkflow?.actions.length > 0) { + var newSelectedApps = [] + for (var actionkey in foundWorkflow.actions) { + const action = foundWorkflow.actions[actionkey] + + if (action?.app_name !== "Singul" && action?.app_id !== "integration") { + continue + } + + for (var paramkey in action.parameters) { + const param = action.parameters[paramkey] + if (!((param.name === "app_name" || param.name === "appName") && param.value !== undefined && param.value !== null && param.value.length > 0)) { + continue + } + + // Find the app in available apps + const parsedname = param.value.replaceAll(" ", "_").toLowerCase() + for (var appkey in apps) { + const appname = apps[appkey].name.replaceAll(" ", "_").toLowerCase() + if (appname === parsedname) { + newSelectedApps.push(apps[appkey]) + break + } + } + + break + } + } + + if (newSelectedApps.length > 0) { + setSelectedApps(newSelectedApps) + } + } + + } + } + + var matchingapps = [] + if (appCategory !== undefined && appCategory !== null && appCategory.length > 0 && apps !== undefined && apps !== null && apps.length > 0) { + for (var appkey in apps) { + const app = apps[appkey] + + for (var categorykey in app.categories) { + const category = app.categories[categorykey] + if (category.toLowerCase() === appCategory.toLowerCase()) { + matchingapps.push(app) + } + } + } + } + return ( // setHovering(true)} + onMouseEnter={() => { + + //if (foundMatchingWorkflow !== null) { + //} else { + setHovering(true) + //} + }} onMouseLeave={() => setHovering(false)} >
@@ -116,18 +200,92 @@ const CollectIngestModal = (props) => {
- +
+ {matchingapps.length > 0 ? + { + setSelectedApps(value) + }} + + getOptionLabel={(option) => { + const parsedname = option.name.replaceAll("_", " ") + + return ( +
+ {option.name} + + {parsedname} + +
+ ) + }} + renderInput={(params) => { + return ( + + ) + }} + /> + : null} + + + +
+ + {foundMatchingWorkflow !== null ? + + + + + + + + : null} {hovering ?
: null} - {isFinished ? + {foundMatchingWorkflow !== null ?
{ }}> {ingestedAmount} / X + + {/* { variant="determinate" fullWidth value={{ingestedAmount}} /> + */}
: null} @@ -189,13 +350,13 @@ const CollectIngestModal = (props) => { - + - + diff --git a/frontend/src/components/DashboardBarchart.jsx b/frontend/src/components/DashboardBarchart.jsx index 74970549..fbeab07e 100644 --- a/frontend/src/components/DashboardBarchart.jsx +++ b/frontend/src/components/DashboardBarchart.jsx @@ -1,7 +1,8 @@ import React from 'react'; -import { Bar } from 'react-chartjs-2'; import { toast } from "react-toastify"; +import LineChartWrapper from '../components/LineChartWrapper.jsx'; + export const LoadStats = (globalUrl, cachekey) => { if (globalUrl === undefined) { console.log("Error: Global URL is undefined") @@ -64,84 +65,13 @@ export const LoadStats = (globalUrl, cachekey) => { }) } +// This wrapper is a waste lol const DashboardBarchart = (props) => { - // this is clearly unfinished and not worth my time. - // refer to health page to see how i made it work there w/o using chartjs + const { timelineData, title, height, } = props; - // const { timelineData, title, height, } = props; - // var inputHeight = 15 - // if (height !== undefined && height !== null) { - // inputHeight = height - // } - - // const barOptions = { - // plugins: { - // tooltip: { - // enabled: true, // Ensure tooltips are enabled - // }, - // }, - // tooltips: { - // mode: 'index', - // intersect: false, - // }, - // legend: { - // display: false - // }, - // layout: { - // padding: { - // top: 0, // Adjust the top padding as needed - // bottom: -10, // Adjust the bottom padding as needed - // left: 0, // Adjust the left padding as needed - // right: 0, // Adjust the right padding as needed - // }, - // }, - // scales: { - // y: { - // beginAtZero: false, - // }, - // yAxes: [{ - // ticks: { - // display: false - // }, - // beginAtZero: false, - // }], - // xAxes: [{ - // ticks: { - // display: false - // }, - // beginAtZero: false, - // }] - // }, - // tooltips: { - // callbacks: { - // label: function (tooltipItem, data) { - // const label = data.labels[tooltipItem.index] - // return label.split('\n')[0] - // }, - // afterLabel: function (tooltipItem, data) { - // const amount = tooltipItem.value === undefined || tooltipItem.value === null ? 0 : tooltipItem.value - // return `Amount: ${amount}` - // }, - // title: function () { - // return title === undefined ? '' : title - // } - // } - // } - // } - - // return ( - // { - // if (elements && elements.length > 0) { - // //toast("Click event") - // console.log("Clicked: ", elements) - // } - // }} - // /> - // ) + return ( + + ) } export default DashboardBarchart; diff --git a/frontend/src/components/EditOrgTab.jsx b/frontend/src/components/EditOrgTab.jsx index 0e2b5176..b1646e68 100644 --- a/frontend/src/components/EditOrgTab.jsx +++ b/frontend/src/components/EditOrgTab.jsx @@ -25,6 +25,7 @@ const EditOrgTab = (props) => { handleGetOrg, selectedStatus, setSelectedStatus, handleEditOrg, + handleStatusChange } = props; const [organizationFeatures, setOrganizationFeatures] = React.useState({}); const [users, setUsers] = React.useState([]); @@ -39,43 +40,6 @@ const EditOrgTab = (props) => { const { themeMode, brandColor } = useContext(Context); const theme = getTheme(themeMode, brandColor); - const handleStatusChange = (event) => { - const { value } = event.target; - setSelectedStatus(value); - - handleEditOrg( - selectedOrganization?.name, - selectedOrganization?.description, - selectedOrganization.id, - selectedOrganization.image, - { - app_download_repo: selectedOrganization?.defaults?.app_download_repo, - app_download_branch: selectedOrganization?.defaults?.app_download_branch, - workflow_download_repo: selectedOrganization?.defaults?.workflow_download_repo, - workflow_download_branch: selectedOrganization?.defaults?.workflow_download_branch, - notification_workflow: selectedOrganization?.defaults?.notification_workflow, - documentation_reference: selectedOrganization?.defaults?.documentation_reference, - workflow_upload_repo: selectedOrganization?.defaults?.workflow_upload_repo, - workflow_upload_branch: selectedOrganization?.defaults?.workflow_upload_branch, - workflow_upload_username: selectedOrganization?.defaults?.workflow_upload_username, - workflow_upload_token: selectedOrganization?.defaults?.workflow_upload_token, - newsletter: selectedOrganization?.defaults?.newsletter, - weekly_recommendations: selectedOrganization?.defaults?.weekly_recommendations, - }, - { - sso_entrypoint: selectedOrganization?.sso_config?.sso_entrypoint, - sso_certificate: selectedOrganization?.sso_config?.sso_certificate, - client_id: selectedOrganization?.sso_config?.client_id, - client_secret: selectedOrganization?.sso_config?.client_secret, - openid_authorization: selectedOrganization?.sso_config?.openid_authorization, - openid_token: selectedOrganization?.sso_config?.openid_token, - SSORequired: selectedOrganization?.sso_config?.SSORequired, - auto_provision: selectedOrganization?.sso_config?.auto_provision, - }, - value.length === 0 ? ["none"] : value, - ); - }; - const getUsers = () => { fetch(globalUrl + "/api/v1/getusers", { @@ -443,6 +407,7 @@ If you're interested, please let me know a time that works for you, or set up a isEditOrgTab={true} handleGetOrg={handleGetOrg} serverside={serverside} + handleStatusChange={handleStatusChange} /> diff --git a/frontend/src/components/Files.jsx b/frontend/src/components/Files.jsx index 71c3af28..50872ae9 100644 --- a/frontend/src/components/Files.jsx +++ b/frontend/src/components/Files.jsx @@ -25,8 +25,12 @@ import { Checkbox, Chip, Menu, + Pagination, + PaginationItem, } from "@mui/material"; +import { DataGrid } from "@mui/x-data-grid"; + import { Link as LinkIcon, OpenInNew as OpenInNewIcon, @@ -45,6 +49,7 @@ import Dropzone from "../components/Dropzone.jsx"; import ShuffleCodeEditor from "../components/ShuffleCodeEditor1.jsx"; import {getTheme} from "../theme.jsx"; import { Context } from "../context/ContextApi.jsx"; +import { red } from "../views/AngularWorkflow.jsx"; const Files = memo((props) => { const { globalUrl, userdata, serverside, selectedOrganization, isCloud,isSelectedFiles } = props; @@ -75,9 +80,370 @@ const Files = memo((props) => { const [showDistributionPopup, setShowDistributionPopup] = useState(false) const [selectedSubOrg, setSelectedSubOrg] = useState([]) const [fileIdSelectedForDistribution, setFileIdSelectedForDistribution] = useState("") + const [totalAmount, setTotalAmount] = useState(0); +const [page, setPage] = useState(0); +const [pageSize, setPageSize] = useState(50) +const [selectedRows, setSelectedRows] = useState([]); +const [filesLoaded, setFilesLoaded] = useState(false); //const alert = useAlert(); const allowedFileTypes = ["txt", "py", "yaml", "yml","json", "html", "js", "csv", "log", "eml", "msg", "md", "xml", "sh", "bat", "ps1", "psm1", "psd1", "ps1xml", "pssc", "psc1", "response"] var upload = ""; + const paginatedRows = files.slice(page * pageSize, (page + 1) * pageSize); + + const columns = [ + { + field : 'filename', + headerName: 'Name', + filterable: true, + sortable: true, + width: 250, + renderCell: (params) => { + if (params.row.filename === undefined || params.row.filename === null || params.row.filename.length < 1) { + return ( + + No name + + ) + } + + return ( + + + {params.row.filename} + + + ); + } + }, + { + field: 'Workflow', + headerName: 'Workflow', + renderCell: (params) => { + const file = params.row; + return ( + file.workflow_id === "global" || !file.workflow_id ? ( + + + + ) : ( + + + + + + + + + + ) + ); + }, +}, +{ + field: 'md5_sum', + headerName: 'MD5', + width: 100, + }, + { + field: "Status", + headerName: "Status", + renderCell: (params) => { + const file = params.row; + return ( + + {file.status.charAt(0).toUpperCase() + file.status.slice(1)} + + ); + } + }, + { + field: "filesize", + headerName: "Filesize", + + }, + { + field: "actions", + headerName: "Actions", + width: 200, + renderCell: (params) => { + const file = params.row; + const filenamesplit = file.filename.split(".") + const iseditable = file.filesize < 2000000 && file.status === "active" && (allowedFileTypes.includes(filenamesplit[filenamesplit.length-1]) || !file?.filename.includes(".")) + return ( + + + + { + e.stopPropagation(); + e.preventDefault(); + setOpenEditor(true) + setOpenFileId(file.id) + readFileData(file) + }} + > + + + + + + + {/* + + + { + // Open the file, without downloading it + window.open(`${globalUrl}/api/v1/files/${file.id}/content?type=text&authorization=${file.public_authorization}`, "_blank noreferrer noopener") + }} + > + + + + + */} + + + { + e.stopPropagation(); + e.preventDefault(); + downloadFile(file); + }} + > + + + + + + + + + + + + { + e.stopPropagation(); + e.preventDefault(); + navigator.clipboard.writeText(file.id); + document.execCommand("copy"); + + toast(file.id + " copied to clipboard"); + }} + > + + + + + + + + + + { + e.stopPropagation(); + e.preventDefault(); + deleteFile(file.id, true); + }} + > + + + + + + + + + ) + } + }, + { + field: "distribution", + headerName: "Distribution", + renderCell: (params) => { + const file = params.row; + const isDistributed = file?.suborg_distribution?.length > 0 ? true : false; + + return ( + <> + {selectedOrganization.id !== undefined && file?.org_id !== selectedOrganization.id ? + + + + : + + { + e.stopPropagation(); + e.preventDefault(); + setShowDistributionPopup(true) + if(file?.suborg_distribution?.length > 0){ + setSelectedSubOrg(file.suborg_distribution) + }else{ + setSelectedSubOrg([]) + } + setFileIdSelectedForDistribution(file.id) + }} + /> + + } + + ) + } + } + ] + const handleKeyDown = (event) => { if (event.key === 'Enter') { @@ -97,7 +463,7 @@ const Files = memo((props) => { editFileConfig(id, "suborg_distribute", [...new Set(selectedSubOrg)]) } - + const editFileConfig = (id, parentAction, selectedSubOrg) => { const data = { id: id, @@ -121,9 +487,9 @@ const Files = memo((props) => { .then((response) => response.json().then((responseJson) => { if (responseJson["success"] === false) { - toast("Failed overwriting files"); + toast.error("Failed overwriting files"); } else { - toast("Successfully updated file!"); + toast.success("File updated!"); setTimeout(() => { getFiles(); }, 1000); @@ -170,6 +536,7 @@ const Files = memo((props) => { } const getFiles = (namespace) => { + setFilesLoaded(false) var parsedurl = `${globalUrl}/api/v1/files` if (namespace === undefined || namespace === null || namespace === "default") { @@ -199,6 +566,11 @@ const Files = memo((props) => { .then((responseJson) => { if (responseJson.files !== undefined && responseJson.files !== null) { setFiles(responseJson.files); + if (responseJson.total_amount !== undefined && responseJson.total_amount !== null && responseJson.total_amount > 0) { + setTotalAmount(responseJson.total_amount) + } else { + setTotalAmount(responseJson.files.length) + } setShowLoader(false) setShowDistributionPopup(false) } else if (responseJson.list !== undefined && responseJson.list !== null) { @@ -227,7 +599,9 @@ const Files = memo((props) => { }) .catch((error) => { toast(error.toString()); - }); + }).finally(() => { + setFilesLoaded(true) + }); }; useEffect(() => { @@ -585,8 +959,12 @@ const Files = memo((props) => { ): null - const deleteFile = (file) => { - fetch(globalUrl + "/api/v1/files/" + file.id, { + const deleteFile = (fileId, showSinglDeleteToast) => { + + console.log("Deleting file with ID: ", fileId) + console.log("showSinglDeleteToast: ", showSinglDeleteToast) + + fetch(globalUrl + "/api/v1/files/" + fileId, { method: "DELETE", headers: { "Content-Type": "application/json", @@ -602,18 +980,20 @@ const Files = memo((props) => { return response.json(); }) .then((responseJson) => { - if (responseJson.success) { - toast("Successfully deleted file") + if (responseJson.success && showSinglDeleteToast === true) { + toast.success("Deleted file") } else if ( responseJson.reason !== undefined && responseJson.reason !== null ) { - toast("Failed to delete file: " + responseJson.reason); + toast.error("Failed to delete file: " + responseJson.reason); } - setTimeout(() => { - getFiles(selectedCategory) - }, 1500); + if (showSinglDeleteToast === true) { + setTimeout(() => { + getFiles(selectedCategory) + }, 1500); + } }) .catch((error) => { toast(error.toString()); @@ -909,7 +1289,7 @@ const Files = memo((props) => { {fileDistributionModal}
-
+
@@ -1131,529 +1511,136 @@ const Files = memo((props) => { backgroundColor: theme.palette.textFieldStyle.backgroundColor, }} />} -
- - + { + setSelectedRows(newSelection); + }} + sx={{ + marginTop: 1, + height: files.length*52, + width: "100%", + '.MuiTablePagination-selectLabel, .MuiTablePagination-select, .MuiTablePagination-selectIcon': { + display: 'none', + }, + marginBottom: 20, + }} + hideFooterSelectedRowCount={true} + hideFooter={true} + pagination + autoHeight={true} + getRowId={(row) => row.id} + keepNonExistentRowsSelected={false} + loading={filesLoaded === false} + /> +
- {[ - - { - setSelectAllChecked((prev) => !prev); - setSelectedFiles((prev) => { - if (prev.length === files.length) { - return [] - } else { - return files.map((_, index) => !prev.includes(index)) - } - }) - if (selectAllChecked) { - setSelectedFileId([]) - } else { - setSelectedFileId( - files - .filter((file) => file.namespace === selectedCategory) - .map((file) => file.id) - ); - } - }} - /> - , - "Name", - "Workflow", - "Md5", - "Status", - "Filesize", - "Actions", - "Distribution" - ] - .filter(Boolean) - .map((header, index) => ( - - ))} - - {showLoader ? - [...Array(6)].map((_, rowIndex) => ( - - {Array(8) - .fill() - .map((_, colIndex) => ( - - - - ))} - - )): - files.length === 0 ? ( -
- - No files found - -
- ):( - files?.map((file, index) => { - if (file.namespace === "") { - file.namespace = "default"; - } - - if (file.namespace !== selectedCategory) { - return null; - } - - var bgColor = themeMode === "dark" ? "#212121" : "#FFFFFF"; - if (index % 2 === 0) { - bgColor = themeMode === "dark" ? "#1A1A1A" : "#EAEAEA"; - } - const isDistributed = file?.suborg_distribution?.length > 0 ? true : false; - const filenamesplit = file.filename.split(".") - const iseditable = file.filesize < 2000000 && file.status === "active" && (allowedFileTypes.includes(filenamesplit[filenamesplit.length-1]) || !file?.filename.includes(".")) - return ( - - {/* - - */} - - {handleFileCheckboxChange(index); setSelectedFileId(prev => { - if (prev.includes(file.id)) { - return prev.filter((item) => item !== file.id) - } else { - return [...prev, file.id] - } - })}} - /> - - - - - - : ( - - - - - - - - - - ) - } - style={{ - display: 'table-cell', - overflow: "hidden", - }} - /> - - {file.md5_sum} - - )} - primaryTypographyProps={{ - style:{ - display: 'table-cell', - marginLeft:isSelectedFiles? 15:null, - overflow: "hidden", - whiteSpace: 'nowrap', - textOverflow: 'ellipsis', - maxWidth: 200, - } - }} - /> - - - - - - { - setOpenEditor(true) - setOpenFileId(file.id) - readFileData(file) - }} - > - - - - - - - {/* - - - { - // Open the file, without downloading it - window.open(`${globalUrl}/api/v1/files/${file.id}/content?type=text&authorization=${file.public_authorization}`, "_blank noreferrer noopener") - }} - > - - - - - */} - - - { - downloadFile(file); - }} - > - - - - - - +
- - - - { - navigator.clipboard.writeText(file.id); - document.execCommand("copy"); - - toast(file.id + " copied to clipboard"); - }} - > - - - - - - - - - - { - deleteFile(file); - }} - > - - - - - - - - + display: "flex", + textAlign: "center", + }}> + + {page * pageSize + 1} - {Math.min((page + 1) * pageSize, totalAmount)} of {totalAmount} + + + { + + return ( + + ) + + }} + onChange={(e, value) => { + if (value < 1) { + return + } + + const newPage = value-1 + console.log("New page: ", value) + // handleChangePage() + + setPage(newPage) + }} + /> + + {selectedRows.length > 0 ? + + : null} +
+
-
+
) }) diff --git a/frontend/src/components/Navbar.jsx b/frontend/src/components/Navbar.jsx index 07be2cb8..41b80efe 100644 --- a/frontend/src/components/Navbar.jsx +++ b/frontend/src/components/Navbar.jsx @@ -49,7 +49,7 @@ const menuData = { { title: "Shuffle", description: - "The most versatile automation engine with focus on security.", + "The most versatile automation engine. Focused on cybersecurity.", icon: "/images/icons/shuffleLogo.svg", path: "/docs/about", gaData: { @@ -61,7 +61,7 @@ const menuData = { { title: "Singul", description: - "Connect your favorite services with a singul line of code.", + "Connect to your favorite services with a singul line of code.", icon: "/images/logos/singul.svg", path: "https://singul.io", gaData: { @@ -70,6 +70,7 @@ const menuData = { label: "singul_click" } }, + /* { title: "API Explorer", description: @@ -82,6 +83,7 @@ const menuData = { label: "api_explorer_click" } }, + */ ], Services: [ { @@ -1245,23 +1247,6 @@ const Navbar = (props) => { > {menuItem.title} - {menuItem.title === "Singul" && ( - - Beta: Coming Soon - - )} { adminTab, selectedStatus, setSelectedStatus, - isEditOrgTab + isEditOrgTab, + handleStatusChange } = props; const classes = useStyles(); @@ -68,7 +69,7 @@ const OrgHeaderexpandedNew = (props) => { style: { maxHeight: ITEM_HEIGHT * 4.5 + ITEM_PADDING_TOP, width: 300, - borderRadius: 20, + borderRadius: 4, overflowY: "scroll", }, }, @@ -93,40 +94,6 @@ const OrgHeaderexpandedNew = (props) => { const { themeMode, supportEmail, brandColor } = useContext(Context); const theme = getTheme(themeMode, brandColor); - const handleStatusChange = (event) => { - const { value } = event.target; - handleEditOrg( - orgName, - orgDescription, - selectedOrganization.id, - selectedOrganization?.image, - { - app_download_repo: selectedOrganization?.defaults?.app_download_repo, - app_download_branch: selectedOrganization?.defaults?.app_download_branch, - workflow_download_repo: selectedOrganization?.defaults?.workflow_download_repo, - workflow_download_branch: selectedOrganization?.defaults?.workflow_download_branch, - notification_workflow: selectedOrganization?.defaults?.notification_workflow, - documentation_reference: selectedOrganization?.defaults?.documentation_reference, - workflow_upload_repo: selectedOrganization?.defaults?.workflow_upload_repo, - workflow_upload_branch: selectedOrganization?.defaults?.workflow_upload_branch, - workflow_upload_username: selectedOrganization?.defaults?.workflow_upload_username, - workflow_upload_token: selectedOrganization?.defaults?.workflow_upload_token, - newsletter: selectedOrganization?.defaults?.newsletter, - weekly_recommendations: selectedOrganization?.defaults?.weekly_recommendations, - }, - { - sso_entrypoint: selectedOrganization?.sso_config?.sso_entrypoint, - sso_certificate: selectedOrganization?.sso_config?.sso_certificate, - client_id: selectedOrganization?.sso_config?.client_id, - client_secret: selectedOrganization?.sso_config?.client_secret, - openid_authorization: selectedOrganization?.sso_config?.openid_authorization, - openid_token: selectedOrganization?.sso_config?.openid_token, - SSORequired: selectedOrganization?.sso_config?.SSORequired, - auto_provision: selectedOrganization?.sso_config?.auto_provision, - }, - value.length === 0 ? ["none"] : value, - ) - } const [appDownloadBranch, setAppDownloadBranch] = React.useState( selectedOrganization.defaults === undefined ? defaultBranch @@ -544,7 +511,7 @@ const OrgHeaderexpandedNew = (props) => { renderValue={(selected) => selected.join(', ')} MenuProps={MenuProps} > - {["contacted", "lead", "demo done", "pov", "customer", "open source", "student", "internal", "creator", "tech partner", "integration partner", "distribution partner", "service partner", "old customer", "old lead"].map((name) => ( + {["contacted", "lead", "demo done", "pov", "customer", "open source", "student", "internal", "creator", "tech partner", "integration partner", "distribution partner", "channel partner", "service partner", "old customer", "old lead"].map((name) => ( -1} /> diff --git a/frontend/src/components/OrganizationTab.jsx b/frontend/src/components/OrganizationTab.jsx index d7f2acc3..ce4e5c9e 100644 --- a/frontend/src/components/OrganizationTab.jsx +++ b/frontend/src/components/OrganizationTab.jsx @@ -36,7 +36,7 @@ const OrganizationTab = (props) => { const [billingInfo, setBillingInfo] = useState({}); const [orgRequest, setOrgRequest] = React.useState(true); const [curIndex, setCurIndex] = React.useState(0); - const items = ['Org Configuration', "SSO", "Notifications", 'Billing & Stats', 'Branding']; + const items = ['Org Configuration', "SSO", "Notifications", 'Billing & Stats']; const [visibleTabs, setVisibleTabs] = useState(items); const [unreadNotifications, setUnreadNotifications] = React.useState( notifications?.filter((notification) => notification.read === false)?.length @@ -157,16 +157,16 @@ const OrganizationTab = (props) => { isLoaded={isLoaded} /> ); - case 'branding': - return ; + // case 'branding': + // return ; // case 'analytics': // return ; default: diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index fe7b5649..dcc64043 100755 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -904,13 +904,12 @@ const ParsedAction = (props) => { if (paramvalue.includes("$")) { let actions = workflow.actions?.map((action) => { - return "$" + action.label?.toLowerCase(); + return "$" + action.label?.toLowerCase().replaceAll(" ", "_"); }) if (newActionList?.length > 0) { - let appParentActions = parentActionList?.map(action => "$" + action.name.toLowerCase()); + let appParentActions = parentActionList?.map(action => "$" + action.name.toLowerCase().replaceAll(" ", "_")); let notPresentAction = actions?.filter((action) => !appParentActions?.includes(action)) - // Extract all variable references from paramvalue // Examples of what it matches: // - Simple variables: $test, $myVar, $x diff --git a/frontend/src/components/PartnerDetails.jsx b/frontend/src/components/PartnerDetails.jsx index 755e187d..397af9eb 100644 --- a/frontend/src/components/PartnerDetails.jsx +++ b/frontend/src/components/PartnerDetails.jsx @@ -358,6 +358,21 @@ const PartnerDetails = (props) => { animation="wave" /> +
+ + Contact Email + + +
@@ -824,7 +839,7 @@ const PartnerDetails = (props) => { value={partnerData?.article_url} onBlur={() => {}} onChange={(e) => { - if (e.target.value.length > 100) { + if (e.target.value.length > 1000) { toast("Choose a shorter article URL."); return; } @@ -851,7 +866,65 @@ const PartnerDetails = (props) => { }, }} /> - + +
+ + Contact Email + + {}} + onChange={(e) => { + setPartnerData({ + ...partnerData, + contact_email: e.target.value, + }); + }} + color="primary" + InputProps={{ + style: { + color: theme.palette.textFieldStyle.color, + height: "35px", + fontSize: "1em", + borderRadius: 4, + backgroundColor: + theme.palette.textFieldStyle.backgroundColor, + }, + classes: { + notchedOutline: isEditOrgTab + ? null + : classes.notchedOutline, + }, + }} + /> +
diff --git a/frontend/src/components/PartnerSettings.jsx b/frontend/src/components/PartnerSettings.jsx index a979c7d7..53ef9bee 100644 --- a/frontend/src/components/PartnerSettings.jsx +++ b/frontend/src/components/PartnerSettings.jsx @@ -54,7 +54,8 @@ const PartnerSettings = (props) => { "tech_partner": "#ff8544", "distribution_partner": "#2BC07E", "service_partner": "#a99cf9", - "integration_partner": "#fb47a0" + "integration_partner": "#fb47a0", + "channel_partner": "#4caf50", } const handleSendUpdateRequest = () => { @@ -85,6 +86,7 @@ const PartnerSettings = (props) => { { field: partnerData?.landscape_image_url, name: "Landscape Image" }, { field: partnerData?.website_url, name: "Website URL" }, { field: partnerData?.article_url, name: "Article URL" }, + { field: partnerData?.contact_email, name: "Contact Email" }, { field: partnerData?.country, name: "Country" }, { field: partnerData?.region, name: "Region" }, ]; @@ -129,6 +131,7 @@ const PartnerSettings = (props) => { description: partnerData.description?.trim(), website_url: partnerData.website_url?.trim(), article_url: partnerData.article_url?.trim(), + contact_email: partnerData.contact_email?.trim(), partner_type: partnerTypes, expertise: partnerData?.expertise || [], services: partnerData?.services || [], @@ -177,6 +180,7 @@ const PartnerSettings = (props) => { description: partnerData.description?.trim(), website_url: partnerData.website_url?.trim(), article_url: partnerData.article_url?.trim(), + contact_email: partnerData.contact_email?.trim(), partner_type: partnerTypes, usecases: partnerData?.usecases || [], expertise: partnerData?.expertise || [], diff --git a/frontend/src/components/PartnerTab.jsx b/frontend/src/components/PartnerTab.jsx index 10c21f06..4a17cba5 100644 --- a/frontend/src/components/PartnerTab.jsx +++ b/frontend/src/components/PartnerTab.jsx @@ -195,14 +195,17 @@ const PartnerTab = (props) => { // Enable the tab by default return false; } + + const isSupportOnlyTab = (tabName) => { + return tabName === "Apps" || tabName === "Articles" || tabName === "AI Agents"; + } return (
{tabsOnPartnerTab?.map((tabName, index) => ( -
+
+ {isSupportOnlyTab(tabName) && userdata?.support && ( +
+ S +
+ )}
))}
diff --git a/frontend/src/components/PartnersUsecasesTab.jsx b/frontend/src/components/PartnersUsecasesTab.jsx index 87e34b52..07866df9 100644 --- a/frontend/src/components/PartnersUsecasesTab.jsx +++ b/frontend/src/components/PartnersUsecasesTab.jsx @@ -29,9 +29,9 @@ import CloseIcon from "@mui/icons-material/Close"; import { toast } from "react-toastify"; import MoreVertIcon from "@mui/icons-material/MoreVert"; import EditIcon from "@mui/icons-material/Edit"; -import StarIcon from "@mui/icons-material/Star"; import DeleteIcon from "@mui/icons-material/Delete"; import AddCircleOutlineIcon from "@mui/icons-material/AddCircleOutline"; +import OpenInNewIcon from '@mui/icons-material/OpenInNew'; import DeleteOutlineIcon from "@mui/icons-material/DeleteOutline"; import KeyboardArrowUpIcon from "@mui/icons-material/KeyboardArrowUp"; import KeyboardArrowDownIcon from "@mui/icons-material/KeyboardArrowDown"; @@ -1083,16 +1083,40 @@ const PartnersUsecasesTab = ({ isCloud, globalUrl, userdata, partnerData, setPar }} > - - Public Workflow - + + Public Workflow + + { + // Open workflow in new tab - adjust URL as needed + window.open(`${window.location.origin}/workflows/${formData.mainContent.publicWorkflowId}`, "_blank"); + }} + > + + + { + setMaxExecutionCount(e.target.value) + submitSearch(workflowId, status, startTime, endTime, rowCursor, e.target.value, suborgWorkflowRuns) + }} + style={{ + height: 50, + backgroundColor: theme.palette.backgroundColor, + color: theme.palette.textColor, + }} + > + 10 + 25 + 50 + 100 + 200 + 500 + + +
{userdata?.active_org?.creator_org?.length === 0 ? (
@@ -956,7 +999,8 @@ const RuntimeDebugger = (props) => { setStartTime("") setEndTime("") setSearchQuery("") - submitSearch("", "", "", "", rowCursor, rowsPerPage, !suborgWorkflowRuns)} + setMaxExecutionCount(50) + submitSearch("", "", "", "", rowCursor, 50, !suborgWorkflowRuns)} } color="secondary" /> @@ -967,7 +1011,7 @@ const RuntimeDebugger = (props) => {
{ - submitSearch(workflowId, status, startTime, endTime, rowCursor, rowsPerPage, suborgWorkflowRuns) + submitSearch(workflowId, status, startTime, endTime, rowCursor, maxExecutionCount, suborgWorkflowRuns) }} style={{display: "flex", justifyContent: "center", alignItems: "center", }}> Status @@ -1158,7 +1202,8 @@ const RuntimeDebugger = (props) => { setEndTime("") setSearchQuery("") setSuborgWorkflowRuns(false) - submitSearch("", "", "", "", rowCursor, rowsPerPage, false) + setMaxExecutionCount(50) + submitSearch("", "", "", "", rowCursor, 50, false) }} > @@ -1169,7 +1214,7 @@ const RuntimeDebugger = (props) => { variant="outlined" color="primary" onClick={() => { - submitSearch(workflowId, status, startTime, endTime, rowCursor, rowsPerPage, suborgWorkflowRuns) + submitSearch(workflowId, status, startTime, endTime, rowCursor, maxExecutionCount, suborgWorkflowRuns) }} disabled={searchLoading} style={{height: 50, minWidth: 100, marginTop: 15, }} @@ -1181,20 +1226,18 @@ const RuntimeDebugger = (props) => { { - setRowsPerPage(newPageSize) - submitSearch(workflowId, status, startTime, endTime, rowCursor, newPageSize, suborgWorkflowRuns) + + onPaginationModelChange={(newPaginationModel) => { + setPaginationModel(newPaginationModel) + setRowsPerPage(newPaginationModel.pageSize) + // No API call needed - this is just for client-side pagination }} - // event for when clicking next page - // Hide page changer - onPageChange={(params) => { - console.log("params: ", params) - }} - onSelectionModelChange={(newSelection) => { + + onRowSelectionModelChange={(newSelection) => { //console.log("newSelection: ", newSelection) //setSelectedWorkflowExecutionsIndexes(newSelection) var found = [] diff --git a/frontend/src/components/ShuffleCodeEditor1.jsx b/frontend/src/components/ShuffleCodeEditor1.jsx index 043c5de2..1490d72e 100644 --- a/frontend/src/components/ShuffleCodeEditor1.jsx +++ b/frontend/src/components/ShuffleCodeEditor1.jsx @@ -2476,9 +2476,14 @@ const CodeEditor = (props) => { }} displayDataTypes={false} onSelect={(select) => { - //HandleJsonCopy(executionResult.result, select, "exec"); + var basename = "exec" + if (selectedAction !== undefined && selectedAction !== null && Object.keys(selectedAction).length !== 0) { + basename = selectedAction.label.toLowerCase().replaceAll(" ", "_") + } + + HandleJsonCopy(executionResult.result, select, basename) }} - name={"Test result"} + name={"Run Output"} /> : diff --git a/frontend/src/components/TenantsTab.jsx b/frontend/src/components/TenantsTab.jsx index ffd15912..16cd5f11 100644 --- a/frontend/src/components/TenantsTab.jsx +++ b/frontend/src/components/TenantsTab.jsx @@ -667,13 +667,12 @@ const TenantsTab = memo((props) => { const handleDeleteAccount = () => { const baseURL = globalUrl; - const url = `${baseURL}/api/v1/orgs/${selectedOrganization?.id}`; + const url = `${baseURL}/api/v1/orgs/${selectedSuborg?.id}`; const data = { - suborg_id : selectedSuborg?.id, password: password, - } - + }; + fetch(url, { mode: "cors", method: "DELETE", diff --git a/frontend/src/components/UserManagmentTab.jsx b/frontend/src/components/UserManagmentTab.jsx index 81b65b94..6293b3ce 100644 --- a/frontend/src/components/UserManagmentTab.jsx +++ b/frontend/src/components/UserManagmentTab.jsx @@ -1465,7 +1465,11 @@ const UserManagmentTab = memo((props) => { {isCloud ? ( )} + primary={( + userRegion ? ( + {data?.user_geo_info?.country?.iso_code} + ) : null + )} style={{ display: 'table-cell', verticalAlign: 'middle', textAlign: 'center' }} />) : null} { leads.push("distribution partner"); } + if (responseJson.lead_info.channel_partner) { + leads.push("channel partner"); + } + if (responseJson.lead_info.service_partner) { leads.push("service partner"); } diff --git a/frontend/src/views/AgentUI.jsx b/frontend/src/views/AgentUI.jsx index ebc5a558..f97f31f5 100644 --- a/frontend/src/views/AgentUI.jsx +++ b/frontend/src/views/AgentUI.jsx @@ -1,11 +1,14 @@ import React, { useState, useEffect, useContext, memo } from "react"; import { Context } from "../context/ContextApi.jsx"; +import { useNavigate, Link, useLocation } from "react-router-dom"; import { getTheme } from "../theme.jsx"; import { toast } from "react-toastify" import ReactJson from "react-json-view-ssr"; +import { v4 as uuidv4} from "uuid"; import { validateJson, collapseField, handleReactJsonClipboard, HandleJsonCopy } from "../views/Workflows.jsx"; import { + Box, Button, ButtonGroup, Typography, @@ -13,6 +16,7 @@ import { CircularProgress, Tooltip, IconButton, + TextField, } from '@mui/material' import { @@ -21,6 +25,8 @@ import { RestartAlt as RestartAltIcon, ExpandMore as ExpandMoreIcon, ExpandLess as ExpandLessIcon, + Send as SendIcon, + Error as ErrorIcon, } from '@mui/icons-material' import { @@ -39,9 +45,12 @@ const AgentUI = (props) => { const [originalStartTime, setOriginalStartTime] = useState(0) const [latestEndTime, setLatestEndTime] = useState(0) + const [showAgentStarter, setShowAgentStarter] = useState(false) + const [actionInput, setActionInput] = useState("") const {themeMode} = useContext(Context) const theme = getTheme(themeMode) + const navigate = useNavigate(); const agentWrapperStyle = { width: 1000, @@ -64,6 +73,10 @@ const AgentUI = (props) => { return } + if (node_id === undefined || node_id === null || node_id === "") { + return + } + var found = false for (var key in execution_data.results) { const item = execution_data.results[key] @@ -86,6 +99,16 @@ const AgentUI = (props) => { if (found === false) { toast.warn("Failed to find the relevant AI Agent result") + + if (execution_data?.results?.length === 1) { + setAgentActionResult(execution_data.results[0]) + const validatedData = validateJson(execution_data.results[0].result) + if (validatedData.valid) { + setData(validatedData.result) + } else { + toast.warn("Action output result is not valid JSON!") + } + } } } @@ -95,10 +118,10 @@ const AgentUI = (props) => { return } - if (node_id === undefined || node_id === null) { - toast.error("No node ID provided. Please provide node_id in the URL.") - return - } + //if (node_id === undefined || node_id === null || node_id === "") { + // toast.error("No node ID provided. Please provide node_id in the URL.") + // return + //} if (authorization === undefined || authorization === null) { toast.error("No authorization provided. Please provide authorization in the URL.") @@ -166,6 +189,7 @@ const AgentUI = (props) => { const url = `${globalUrl}/api/v1/apps/agent/run?rerun=true&decision_id=${decision?.run_details?.id}` var body = agentActionResult.action + console.log("BODY: ", body) body.source_execution = execution.execution_id body.source_workflow = execution.workflow.id @@ -202,10 +226,11 @@ const AgentUI = (props) => { const executionId = params.get("execution_id") const nodeId = params.get("node_id") const authorization = params.get("authorization") - if (executionId !== undefined && executionId !== null && nodeId !== undefined && nodeId !== null && authorization !== undefined && authorization !== null) { + if (executionId !== undefined && executionId !== null && authorization !== undefined && authorization !== null) { GetExecution(executionId, nodeId, authorization) } else { - toast.warn("No execution ID or node ID provided. Please provide execution_id and node_id in the URL.") + setShowAgentStarter(true) + //toast.warn("No execution ID or node ID provided. Please provide execution_id and node_id in the URL.") } }, []) @@ -221,6 +246,11 @@ const AgentUI = (props) => { + : + item.status === "ABORTED" || item.status === "FAILURE" ? + + + : @@ -246,11 +276,13 @@ const AgentUI = (props) => { const validate = validateJson(item.details) const itemStartTime = item.start_time var itemEndTime = item.end_time - if (itemStartTime !== undefined && (itemStartTime < originalStartTime || originalStartTime === 0)) { - setOriginalStartTime(itemStartTime) + if (itemStartTime !== undefined && itemStartTime !== originalStartTime && (itemStartTime < originalStartTime || originalStartTime === 0)) { + console.log("Rerender 1") + //setOriginalStartTime(itemStartTime) } if (itemEndTime !== undefined && itemEndTime > latestEndTime) { + console.log("Rerender 2") setLatestEndTime(itemEndTime) } @@ -280,23 +312,47 @@ const AgentUI = (props) => { borderTop: "1px solid " + theme.palette.surfaceColor, borderRadius: theme.palette.borderRadius, }} + onMouseEnter={() => { + if (!hovered) { + console.log("HOVER") + setHovered(true) + } + }} + onMouseLeave={() => { + if (hovered) { + setHovered(false) + } + }} >
setHovered(true)} - onMouseLeave={() => setHovered(false)} onClick={(e) => { if (item.details === undefined || item.details === null || item.details === "") { - toast("No details to open") + + if (item?.category === "agent" && item?.type === "agent") { + // Show all the data + if (openIndexes.includes(index)) { + console.log("Rerender 3") + setOpenIndexes(openIndexes.filter((i) => i !== index)) + } else { + console.log("Rerender 4") + setOpenIndexes([...openIndexes, index]) + } + } else { + toast.warn("No details to open") + } + return } if (openIndexes.includes(index)) { + console.log("Rerender 5") setOpenIndexes(openIndexes.filter((i) => i !== index)) } else { + console.log("Rerender 6") setOpenIndexes([...openIndexes, index]) } }} @@ -421,18 +477,31 @@ const AgentUI = (props) => { const TimelineRender = (props) => { const { agent_data } = props; + + const actionResult = execution?.results?.length > 0 ? execution.results[0] : execution var timelineItems = [ { "label": "AI Agent 2", "type": "agent", "category": "agent", + "details": actionResult?.result, - "status": agent_data.status, - "start_time": agent_data.started_at, - "end_time": agent_data.completed_at, + "status": agent_data?.status, + "start_time": agent_data?.started_at, + "end_time": agent_data?.completed_at, }, ] + // Autofixer for result lol + if ((agent_data?.decisions === undefined || agent_data?.decisions === null)) { + const verifiedInput = validateJson(actionResult?.result) + if (verifiedInput.valid === true && verifiedInput.result?.decisions !== undefined && verifiedInput.result?.decisions !== null) { + agent_data.decisions = verifiedInput.result?.decisions + + setAgentActionResult(actionResult) + } + } + var sortedTimelineItems = [] for (var key in agent_data?.decisions) { const item = agent_data.decisions[key] @@ -501,39 +570,136 @@ const AgentUI = (props) => { ) } + const submitInput = (inputText) => { + toast.info("Submitting AI Agent input: " + inputText); + + //setShowAgentStarter(false); + //GetExecution(execution?.execution_id, execution?.node_id, execution?.authorization); + + if (inputText === undefined || inputText === null || inputText === "") { + toast.error("Please provide a valid input for the AI Agent.") + return + } + + // 1. Run the execution. Can this be a single-action run? + // 2. Get the execution ID and node ID from the response. + const uuid = uuidv4() + const data = { + "id": uuid, + "name":"agent", + //"app_name":"Shuffle AI", + "app_name":"AI Agent", // Failover for rerun + "app_id":"shuffle_agent", + "app_version":"1.0.0", + + "environment":"cloud", + "parameters":[ + { + "name":"app_name", + "value":"openai" + }, + { + "name":"input", + "value": inputText + }, + { + "name":"action", + "value":"list_tickets" + } + ]} + + const url = `${globalUrl}/api/v1/apps/agent_starter/run` + fetch(url, { + method: "POST", + body: JSON.stringify(data), + credentials: "include", + }) + .then((response) => { + return response.json() + }) + .then((responseJson) => { + toast.success("Got response!") + console.log("Agent run response: ", responseJson) + + if (responseJson.success === true && responseJson.authorization !== undefined && responseJson.execution_id !== undefined) { + navigate("?execution_id=" + responseJson.execution_id + "&authorization=" + responseJson.authorization) + setShowAgentStarter(false) + GetExecution(responseJson.execution_id, "", responseJson.authorization) + } + }) + .catch((error) => { + toast.error("Error: " + error) + }) + + } + return (
- {/* - - Agent Input: {data.input} - - */} - - - - + {showAgentStarter ? + { + e.preventDefault(); + submitInput(actionInput); + }}> + +
- {buttonState === "timeline" ? - - : - null + + Shuffle AI Agents + + { + setActionInput(e.target.value) + }} + InputProps={{ + endAdornment: ( + + + + + + ), + }} + /> + + : +
+ + + + + + {buttonState === "timeline" ? + + : + null + } +
}
) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 0466fb85..cd1be3dd 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -401,7 +401,6 @@ export function SetJsonDotnotation(jsonInput, inputKey) { //export const green = "#86c142"; export const green = "#02CB70" export const yellow = "#FECC00"; -//export const red = "#ff3632"; export const red = "#F53434"; export const grey = "#b0b0b0"; @@ -729,6 +728,8 @@ const AngularWorkflow = (defaultprops) => { "List tickets", "Send Email", "Get specific ticket", + "Update ticket", + "Add ticket comment", ], "multiselect": true, }, @@ -2258,7 +2259,7 @@ const AngularWorkflow = (defaultprops) => { currentnode.removeClass("shuffle-hover-highlight"); currentnode.removeClass("awaiting-data-highlight"); currentnode.addClass("success-highlight"); - incomingEdges.addClass("success-highlight"); + outgoingEdges.addClass("success-highlight"); if (visited !== undefined && visited !== null && !visited.includes(label)) { @@ -5368,7 +5369,8 @@ const AngularWorkflow = (defaultprops) => { if (!branchFound) { var relevantNodes = [] - const minDistance = 185 + //const minDistance = 185 + const minDistance = 85 const draggedNode = event.target const allnodes = cy.nodes().jsons() for (var nodekey in allnodes) { @@ -5398,7 +5400,7 @@ const AngularWorkflow = (defaultprops) => { if (decoratorNodeIds.includes(node.data.id)) { // Drag a little farther to remove it - if (distance > minDistance + 75) { + if (distance > minDistance + 125) { // Remove the branch? Why? const edgeToRemove = cy.getElementById(branches[branchkey].data.id) if (edgeToRemove !== null && edgeToRemove !== undefined) { @@ -18187,8 +18189,8 @@ const AngularWorkflow = (defaultprops) => {
- const defaultEnvironment = environments && environments?.find( - (env) => env?.default && env?.Name?.toLowerCase() !== "cloud" + const defaultEnvironment = environments.find( + (env) => env.default && env.Name.toLowerCase() !== "cloud" ); if (selectedTrigger.trigger_type === "PIPELINE" && selectedTrigger.environment === "onprem" && defaultEnvironment !== undefined) { @@ -20997,7 +20999,10 @@ const AngularWorkflow = (defaultprops) => { */} - {userdata.support === true || (userdata.avatar !== undefined && (userdata.avatar === creatorProfile.github_avatar || allowList.includes(userdata.public_username))) ? + {userdata.support || + (userdata.avatar !== undefined && (userdata.avatar === creatorProfile.github_avatar || allowList.includes(userdata.public_username))) || + (workflow?.owner?.length > 0 && workflow.owner === userdata?.active_org?.id && userdata?.active_org.role === "admin") || + (workflow?.owner?.length > 0 && workflow.owner === userdata?.id)?
You can see these buttons because you may have the correct access rights as a creator to help modify this workflow. @@ -22738,7 +22743,7 @@ const AngularWorkflow = (defaultprops) => { style={{ zIndex: 50000 }} > 0 ? yellow : theme.palette.textColor, + color: relevant_errors.length > 0 ? red : theme.palette.textColor, }} /> @@ -22783,6 +22788,26 @@ const AngularWorkflow = (defaultprops) => { : null} + {data?.action?.name === "run_schemaless" || data?.action?.name === "run_singul" || data?.action?.name === "singul" && data?.action?.parameters?.length > 4 ? + + : null} + {data.action.app_name === "shuffle-subflow" && validate.result.success !== undefined && validate.result.success === true ? ( @@ -22944,7 +22969,7 @@ const AngularWorkflow = (defaultprops) => { > Action Logs - + More log details for this action are not available without an onprem environment with the SHUFFLE_LOGS_DISABLED environment variable set to false: SHUFFLE_LOGS_DISABLED=false. Logs are enabled by default, except in scale mode.
@@ -23002,7 +23027,9 @@ const AngularWorkflow = (defaultprops) => { variant="body1" style={{}} > - {data.name}: {showVariable ? data.value : null} + {data.name}: + {showVariable ? data.value : null} + } {open ? @@ -23024,8 +23051,8 @@ const AngularWorkflow = (defaultprops) => { { @@ -23035,7 +23062,7 @@ const AngularWorkflow = (defaultprops) => { window.open(data.value, "_blank") } }} - color={showlink ? "inherit" : "textSecondary"} + color={showlink ? "#ff8544" : "textSecondary"} > {data.value} @@ -23199,8 +23226,9 @@ const AngularWorkflow = (defaultprops) => { sx: { pointerEvents: "auto", color: theme.palette.text.primary, - minWidth: isMobile ? "90%" : "750px", - maxHeight: "550px", + minWidth: isMobile ? "90%" : 900, + minHeight: 500, + maxHeight: 650, overflowY: "auto", overflowX: "hidden", border: theme.palette.defaultBorder, @@ -23418,7 +23446,9 @@ const AngularWorkflow = (defaultprops) => { > {selectedResult.action.label.replaceAll("_", " ")} -
{selectedResult.action.name}
+ + {selectedResult.action.name} + @@ -25588,12 +25618,14 @@ const AngularWorkflow = (defaultprops) => { const actionIndex = workflow?.actions.findIndex(action => action.id === actionId); if (actionIndex >= 0) { // Find the parameter with matching name - console.log("fieldName", fieldName) const paramIndex = workflow.actions[actionIndex].parameters.findIndex(param => param.name === fieldName); if (paramIndex >= 0) { // Update the parameter value workflow.actions[actionIndex].parameters[paramIndex].value = newData; - + if(selectedAction !== undefined && selectedAction !== null && selectedAction.id === actionId) { + selectedAction.parameters[paramIndex].value = newData; + setSelectedAction(selectedAction) + } // Update workflow state to trigger re-render setWorkflow({...workflow}); setLastSaved(false); diff --git a/frontend/src/views/AppExplorer.jsx b/frontend/src/views/AppExplorer.jsx index f0069e4f..f0f70021 100644 --- a/frontend/src/views/AppExplorer.jsx +++ b/frontend/src/views/AppExplorer.jsx @@ -233,6 +233,7 @@ const buttonBackground = "linear-gradient(to right, #f86a3e, #f34079)"; const [secondaryApp, setSecondaryApp] = useState({}); const [firstRequest, setFirstRequest] = useState(true); const [publishModalOpen, setPublishModalOpen] = React.useState(false); + const [showDistributionPopup, setShowDistributionPopup] = React.useState(false); const [categories, setCategories] = useState(appCategories) const [newWorkflowCategories, setNewWorkflowCategories] = React.useState([]); @@ -743,7 +744,7 @@ const buttonBackground = "linear-gradient(to right, #f86a3e, #f34079)"; } }; - const activateApp = (action) => { + const activateApp = (action, org_id, multiple_request = false) => { if (serverside === true) { return } @@ -753,11 +754,14 @@ const buttonBackground = "linear-gradient(to right, #f86a3e, #f34079)"; if (action !== undefined && action !== null) { url = `${globalUrl}/api/v1/apps/${appId}/${action}` } - fetch(url, { + + + fetch(url, { method: "GET", headers: { "Content-Type": "application/json", - Accept: "application/json", + "Accept": "application/json", + "Org-Id": org_id !== undefined && org_id !== null && org_id?.length > 0 ? org_id : userdata.active_org.id }, credentials: "include", }) @@ -784,7 +788,7 @@ const buttonBackground = "linear-gradient(to right, #f86a3e, #f34079)"; } } } else { - if (checkLogin !== undefined && checkLogin !== null) { + if ((checkLogin !== undefined && checkLogin !== null) && !multiple_request) { checkLogin() } @@ -795,6 +799,9 @@ const buttonBackground = "linear-gradient(to right, #f86a3e, #f34079)"; toast("App activated for your organization!") } } else { + if (responseJson.success && !multiple_request &&(responseJson.reason !== undefined || responseJson.reason !== null)) { + toast.success(`${responseJson.reason}`); + } } } }) @@ -3909,6 +3916,212 @@ const buttonBackground = "linear-gradient(to right, #f86a3e, #f34079)"; ) : null; + + const handleActivateApp = (id, action) => { + if (action === "activate_all") { + const childOrgs = userdata.orgs.filter( + (data) => data.creator_org === userdata.active_org.id + ); + const orgIds = childOrgs.map((data) => data.id); + // run app activation requset for each org + orgIds.forEach((orgId) => { + activateApp("activate", orgId, true) + }); + setTimeout(() => { + toast.success("App activated for all sub-orgs"); + }, 5000); + } else if (action === "deactivate_all") { + const childOrgs = userdata.orgs.filter( + (data) => data.creator_org === userdata.active_org.id + ); + const orgIds = childOrgs.map((data) => data.id); + // run app deactivation request for each org + orgIds.forEach((orgId) => { + activateApp("deactivate", orgId, true) + }); + + setTimeout(() => { + toast.success("App deactivated for all sub-orgs"); + }, 5000); + + } else if (action === "activate_single") { + if (id === null) { + toast.error("Please select a sub-org to activate the app for."); + return; + } + activateApp("activate", id); + } else if (action === "deactivate_single") { + if (id === null) { + toast.error("Please select a sub-org to deactivate the app for."); + return; + } + activateApp("deactivate", id); + } + }; + + + + + const appDistributinModal = showDistributionPopup ? ( + setShowDistributionPopup(false)} + PaperProps={{ + sx: { + borderRadius: theme?.palette?.DialogStyle?.borderRadius || 3, + border: theme?.palette?.DialogStyle?.border, + fontFamily: theme?.typography?.fontFamily, + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + zIndex: 1000, + minWidth: "600px", + minHeight: "320px", + overflow: "auto", + '& .MuiDialogContent-root': { + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + }, + '& .MuiDialogTitle-root': { + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + p: 2, + }, + '& .MuiDialogActions-root': { + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + }, + }, + }} + > + + + Select sub-org to distribute App + + setShowDistributionPopup(false)} + sx={{ + color: theme.palette.text.primary, + }} + > + + + + + + handleActivateApp(null, "deactivate_all")} + sx={{ + borderRadius: 1, + px: 2, + '&:hover': { + backgroundColor: 'rgba(255,255,255,0.08)', + }, + }} + > + Deactivate for all suborgs + + + handleActivateApp(null, "activate_all")} + sx={{ + borderRadius: 1, + px: 2, + '&:hover': { + backgroundColor: 'rgba(255,255,255,0.08)', + }, + }} + > + Activate for all suborgs + + + {userdata.orgs.map((data, index) => { + if (data.creator_org !== userdata.active_org.id) return null; + + const imageSize = 28; + const imageStyle = { + width: imageSize, + height: imageSize, + borderRadius: '50%', + objectFit: 'cover', + marginRight: 12, + }; + + const imageSrc = data.image || theme.palette.defaultImage; + + return ( + + + {data.name} + + {data.name} + + + + + + + + ); + })} + + +) : null; + + const landingpageDataBrowser = (
{publishModal} + {appDistributinModal}
{isMobile ? null : ( } - {isMobile || app?.reference_org === userdata?.active_org?.id ? null : ( + {isMobile || app?.reference_org === userdata?.active_org?.id || (app?.suborg_distribution?.includes(userdata?.active_org?.id)) ? null : ( + )}
) : ( { // What data do we fill in here? Idk const Dashboard = (props) => { - const { globalUrl, isLoggedIn } = props; + const { globalUrl, userdata, isLoggedIn } = props; //const alert = useAlert(); const [bigChartData, setBgChartData] = useState("data1"); const [dayAmount, setDayAmount] = useState(7); @@ -323,10 +323,10 @@ const Dashboard = (props) => { const [stats, setStats] = useState({}); const [changeme, setChangeme] = useState(""); const [statsRan, setStatsRan] = useState(false); - const [keys, setKeys] = useState([]) - const [treeKeys, setTreeKeys] = useState([]) + const [keys, setKeys] = useState([]) + const [treeKeys, setTreeKeys] = useState([]) - const [selectedUsecaseCategory, setSelectedUsecaseCategory] = useState(""); + const [selectedUsecaseCategory, setSelectedUsecaseCategory] = useState("") const [selectedUsecases, setSelectedUsecases] = useState([]); const [usecases, setUsecases] = useState([]); const [workflows, setWorkflows] = useState([]); @@ -750,8 +750,6 @@ const Dashboard = (props) => { const newname = data.key !== undefined ? data.key.replaceAll("_", " ") : "" - console.log("KEYDATA: ", data) - const loadNewStats = (newkey) => { const resp = LoadStats(globalUrl, newkey) if (resp !== undefined) { @@ -807,7 +805,7 @@ const Dashboard = (props) => { backgroundColor: theme.palette.inputColor, color: "white", height: 40, - maxWidth: 150, + maxWidth: 200, borderRadius: theme.palette?.borderRadius, }} > @@ -829,10 +827,12 @@ const Dashboard = (props) => { }
+ + ) @@ -848,14 +848,6 @@ const Dashboard = (props) => { : null} - {/*widgetData === undefined || widgetData === null || widgetData === [] || widgetData.length === 0 ? null : - - - - - - */} - {newWidgetData === undefined || newWidgetData === null || newWidgetData === [] ? null : newWidgetData.map((data, index) => { @@ -872,7 +864,9 @@ const Dashboard = (props) => { ); const dataWrapper = ( -
{data}
+
+ {data} +
); return dataWrapper; diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 3fe551f4..32859549 100755 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -558,7 +558,7 @@ export const handleReactJsonClipboard = (copy) => { document.execCommand("copy"); console.log("COPYING!"); - toast("Copied value to clipboard, NOT json path.") + toast.success("Copied Value, NOT json path.") } else { console.log("Failed to copy from " + elementName + ": ", copyText); } From b2dfe8d8f991cdaab0a839f2f9c3608d4f2fb522 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Mon, 14 Jul 2025 20:33:50 +0530 Subject: [PATCH 12/59] added cron support --- backend/go-app/go.mod | 3 +++ backend/go-app/main.go | 23 ++++++++++++++----- backend/go-app/walkoff.go | 48 +++++++++++++++++++++++++++------------ 3 files changed, 54 insertions(+), 20 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 3c21ac4c..d76f453f 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -74,6 +74,7 @@ require ( github.com/felixge/httpsnoop v1.0.4 // indirect github.com/frikky/schemaless v0.0.17 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-co-op/gocron v1.37.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-jose/go-jose/v4 v4.0.5 // indirect github.com/go-logr/logr v1.4.2 // indirect @@ -118,6 +119,7 @@ require ( github.com/pjbgf/sha1cd v0.3.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect github.com/sashabaranov/go-openai v1.40.1 // indirect github.com/sendgrid/rest v2.6.9+incompatible // indirect github.com/sendgrid/sendgrid-go v3.16.1+incompatible // indirect @@ -141,6 +143,7 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect go.opentelemetry.io/otel/trace v1.36.0 // indirect go.opentelemetry.io/proto/otlp v1.5.0 // indirect + go.uber.org/atomic v1.9.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/net v0.40.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect diff --git a/backend/go-app/main.go b/backend/go-app/main.go index fdd9dc05..3e8a024e 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -4034,6 +4034,7 @@ func runInitEs(ctx context.Context) { // FIXME: This should ONLY run on one backend instance schedules, err := shuffle.GetAllSchedules(ctx, "ALL") + log.Printf("Schedules %s", schedules) if err != nil { log.Printf("[WARNING] Failed getting schedules during service init: %s", err) } else { @@ -4078,14 +4079,23 @@ func runInitEs(ctx context.Context) { //log.Printf("Schedule: %#v", schedule) //log.Printf("Schedule time: every %d seconds", schedule.Seconds) - jobret, err := newscheduler.Every(schedule.Seconds).Seconds().NotImmediately().Run(job(schedule)) - if err != nil { - log.Printf("[ERROR] Failed to start schedule for workflow %s: %s", schedule.WorkflowId, err) + if schedule.Seconds == 0 && len(schedule.Frequency) > 0 { + cronJob, err := CronScheduler.Cron(schedule.Frequency).Do(job(schedule)) + if err != nil { + log.Printf("[ERROR] Failed to start schedule for workflow %s: %s", schedule.WorkflowId, err) + } else { + log.Printf("[DEBUG] Successfully started schedule for workflow %s", schedule.WorkflowId) + } + cronJobs[schedule.Id] = cronJob } else { - log.Printf("[DEBUG] Successfully started schedule for workflow %s", schedule.WorkflowId) + jobret, err := newscheduler.Every(schedule.Seconds).Seconds().NotImmediately().Run(job(schedule)) + if err != nil { + log.Printf("[ERROR] Failed to start schedule for workflow %s: %s", schedule.WorkflowId, err) + } else { + log.Printf("[DEBUG] Successfully started schedule for workflow %s", schedule.WorkflowId) + } + scheduledJobs[schedule.Id] = jobret } - - scheduledJobs[schedule.Id] = jobret } } @@ -5090,6 +5100,7 @@ func handleAppZipUpload(resp http.ResponseWriter, request *http.Request) { func initHandlers() { var err error ctx := context.Background() + CronScheduler.StartAsync() log.Printf("[DEBUG] Starting Shuffle backend - initializing database connection") //requestCache = cache.New(5*time.Minute, 10*time.Minute) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index f2d8a118..eb174870 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -26,6 +26,7 @@ import ( uuid "github.com/satori/go.uuid" newscheduler "github.com/carlescere/scheduler" + "github.com/go-co-op/gocron" "github.com/frikky/kin-openapi/openapi3" "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" @@ -42,17 +43,22 @@ var baseEnvironment = "onprem" var cloudname = "cloud" var scheduledJobs = map[string]*newscheduler.Job{} +var cronJobs = map[string]*gocron.Job{} var scheduledOrgs = map[string]*newscheduler.Job{} +var CronScheduler = gocron.NewScheduler(time.UTC) + // Frequency = cronjob OR minutes between execution func createSchedule(ctx context.Context, scheduleId, workflowId, name, startNode, frequency, orgId string, body []byte) error { var err error testSplit := strings.Split(frequency, "*") cronJob := "" + isCron := false newfrequency := 0 if len(testSplit) > 5 { cronJob = frequency + isCron = true } else { newfrequency, err = strconv.Atoi(frequency) if err != nil { @@ -65,12 +71,7 @@ func createSchedule(ctx context.Context, scheduleId, workflowId, name, startNode //} else if int(newfrequency) < } - // Reverse. Can't handle CRON, only numbers - if len(cronJob) > 0 { - return errors.New("cronJob isn't formatted correctly") - } - - if newfrequency < 1 { + if newfrequency < 1 && !isCron { return errors.New("Frequency has to be more than 0") } @@ -96,17 +97,27 @@ func createSchedule(ctx context.Context, scheduleId, workflowId, name, startNode } } - log.Printf("[INFO] Starting frequency for execution: %d", newfrequency) + log.Printf("[INFO] Starting frequency for execution: %s", frequency) - //jobret, err := newscheduler.Every(newfrequency).Seconds().NotImmediately().Run(job) - jobret, err := newscheduler.Every(newfrequency).Seconds().Run(job) - if err != nil { - log.Printf("Failed to schedule workflow: %s", err) - return err + if isCron { + cronJob, err := CronScheduler.Cron(cronJob).Do(job) + if err != nil { + log.Printf("[ERROR] Failed to start schedule with cron(%s): %s", cronJob, err) + } + + cronJobs[scheduleId] = cronJob + } else { + //jobret, err := newscheduler.Every(newfrequency).Seconds().NotImmediately().Run(job) + jobret, err := newscheduler.Every(newfrequency).Seconds().Run(job) + if err != nil { + log.Printf("Failed to schedule workflow: %s", err) + return err + } + + scheduledJobs[scheduleId] = jobret } //scheduledJobs = append(scheduledJobs, jobret) - scheduledJobs[scheduleId] = jobret // Doesn't need running/not running. If stopped, we just delete it. timeNow := int64(time.Now().Unix()) @@ -117,6 +128,7 @@ func createSchedule(ctx context.Context, scheduleId, workflowId, name, startNode Argument: string(body), WrappedArgument: bodyWrapper, Seconds: newfrequency, + Frequency: frequency, CreationTime: timeNow, LastModificationtime: timeNow, LastRuntime: timeNow, @@ -1541,7 +1553,15 @@ func deleteSchedule(ctx context.Context, id string) error { value.Lock() } else { // FIXME - allow it to kind of stop anyway? - return errors.New("Can't find the schedule.") + if j, ok := cronJobs[id]; ok { + err := CronScheduler.RemoveByID(j) + if err != nil { + log.Printf("[ERROR] Failed to remove the scheduler %s", err) + return err + } + } else { + return errors.New("Can't find the schedule.") + } } } From b798b7a5874caf09cc77782f03bb6e254bd8c191 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Mon, 14 Jul 2025 20:42:41 +0530 Subject: [PATCH 13/59] remove a log --- backend/go-app/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 3e8a024e..40cc3ace 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -4034,7 +4034,6 @@ func runInitEs(ctx context.Context) { // FIXME: This should ONLY run on one backend instance schedules, err := shuffle.GetAllSchedules(ctx, "ALL") - log.Printf("Schedules %s", schedules) if err != nil { log.Printf("[WARNING] Failed getting schedules during service init: %s", err) } else { From cb100292c92eef914e0c0af9fc7dbee1c683dd3f Mon Sep 17 00:00:00 2001 From: Frikky Date: Tue, 15 Jul 2025 10:45:30 +0200 Subject: [PATCH 14/59] Cloud sync --- frontend/src/components/LeftSideBar.jsx | 2 +- frontend/src/views/RunWorkflow.jsx | 4 +++- frontend/src/views/Workflows2.jsx | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/frontend/src/components/LeftSideBar.jsx b/frontend/src/components/LeftSideBar.jsx index eeae20bc..07c708a0 100644 --- a/frontend/src/components/LeftSideBar.jsx +++ b/frontend/src/components/LeftSideBar.jsx @@ -578,7 +578,7 @@ const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, }) => { }, }} > - {userdata && (userdata?.org_status?.includes("integration_partner") && userdata?.org_status?.includes("sub_org")) ? null : ( + {userdata && (userdata?.org_status?.includes("integration_partner") && userdata?.active_org?.role !== "admin") ? null : ( <> { const { globalUrl, userdata, isLoaded, isLoggedIn, setIsLoggedIn, setCookie, register, serverside } = defaultprops; + const { themeMode, brandColor } = useContext(Context); + const theme = getTheme(themeMode, brandColor); const { supportEmail } = useContext(Context); let navigate = useNavigate(); diff --git a/frontend/src/views/Workflows2.jsx b/frontend/src/views/Workflows2.jsx index d702f24f..65f55b2a 100644 --- a/frontend/src/views/Workflows2.jsx +++ b/frontend/src/views/Workflows2.jsx @@ -2695,7 +2695,7 @@ const Workflows2 = (props) => { style={{ height: 24, width: 24, - filter: themeMode === "dark" ? "brightness(0.6)" : "brightness(0.9)", + filter: themeMode === "dark" ? "brightness(0.6)" : "brightness(1)", cursor: "pointer", }} onClick={() => { @@ -2716,7 +2716,7 @@ const Workflows2 = (props) => { style={{ height: 24, width: 24, - filter: "brightness(0.6)", + filter: themeMode === "dark" ? "brightness(0.6)" : "brightness(1)", cursor: "pointer", }} onClick={() => { From 772d8699b66f12a345d29915aaa111d72102c64e Mon Sep 17 00:00:00 2001 From: Frikky Date: Tue, 15 Jul 2025 10:55:55 +0200 Subject: [PATCH 15/59] Added singul endpoints --- backend/go-app/go.mod | 12 +++++------- backend/go-app/go.sum | 29 +++++++++++++++++++++-------- backend/go-app/main.go | 3 +++ 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index d76f453f..98a348bd 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -18,12 +18,14 @@ require ( github.com/frikky/kin-openapi v0.42.0 github.com/fsouza/go-dockerclient v1.12.1 github.com/ghodss/yaml v1.0.0 + github.com/go-co-op/gocron v1.37.0 github.com/go-git/go-billy/v5 v5.6.2 github.com/go-git/go-git/v5 v5.16.1 github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.88 + github.com/shuffle/shuffle-shared v0.8.92 + github.com/shuffle/singul v0.0.15 golang.org/x/crypto v0.38.0 google.golang.org/api v0.236.0 google.golang.org/grpc v1.72.2 @@ -55,7 +57,6 @@ require ( github.com/bitly/go-simplejson v0.5.1 // indirect github.com/bradfitz/gomemcache v0.0.0-20250403215159-8d39553ac7cf // indirect github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 // indirect - github.com/cenkalti/backoff/v5 v5.0.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudflare/circl v1.6.1 // indirect github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 // indirect @@ -72,9 +73,8 @@ require ( github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/frikky/schemaless v0.0.17 // indirect + github.com/frikky/schemaless v0.0.18 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect - github.com/go-co-op/gocron v1.37.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-jose/go-jose/v4 v4.0.5 // indirect github.com/go-logr/logr v1.4.2 // indirect @@ -127,7 +127,7 @@ require ( github.com/sirupsen/logrus v1.9.3 // indirect github.com/skeema/knownhosts v1.3.1 // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/pflag v1.0.6 // indirect github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect @@ -137,12 +137,10 @@ require ( go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect go.opentelemetry.io/otel v1.36.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 // indirect go.opentelemetry.io/otel/metric v1.36.0 // indirect go.opentelemetry.io/otel/sdk v1.36.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect go.opentelemetry.io/otel/trace v1.36.0 // indirect - go.opentelemetry.io/proto/otlp v1.5.0 // indirect go.uber.org/atomic v1.9.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/net v0.40.0 // indirect diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 098db5c0..51d36d6e 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -152,8 +152,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0= github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4= -github.com/frikky/schemaless v0.0.16 h1:4d2ZktB9xGsAusbbKliOI8TuriSrdIMzD/6ToY3wkz8= -github.com/frikky/schemaless v0.0.16/go.mod h1:jT48kTcmr1q3o8i+8qe7g+eCsbwaz2Q9CjOJevQQzQs= +github.com/frikky/schemaless v0.0.18 h1:w2kMDLGZD+1sJOQx5EycW/8Z5wo1p3JlV4kEFp1kWTY= +github.com/frikky/schemaless v0.0.18/go.mod h1:jT48kTcmr1q3o8i+8qe7g+eCsbwaz2Q9CjOJevQQzQs= github.com/fsouza/go-dockerclient v1.12.1 h1:FMoLq+Zhv9Oz/rFmu6JWkImfr6CBgZOPcL+bHW4gS0o= github.com/fsouza/go-dockerclient v1.12.1/go.mod h1:OqsgJJcpCwqyM3JED7TdfM9QVWS5O7jSYwXxYKmOooY= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= @@ -162,6 +162,8 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= +github.com/go-co-op/gocron v1.37.0 h1:ZYDJGtQ4OMhTLKOKMIch+/CY70Brbb1dGdooLEhh7b0= +github.com/go-co-op/gocron v1.37.0/go.mod h1:3L/n6BkO7ABj+TrfSVXLRzsP26zmikL4ISkLQ0O8iNY= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= github.com/go-git/go-billy/v5 v5.6.2 h1:6Q86EsPXMa7c3YZ3aLAQsMA0VlWmy43r6FHqa/UNbRM= @@ -243,6 +245,7 @@ github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAx github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= @@ -278,6 +281,7 @@ github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zt github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -329,6 +333,7 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= github.com/pjbgf/sha1cd v0.3.2/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= @@ -336,7 +341,11 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= @@ -350,8 +359,10 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.84 h1:ElIMQYjKBVOiadbiGkSzt/lPU5xqaQwRxQvk9wx/xYM= -github.com/shuffle/shuffle-shared v0.8.84/go.mod h1:RdfNxqCPI+zU4jQKy3E/p4Io2injm7LpSKQUCDHNtLk= +github.com/shuffle/shuffle-shared v0.8.91 h1:ZcxSKhns96w1GQG/EjEJSLBvuEA3KVjvORAetV3W4bM= +github.com/shuffle/shuffle-shared v0.8.91/go.mod h1:PFLG4eaxz8zQc9Lbc0Ok/nG8rNsaBkZ0TNar+VU+J/g= +github.com/shuffle/singul v0.0.15 h1:a5Qro1BKs9a+mWE289oYTkDeVFoKEd/H9qu9XJGsVxo= +github.com/shuffle/singul v0.0.15/go.mod h1:z1RRZsymTYxsB8WMQwwswypruy4j5EugGPFEZspTEA4= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -359,8 +370,8 @@ github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnB github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -417,8 +428,10 @@ go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFw go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= -go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= -go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= +go.opentelemetry.io/proto/otlp v1.6.0 h1:jQjP+AQyTf+Fe7OKj/MfkDrmK4MNVtw2NpXsf9fefDI= +go.opentelemetry.io/proto/otlp v1.6.0/go.mod h1:cicgGehlFuNdgZkcALOCh3VE6K/u2tAjzlRhDwmVpZc= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 40cc3ace..6f9d5dc7 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3,6 +3,7 @@ package main import ( uuid "github.com/satori/go.uuid" "github.com/shuffle/shuffle-shared" + "github.com/shuffle/singul/pkg" "archive/zip" "bufio" @@ -5191,6 +5192,8 @@ func initHandlers() { r.HandleFunc("/api/v1/apps/{key}/execute", executeSingleAction).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/{key}/run", executeSingleAction).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/categories", shuffle.GetActiveCategories).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/apps/categories/run", singul.RunCategoryAction).Methods("POST", "OPTIONS") + //r.HandleFunc("/api/v1/apps/categories/run", shuffle.RunCategoryAction).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/upload", handleAppZipUpload).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/{appId}/activate", activateWorkflowAppDocker).Methods("GET", "OPTIONS") From af9c69a2f35032dc5e18b1e96d1781edf82cc893 Mon Sep 17 00:00:00 2001 From: "lalitdeore12@gmail.com" Date: Mon, 21 Jul 2025 13:42:52 +0530 Subject: [PATCH 16/59] Fix the branding issue for partners --- backend/go-app/main.go | 67 ++++++++++++++++++++++++++++++++++++++++++ frontend/src/App.jsx | 45 ++++++++++++++++++++++++---- 2 files changed, 106 insertions(+), 6 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 6f9d5dc7..4e4cce58 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -1069,6 +1069,72 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { activatedAppIds = append(activatedAppIds, app.ID) } + parsedStatus := []string{} + if len(org.ManagerOrgs) > 0 || userInfo.ActiveOrg.CreatorOrg != "" { + parsedStatus = append(parsedStatus, "sub_org") + + parentOrgId := userInfo.ActiveOrg.CreatorOrg + if len(userInfo.ActiveOrg.CreatorOrg) == 0 { + parentOrgId = org.ManagerOrgs[0].Id + } + + // Check for licensing/branding of parent and override + parentOrg, err := shuffle.GetOrg(ctx, parentOrgId) + if err == nil { + if parentOrg.LeadInfo.IntegrationPartner { + parsedStatus = append(parsedStatus, "integration_partner") + + // except theme take from parent org + org.Branding.EnableChat = parentOrg.Branding.EnableChat + org.Branding.HomeUrl = parentOrg.Branding.HomeUrl + org.Branding.DocumentationLink = parentOrg.Defaults.DocumentationReference + org.Branding.SupportEmail = parentOrg.Branding.SupportEmail + org.Branding.LogoutUrl = parentOrg.Branding.LogoutUrl + org.Branding.BrandColor = parentOrg.Branding.BrandColor + org.Branding.BrandName = parentOrg.Branding.BrandName + org.Branding.GlobalUser = parentOrg.Branding.GlobalUser + + if len(org.Branding.Theme) == 0 { + org.Branding.Theme = parentOrg.Branding.Theme + } + + userInfo.ActiveOrg.Branding = parentOrg.Branding + userInfo.ActiveOrg.Image = parentOrg.Image + userInfo.ActiveOrg.Branding.DocumentationLink = parentOrg.Defaults.DocumentationReference + userInfo.ActiveOrg.Branding.BrandColor = parentOrg.Branding.BrandColor + userInfo.ActiveOrg.Branding.SupportEmail = org.Branding.SupportEmail + userInfo.ActiveOrg.Branding.LogoutUrl = org.Branding.LogoutUrl + userInfo.ActiveOrg.Branding.BrandName = parentOrg.Branding.BrandName + + if len(org.Branding.Theme) == 0 { + userInfo.ActiveOrg.Branding.Theme = parentOrg.Branding.Theme + } else { + userInfo.ActiveOrg.Branding.Theme = org.Branding.Theme + } + + // check whether current is global user or not? means is user part of parent org or not + for _, user := range parentOrg.Users { + if user.Id == userInfo.Id && user.Role == "admin" { + userInfo.ActiveOrg.Branding.GlobalUser = true + break + } + } + } + } + } else { + // for parent org branding + if org.LeadInfo.IntegrationPartner { + userInfo.ActiveOrg.Branding.Theme = org.Branding.Theme + userInfo.ActiveOrg.Branding.DocumentationLink = org.Defaults.DocumentationReference + userInfo.ActiveOrg.Branding.SupportEmail = org.Branding.SupportEmail + userInfo.ActiveOrg.Branding.LogoutUrl = org.Branding.LogoutUrl + userInfo.ActiveOrg.Branding.BrandColor = org.Branding.BrandColor + userInfo.ActiveOrg.Branding.BrandName = org.Branding.BrandName + + parsedStatus = append(parsedStatus, "integration_partner") + } + } + returnValue := shuffle.HandleInfo{ Success: true, Username: userInfo.Username, @@ -1092,6 +1158,7 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { Licensed: licensed, ActiveApps: activatedAppIds, Theme: userInfo.Theme, + OrgStatus: parsedStatus, } returnData, err := json.Marshal(returnValue) diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 13289352..9425a1c6 100755 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -92,7 +92,7 @@ const App = (message, props) => { const [dataset, setDataset] = useState(false) const [isLoaded, setIsLoaded] = useState(false) const [curpath, setCurpath] = useState(typeof window === "undefined" || window.location === undefined ? "" : window.location.pathname) - const { themeMode, handleThemeChange, setBrandColor, brandColor,setThemeMode} = useContext(Context); + const { themeMode, handleThemeChange, setBrandColor, brandColor,setThemeMode, setSupportEmail, setLogoutUrl, setBrandName} = useContext(Context); const currentTheme = getTheme(themeMode, brandColor); const mainColor = currentTheme?.palette?.backgroundColor const [isPreviousThemeLight, setIsPreviousThemeLight] = useState(false) @@ -192,11 +192,44 @@ const App = (message, props) => { { path: "/" } ); } - if (responseJson?.theme?.length > 0) { - handleThemeChange(responseJson.theme) - }else{ - handleThemeChange("dark") - } + if (responseJson?.org_status?.includes("integration_partner")) { + if (responseJson?.active_org?.branding?.enable_chat !== true) { + + // Find the drift chatbox and remove it + } + + if (responseJson?.active_org?.branding?.theme?.length > 0 ) { + handleThemeChange(responseJson?.active_org?.branding?.theme) + } + + if (responseJson?.active_org?.branding?.brand_color?.length > 0 ) { + setBrandColor(responseJson?.active_org?.branding?.brand_color) + localStorage.setItem("brandColor", responseJson?.active_org?.branding?.brand_color) + } + + if (responseJson?.active_org?.branding?.brand_name?.length > 0 ) { + setBrandName(responseJson?.active_org?.branding?.brand_name) + localStorage.setItem("brandName", responseJson?.active_org?.branding?.brand_name) + } + + if (responseJson?.active_org?.branding?.support_email?.length > 0 ) { + setSupportEmail(responseJson?.active_org?.branding?.support_email) + } + + if (responseJson?.active_org?.branding?.logout_url?.length > 0 ) { + setLogoutUrl(responseJson?.active_org?.branding?.logout_url) + } + } else { + localStorage.removeItem("brandColor") + setBrandColor("#ff8544") + setBrandName("Shuffle") + localStorage.removeItem("brandName") + if(responseJson?.theme?.length > 0){ + handleThemeChange(responseJson?.theme) + }else{ + handleThemeChange("dark") + } + } }else { handleThemeChange("dark") setThemeMode("dark") From 06703c8e159126c955008553476a4bb2c1b6043c Mon Sep 17 00:00:00 2001 From: "lalitdeore12@gmail.com" Date: Mon, 21 Jul 2025 15:13:35 +0530 Subject: [PATCH 17/59] Added Line chart component from cloud --- frontend/src/components/LineChartWrapper.jsx | 107 +++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 frontend/src/components/LineChartWrapper.jsx diff --git a/frontend/src/components/LineChartWrapper.jsx b/frontend/src/components/LineChartWrapper.jsx new file mode 100644 index 00000000..95aa8024 --- /dev/null +++ b/frontend/src/components/LineChartWrapper.jsx @@ -0,0 +1,107 @@ +import React, { useState, useEffect, useContext, memo, useMemo } from 'react' +import {getTheme} from '../theme.jsx'; +import { Context } from '../context/ContextApi.jsx'; + +import { + Typography, +} from '@mui/material'; + +import { + BarChart, + BarSeries, + Bar, + BarLabel, + + GridlineSeries, + Gridline, + TooltipArea, + ChartTooltip, + TooltipTemplate, +} from 'reaviz'; + +const LineChartWrapper = (props) => { + const {keys, inputname, height, width, border} = props + + const [hovered, setHovered] = useState(""); + const {themeMode} = useContext(Context) + const theme = getTheme(themeMode) + + var inputdata = keys.data === undefined ? keys : keys.data + + var newname = inputname === undefined || inputname === null ? "" : inputname.trim().replaceAll("_", " ") + newname = newname.charAt(0).toUpperCase() + newname.slice(1) + + if (inputdata?.key !== undefined && inputdata?.key !== null && inputdata?.key !== "" && inputdata?.datasets !== undefined && inputdata?.datasets !== null && inputdata?.datasets.length > 0 && inputdata?.labels !== undefined && inputdata?.labels !== null && inputdata?.labels.length > 0) { + + var tmpdata = inputdata?.datasets[0] + if (tmpdata?.data !== undefined && tmpdata?.data !== null && tmpdata?.data.length > 0 && inputdata?.labels?.length === tmpdata?.data?.length) { + console.log("Fix it!") + var newarray = [] + for (var key in tmpdata.data) { + var entry = { + "key": inputdata.labels[key] !== undefined ? inputdata.labels[key] : key, + "data": tmpdata.data[key], + } + + newarray.push(entry) + } + + inputdata = newarray + } + + } + + if (inputdata === undefined || inputdata === null) { + return ( + + Invalid linegraph data format + + ) + } + + var defaultStyle = { + color: "white", + padding: 30, + marginTop: 15, + overflow: "hidden", + + border: "1px solid rgba(255,255,255,0.3)", + borderRadius: theme.palette?.borderRadius, + backgroundColor: theme.palette.platformColor, + } + + if (border === false) { + defaultStyle.border = "none" + defaultStyle.borderRadius = 0 + defaultStyle.backgroundColor = "transparent" + } + + return ( +
+ + {newname} + + + + } + /> + } + gridlines={ + } /> + } + /> + +
+ ) +} + +export default LineChartWrapper; From 5880cb525c36538ac64d3205ce81416c6f277703 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Mon, 21 Jul 2025 18:24:23 +0530 Subject: [PATCH 18/59] chore: shuffle-shared sync up --- backend/go-app/go.mod | 2 +- functions/onprem/orborus/go.mod | 4 ++-- functions/onprem/worker/go.mod | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 98a348bd..2861454a 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -24,7 +24,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.92 + github.com/shuffle/shuffle-shared v0.8.96 github.com/shuffle/singul v0.0.15 golang.org/x/crypto v0.38.0 google.golang.org/api v0.236.0 diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 3fc74c7e..c34b4b32 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -10,7 +10,7 @@ require ( github.com/docker/docker v28.2.2+incompatible github.com/docker/go-connections v0.5.0 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.88 + github.com/shuffle/shuffle-shared v0.8.96 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 ) @@ -58,7 +58,7 @@ require ( github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/frikky/kin-openapi v0.42.0 // indirect - github.com/frikky/schemaless v0.0.16 // indirect + github.com/frikky/schemaless v0.0.17 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index f22f6fdf..083d46a1 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -10,7 +10,7 @@ require ( github.com/docker/docker v28.2.2+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.88 + github.com/shuffle/shuffle-shared v0.8.96 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 k8s.io/client-go v0.33.1 @@ -60,7 +60,7 @@ require ( github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/frikky/kin-openapi v0.42.0 // indirect - github.com/frikky/schemaless v0.0.16 // indirect + github.com/frikky/schemaless v0.0.17 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect From d234214967fe306741d9fdb8cbc9f600838e9e15 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Tue, 22 Jul 2025 18:43:11 +0530 Subject: [PATCH 19/59] fix: image build problems due to import --- frontend/src/views/AppCreator.jsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 1eaf5fcb..414a7af0 100755 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -54,7 +54,7 @@ import YAML from "yaml"; import { MuiChipsInput } from "mui-chips-input"; //import { useAlert import { ToastContainer, toast } from "react-toastify" -import words from "shellwords"; +import { split } from "shellwords"; import AvatarEditor from "react-avatar-editor"; import { Context } from "../context/ContextApi.jsx"; @@ -119,7 +119,7 @@ const parseCurl = (s) => { } try { - var args = rewrite(words.split(s)); + var args = rewrite(split(s)); } catch (e) { return s; } From acd2116a492239ce97f20f6636b19031b21e747f Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Fri, 1 Aug 2025 21:39:30 +0530 Subject: [PATCH 20/59] fix: adding debug log fixes --- functions/onprem/orborus/orborus.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index e350082b..de74c974 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -4013,7 +4013,7 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest, image string, debugCommand := fmt.Sprintf("docker service logs shuffle-workers 2>&1 -f | grep %s", workflowExecution.ExecutionId) if isKubernetes == "true" { - debugCommand = fmt.Sprintf("kubectl logs -n %s container=shuffle-worker | grep %s", kubernetesNamespace, workflowExecution.ExecutionId) + debugCommand = fmt.Sprintf("kubectl logs -n %s deployment/shuffle-workers | grep %s", kubernetesNamespace, workflowExecution.ExecutionId) } log.Printf("[DEBUG] Ran worker from request with execution ID: %s. Worker URL: %s. DEBUGGING:\n%s", workflowExecution.ExecutionId, streamUrl, debugCommand) From b332276d20af493684135089c02605d665ddaf7a Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Wed, 6 Aug 2025 04:09:52 +0530 Subject: [PATCH 21/59] keep context of downloaded images --- functions/onprem/worker/worker.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 2fbf51e3..5c31f7e3 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -840,7 +840,10 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] // Running as coroutine for eventual completeness // FIXME: With goroutines it got too much trouble of deploying with an older version // Allowing slow startups, as long as it's eventually fast, and uses the same registry as on host. - shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, image) + err := shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, image) + if err == nil { + downloadedImages = append(downloadedImages, image) + } } var exposedPort int @@ -1598,6 +1601,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { executed := false if err == nil { log.Printf("[DEBUG] Downloaded image %s from backend (CLEANUP)", imageName) + downloadedImages = append(downloadedImages, imageName) //err = deployApp(dockercli, image, identifier, env, workflow, action) err = deployApp(dockercli, imageName, identifier, env, workflowExecution, action) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { @@ -1712,6 +1716,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { executed := false if err == nil { log.Printf("[DEBUG] Downloaded image %s from backend (CLEANUP)", imageName) + downloadedImages = append(downloadedImages, imageName) //err = deployApp(dockercli, image, identifier, env, workflow, action) err = deployApp(dockercli, imageName, identifier, env, workflowExecution, action) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { @@ -4415,7 +4420,10 @@ func handleDownloadImage(resp http.ResponseWriter, request *http.Request) { } log.Printf("[INFO] Downloading image %s", imageBody.Image) - shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, imageBody.Image) + err = shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, imageBody.Image) + if err == nil { + downloadedImages = append(downloadedImages, imageBody.Image) + } // return success resp.WriteHeader(200) From 4f98b583a423bb1a2200a4bfc8121ed0f862cd5f Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Thu, 7 Aug 2025 23:04:38 +0530 Subject: [PATCH 22/59] fixing some old bugs --- backend/go-app/walkoff.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index eb174870..c7d3658d 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -309,10 +309,10 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { } if !found { - log.Printf("[ERROR] Failed to find environment(%s) for org(%s)", environment, orgId) - resp.WriteHeader(400) - resp.Write([]byte(`{"success":false,"reason":"environment not found"}`)) - return + env, err = shuffle.GetEnvironment(ctx, environment, "") + if err != nil { + log.Printf("[WARNING] Failed to find the environment(%s) in org(%s). Could cause with Failover test", environment, orgId) + } } timeNow := time.Now().Unix() From 02000d7e000ed7fc9fd5759cc69d56f629e4b995 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Mon, 18 Aug 2025 13:56:42 +0530 Subject: [PATCH 23/59] chore: bump shuffle-shared --- backend/go-app/go.mod | 2 +- functions/onprem/orborus/go.mod | 2 +- functions/onprem/worker/go.mod | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 2861454a..9929130c 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -24,7 +24,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.96 + github.com/shuffle/shuffle-shared v0.8.99 github.com/shuffle/singul v0.0.15 golang.org/x/crypto v0.38.0 google.golang.org/api v0.236.0 diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index c34b4b32..8e32a56d 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -10,7 +10,7 @@ require ( github.com/docker/docker v28.2.2+incompatible github.com/docker/go-connections v0.5.0 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.96 + github.com/shuffle/shuffle-shared v0.8.99 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 ) diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 083d46a1..7b39554b 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -10,7 +10,7 @@ require ( github.com/docker/docker v28.2.2+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.96 + github.com/shuffle/shuffle-shared v0.8.99 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 k8s.io/client-go v0.33.1 From db24545fc0f92e23e4d782531948b054d90b11fb Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Mon, 18 Aug 2025 14:25:06 +0000 Subject: [PATCH 24/59] feat: add local AI availability check at server startup --- backend/go-app/main.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 4e4cce58..11d4473d 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -1159,6 +1159,7 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { ActiveApps: activatedAppIds, Theme: userInfo.Theme, OrgStatus: parsedStatus, + AIEnabled: org.LocalAIEnabled, } returnData, err := json.Marshal(returnValue) @@ -5479,6 +5480,23 @@ func initHandlers() { r.Use(shuffle.RequestMiddleware) http.Handle("/", r) + + go func() { + log.Printf("[DEBUG] Starting AI availability check for all organizations") + activeOrgs, err := shuffle.GetAllOrgs(ctx) + if err != nil { + log.Printf("[ERROR] Error getting organizations for AI availability check: %s", err) + return + } + + for _, org := range activeOrgs { + if len(org.Id) == 0 { + continue + } + + go shuffle.CheckAndUpdateAIAvailability(ctx, org.Id) + } + }() } // Had to move away from mux, which means Method is fucked up right now. From 45ca47c28227e69bfb532142a997bea2cc409cef Mon Sep 17 00:00:00 2001 From: Frikky Date: Mon, 18 Aug 2025 17:03:54 +0200 Subject: [PATCH 25/59] Many small fixes :) --- backend/go-app/go.mod | 2 +- backend/go-app/main.go | 3 +- frontend/src/components/EditWorkflow.jsx | 401 +++++++++++++++++---- frontend/src/components/EnvironmentTab.jsx | 57 +-- frontend/src/components/PartnerHeader.jsx | 4 +- frontend/src/theme.jsx | 92 ++++- frontend/src/views/LoginPage.jsx | 86 ++++- 7 files changed, 529 insertions(+), 116 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 9929130c..aa007e28 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -5,7 +5,7 @@ go 1.24.0 toolchain go1.24.3 //replace github.com/frikky/schemaless => ../../../schemaless -//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared +replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 4e4cce58..4279e19f 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -4099,8 +4099,9 @@ func runInitEs(ctx context.Context) { time.Sleep(30 * time.Second) } - // FIXME: This should ONLY run on one backend instance + shuffle.InitOpensearchIndexes() + // FIXME: This should ONLY run on one backend instance. This may cause interference. schedules, err := shuffle.GetAllSchedules(ctx, "ALL") if err != nil { log.Printf("[WARNING] Failed getting schedules during service init: %s", err) diff --git a/frontend/src/components/EditWorkflow.jsx b/frontend/src/components/EditWorkflow.jsx index dbd8d3ea..00b9608d 100644 --- a/frontend/src/components/EditWorkflow.jsx +++ b/frontend/src/components/EditWorkflow.jsx @@ -63,6 +63,7 @@ import { Add as AddIcon, Remove as RemoveIcon, EditNote as EditNoteIcon, + AutoAwesome as AutoAwesomeIcon } from "@mui/icons-material"; const EditWorkflow = (props) => { @@ -72,6 +73,7 @@ const EditWorkflow = (props) => { const {themeMode, brandColor} = useContext(Context) const theme = getTheme(themeMode, brandColor) const [submitLoading, setSubmitLoading] = React.useState(false); + const [aiGenerateLoading, setAiGenerateLoading] = React.useState(false); const [showMoreClicked, setShowMoreClicked] = React.useState(isEditing !== false ? true : false); const [innerWorkflow, setInnerWorkflow] = React.useState(workflow) @@ -220,7 +222,7 @@ const EditWorkflow = (props) => {
- + {newWorkflow ? "New" : "Editing"} Workflow @@ -298,93 +300,316 @@ const EditWorkflow = (props) => { paddingLeft: 30, backgroundColor: themeMode === "dark" ? "#262626" : theme.palette.DialogStyle.backgroundColor, }}> - + + + Generate using the name, description, usecases and tags provided. Required: Name + Description + + } + > + + + + +
+ ) : ( + + // If new workflow, don't close it + if (isEditing) { + setModalOpen(false) + } + }} + color="primary" + > + {submitLoading ? : "Save Changes"} + + )}
@@ -409,6 +634,29 @@ const EditWorkflow = (props) => { id="Enter-Workflow-Name" /> + {newWorkflow === true ? + { + innerWorkflow.default_return_value = event.target.value + setInnerWorkflow(innerWorkflow) + setUpdate(Math.random()) + }} + InputProps={{ + style: { + color: "white", + }, + }} + color="primary" + defaultValue={innerWorkflow.default_return_value} + placeholder="Please describe your workflow below so the AI can generate it." + rows="3" + multiline + label="Description" + margin="dense" + fullWidth + /> + : null} +
{usecases !== null && usecases !== undefined && usecases.length > 0 ? @@ -1285,7 +1533,7 @@ const EditWorkflow = (props) => { - {newWorkflow === true ? + {/*newWorkflow === true ? Relevant Workflows @@ -1311,9 +1559,8 @@ const EditWorkflow = (props) => { } - : null} - - {/*newWorkflow === true && name.length > 2 ? + : null*/} + {/*newWorkflow === true && name.length > 2 ?
{ //toast("No Orborus necessary for environment cloud. Create and use a different environment to run executions on-premises.",) return } - + if ( props.userdata.active_org === undefined || props.userdata.active_org === null ) { return; } - + const elementName = "copy_element_shuffle"; var auth = environment.auth === "" ? "cb5st3d3Z!3X3zaJ*Pc" : environment.auth - - // Escape exclamation marks for copying - auth = auth.replace("\\!", "!").replace(/!/g, "\\!") - + // Escape exclamation marks for copying + auth = auth.replace("\\!", "!").replace(/!/g, "\\!") const newUrl = globalUrl === "https://shuffler.io" ? "https://shuffle-backend-stbuwivzoq-nw.a.run.app" : globalUrl; - + var skipPipeline = false if (commandController.pipelines === true) { skipPipeline = true } - + var addProxy = false if (commandController.proxies === true) { addProxy = true } - + if (installationTab === 1) { return (`docker run -d \\ --restart=always \\ @@ -428,14 +426,32 @@ const EnvironmentTab = memo((props) => { -e SHUFFLE_SWARM_CONFIG=run \\ -e SHUFFLE_LOGS_DISABLED=true \\ -e BASE_URL="${newUrl}" \\${addProxy ? ` - -e HTTPS_PROXY=IP:PORT \\` : ""}${skipPipeline ? ` - -e SHUFFLE_SKIP_PIPELINES=true \\` : ""} + -e HTTPS_PROXY=IP:PORT \\` : ""}${skipPipeline ? ` + -e SHUFFLE_SKIP_PIPELINES=true \\` : ""} ghcr.io/shuffle/shuffle-orborus:latest `) } else if (installationTab === 2) { - return `https://shuffler.io/docs/configuration#kubernetes` + return `helm install shuffle-orborus oci://ghcr.io/shuffle/charts/shuffle \\ + --namespace shuffle --create-namespace \\ + --set opensearch.enabled=false \\ + --set backend.replicaCount=0 \\ + --set frontend.replicaCount=0 \\ + --set 'orborus.extraEnvVars[0].name=AUTH' \\ + --set 'orborus.extraEnvVars[0].value=${auth}' \\ + --set 'orborus.extraEnvVars[1].name=BASE_URL' \\ + --set 'orborus.extraEnvVars[1].value=${newUrl}' \\ + --set 'orborus.extraEnvVars[2].name=ENVIRONMENT_NAME' \\ + --set 'orborus.extraEnvVars[2].value=${environment.Name}' \\ + --set 'orborus.extraEnvVars[3].name=ORG' \\ + --set 'orborus.extraEnvVars[3].value=${environment.org_id}' \\ + --set shuffle.org="${environment.org_id}"${addProxy ? ` \\ + --set 'orborus.extraEnvVars[4].name=HTTPS_PROXY' \\ + --set 'orborus.extraEnvVars[4].value=IP:PORT'` : ""}${skipPipeline ? ` \\ + --set 'orborus.extraEnvVars[${addProxy ? 5 : 4}].name=SHUFFLE_SKIP_PIPELINES' \\ + --set 'orborus.extraEnvVars[${addProxy ? 5 : 4}].value=true'` : ""} \\ + --set persistence.storageClass=standard` } - + const commandData = `docker rm shuffle-orborus --force; \\\ndocker run -d \\ --restart=always \\ --name="shuffle-orborus" \\ @@ -445,12 +461,13 @@ const EnvironmentTab = memo((props) => { -e ENVIRONMENT_NAME="${environment.Name}" \\ -e ORG="${props.userdata.active_org.id}" \\ -e BASE_URL="${newUrl}" \\${addProxy ? ` - -e HTTPS_PROXY=IP:PORT \\` : ""}${skipPipeline ? ` - -e SHUFFLE_SKIP_PIPELINES=true \\` : ""} + -e HTTPS_PROXY=IP:PORT \\` : ""}${skipPipeline ? ` + -e SHUFFLE_SKIP_PIPELINES=true \\` : ""} ghcr.io/shuffle/shuffle-orborus:latest` - + return commandData - }; + }; + const submitEnvironment = (data) => { // FIXME - add some check here ROFL @@ -1475,7 +1492,7 @@ const EnvironmentTab = memo((props) => { {installationTab === 2 ? - Check our Kubernetes documentation for more information on how to run Shuffle on Kubernetes. The status of the node will change when connected. + Simply connect to your Kubernetes cluster and run the following command: : @@ -1489,7 +1506,6 @@ const EnvironmentTab = memo((props) => { "2. Run this command on the server you want to run workflows or store Pipeline data on"} - {installationTab === 2 ? null :
{ }} />
-
- } +
{installationTab === 2 ? null : diff --git a/frontend/src/components/PartnerHeader.jsx b/frontend/src/components/PartnerHeader.jsx index 8cb84c0a..3a6a910b 100644 --- a/frontend/src/components/PartnerHeader.jsx +++ b/frontend/src/components/PartnerHeader.jsx @@ -741,7 +741,7 @@ const PartnerHeader = (props) => {
From 315d65898b8ae5acf5f251ed9bcd5059f06fe66a Mon Sep 17 00:00:00 2001 From: Frikky Date: Mon, 18 Aug 2025 17:17:33 +0200 Subject: [PATCH 26/59] Added Opensearch index control and rollover config control --- backend/go-app/go.mod | 4 +- backend/go-app/go.sum | 4 +- frontend/src/components/HealthBarChart.jsx | 103 ++++++-- frontend/src/components/HealthPage.jsx | 286 ++++++++++++++++----- 4 files changed, 317 insertions(+), 80 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index aa007e28..7c857d01 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -5,7 +5,7 @@ go 1.24.0 toolchain go1.24.3 //replace github.com/frikky/schemaless => ../../../schemaless -replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared +//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi @@ -24,7 +24,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.99 + github.com/shuffle/shuffle-shared v0.9.0 github.com/shuffle/singul v0.0.15 golang.org/x/crypto v0.38.0 google.golang.org/api v0.236.0 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 51d36d6e..163f7931 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -359,8 +359,8 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.91 h1:ZcxSKhns96w1GQG/EjEJSLBvuEA3KVjvORAetV3W4bM= -github.com/shuffle/shuffle-shared v0.8.91/go.mod h1:PFLG4eaxz8zQc9Lbc0Ok/nG8rNsaBkZ0TNar+VU+J/g= +github.com/shuffle/shuffle-shared v0.9.0 h1:hUNtW3qHkji2cUVyR1cIXAz6qxpAHHP5hBmSAe3pIhs= +github.com/shuffle/shuffle-shared v0.9.0/go.mod h1:PFLG4eaxz8zQc9Lbc0Ok/nG8rNsaBkZ0TNar+VU+J/g= github.com/shuffle/singul v0.0.15 h1:a5Qro1BKs9a+mWE289oYTkDeVFoKEd/H9qu9XJGsVxo= github.com/shuffle/singul v0.0.15/go.mod h1:z1RRZsymTYxsB8WMQwwswypruy4j5EugGPFEZspTEA4= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= diff --git a/frontend/src/components/HealthBarChart.jsx b/frontend/src/components/HealthBarChart.jsx index a4f05273..a5d1dec4 100644 --- a/frontend/src/components/HealthBarChart.jsx +++ b/frontend/src/components/HealthBarChart.jsx @@ -1,22 +1,93 @@ -import React from 'react'; -import { Bar } from 'react-chartjs-2'; +import React, { useState } from 'react'; const HealthBarChart = (props) => { - const { globalUrl, filteredData, options, onBarClick } = props; - - return ( - { - if (elements && elements.length > 0) { - onBarClick(elements); - } + const { filteredData, onBarClick } = props; + const [hoveredBar, setHoveredBar] = useState(null); + + if (!filteredData || !Array.isArray(filteredData)) { + console.error('Invalid chart data format'); + return null; + } + + const formatData = () => { + return filteredData.map((item) => ({ + label: item.date, + value: item.avgRunFinished, + color: item.color, + executionIds: item.executionIds, + })); + }; + + const handleBarClick = (data) => { + if (onBarClick) { + onBarClick(data); + } + }; + + const chartData = formatData(); + + const barWidth = `calc((100% - ${(chartData.length - 1) * 5}px) / ${chartData.length})`; + const barGap = '5px'; + + return ( +
+ {/* Chart Container */} +
+ {chartData.map((item, index) => ( +
- ); + onMouseEnter={() => setHoveredBar(item)} + onMouseLeave={() => setHoveredBar(null)} + onClick={() => handleBarClick(item)} + /> + ))} +
+ + {/* Tooltip */} + {hoveredBar && ( +
+
Date: {hoveredBar.label}
+
Uptime: {hoveredBar.value}%
+
Executions: {hoveredBar.executionIds.length}
+
+ )} +
+ ); }; export default HealthBarChart; diff --git a/frontend/src/components/HealthPage.jsx b/frontend/src/components/HealthPage.jsx index fcadb776..660719e0 100644 --- a/frontend/src/components/HealthPage.jsx +++ b/frontend/src/components/HealthPage.jsx @@ -1,31 +1,35 @@ import React, { useEffect, useState, useCallback } from 'react'; -import { Bar } from 'react-chartjs-2'; -//import healthData1 from '../healthstats1.json'; -import { toast } from "react-toastify" -import CheckOutlinedIcon from '@mui/icons-material/CheckOutlined'; +import { toast } from "react-toastify"; +import { + CheckOutlined as CheckOutlinedIcon, +} from '@mui/icons-material'; + import { Button, ButtonGroup, Typography, + LinearProgress, } from "@mui/material"; + import HealthBarChart from '../components/HealthBarChart.jsx'; +import LiveExecutionsChart from '../components/LiveExecutionsGraph.jsx'; const HealthPage = (props) => { - const { globalUrl, userdata } = props; + const { userdata, globalUrl } = props; const [healthData, setHealthData] = useState(null); const [selectedRange, setSelectedRange] = useState('30d'); + const [liveExecutionsData, setLiveExecutionsData] = useState([]); const [filteredData, setFilteredData] = useState([]); const [averageUptime, setAverageUptime] = useState(0); + const [liveExecutionsRange, setLiveExecutionsRange] = useState('1h'); // Default to 1h + const [isHealthLoading, setIsHealthLoading] = useState(false); // Loading state for HealthBarChart + const [isLiveExecutionsLoading, setIsLiveExecutionsLoading] = useState(false); // Loading state for LiveExecutionsChart - const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; - //const globalUrl = `https://shuffler.io` - - console.log("HEALTHPAGE 1") + const isCloud = (window.location.host === "localhost:3002" || window.location.host === "shuffler.io") ? true : (process.env.IS_SSR === "true"); const fetchHealthStats = useCallback(async () => { - //const after = new Date().getTime() - 90 * 24 * 60 * 60 + setIsHealthLoading(true); // Start loading for HealthBarChart try { - //const response = await fetch(`${globalUrl}/api/v1/health/stats?after=${after}`, { const response = await fetch(`${globalUrl}/api/v1/health/stats`, { method: "GET", headers: { @@ -42,31 +46,118 @@ const HealthPage = (props) => { } catch (error) { console.error("Error fetching health stats:", error); toast.error("Failed loading health stats"); + } finally { + setIsHealthLoading(false); // Stop loading for HealthBarChart + } + }, [globalUrl]); + + const fetchLiveExecutions = useCallback(async (range = '1h') => { + setIsLiveExecutionsLoading(true); // Start loading for LiveExecutionsChart + try { + const fetchOptions = { + method: "GET", + credentials: "include", + }; + + if (window.location.host !== "localhost:3002") { + fetchOptions.headers = { + "Content-Type": "application/json", + Accept: "application/json", + }; + } + + const now = Math.floor(Date.now() / 1000); + // let after = now - 3600; // Default to 1h + let mode = "" + + switch (range) { + case '1h': + mode = "1h" + break; + case '7h': + mode = "7h" + break; + case '1d': + mode = "1d" + break; + case '7d': + mode = "7d" + break; + case 'month': + mode = "month" + break; + default: + mode = "1h" + } + + if (!userdata.support_access) { + return + } + + const response = await fetch( + `${globalUrl}/api/v1/health/executions/live?mode=${mode}`, + fetchOptions + ); + + if (!response.ok) { + throw new Error("Failed to fetch live executions"); + } + + const data = await response.json(); + console.log("Raw live executions data:", data); + + if (Array.isArray(data)) { + const formattedData = data + .map(item => ({ + ...item, + // failed: Number(item.failed) || 0, + executing: Number(item.executing) || 0, + finished: Number(item.finished) || 0, + aborted: Number(item.aborted) || 0, + created_at: Number(item.created_at) || 0 + })) + .sort((a, b) => a.created_at - b.created_at); + + console.log("Formatted live executions data:", formattedData); + setLiveExecutionsData(formattedData); + } else { + console.error("Received invalid data format:", data); + setLiveExecutionsData([]); + } + } catch (error) { + console.error("Error fetching live executions:", error); + toast.error("Failed loading live executions data"); + } finally { + setIsLiveExecutionsLoading(false); // Stop loading for LiveExecutionsChart } }, [globalUrl]); useEffect(() => { fetchHealthStats(); - }, [fetchHealthStats]); + fetchLiveExecutions(liveExecutionsRange); + + const interval = setInterval(() => fetchLiveExecutions(liveExecutionsRange), 60000); + return () => clearInterval(interval); + }, [fetchHealthStats, fetchLiveExecutions, liveExecutionsRange]); const extractRunFinished = (data, range) => { if (!data || !Array.isArray(data)) return []; - + const currentDate = new Date().getTime(); const rangeInMillis = { '24hr': 24 * 60 * 60 * 1000, '7day': 7 * 24 * 60 * 60 * 1000, '30d': 30 * 24 * 60 * 60 * 1000, '90d': 90 * 24 * 60 * 60 * 1000 - } - const filteredData = data.filter(item => currentDate - item.updated * 1000 <= rangeInMillis[range]) - - const aggregatedData = new Map() - + }; + const filteredData = data.filter(item => currentDate - item.updated * 1000 <= rangeInMillis[range]); + + const aggregatedData = new Map(); + filteredData.forEach(item => { const timestamp = item.updated * 1000; // Convert Unix timestamp to milliseconds let key; - + switch (range) { case '24hr': const date = new Date(timestamp); @@ -83,7 +174,7 @@ const HealthPage = (props) => { default: key = new Date(timestamp).toLocaleDateString(); } - + // Check if date already exists in the map if (aggregatedData.has(key)) { // Update aggregated values @@ -100,13 +191,13 @@ const HealthPage = (props) => { }); } }); - + // Calculate averages and assign colors const result = Array.from(aggregatedData.entries()).map(([key, { totalEntries, totalRunFinished, executionIds }]) => { const avg = totalEntries > 0 ? totalRunFinished / totalEntries : 0; const FinalAvg = avg * 100; let color; - + if (FinalAvg >= 100) { color = '#00F670'; } else if (FinalAvg >= 98.50 && FinalAvg <= 99.99) { @@ -114,7 +205,7 @@ const HealthPage = (props) => { } else if (FinalAvg <= 98.49) { color = '#FF354C'; } - + return { date: range === '24hr' ? `${key}:00` : key, avgRunFinished: FinalAvg, @@ -122,11 +213,10 @@ const HealthPage = (props) => { executionIds }; }); - + return result; }; - - + useEffect(() => { if (healthData) { const newData = extractRunFinished(healthData, selectedRange); @@ -160,19 +250,19 @@ const HealthPage = (props) => { const labels = filteredData.map((value, i) => { if (selectedRange === '24hr') { const [datePart, hourPart] = value.date.split(' '); - const [day, month, year] = datePart.split('/'); + const [month, day, year] = datePart.split('/'); const monthIndex = parseInt(month, 10) - 1; const date = new Date(year, monthIndex, day); let formattedDate = `${date.toLocaleString('en-US', { month: 'short', day: '2-digit' })}`; - // Add hour part if available if (hourPart) { formattedDate += `, ${hourPart.split(':').slice(0, 2).join(':')}`; } + return `${formattedDate} \nUptime: ${value.avgRunFinished.toFixed(2)}%`; } else if (selectedRange === '7day') { const [datePart, hourPart] = value.date.split(' '); - const [day, month, year] = datePart.split('/'); + const [month, day, year] = datePart.split('/'); const monthIndex = parseInt(month, 10) - 1; const date = new Date(year, monthIndex, day); let formattedDate = `${date.toLocaleString('en-US', { month: 'short', day: '2-digit' })}`; @@ -185,12 +275,12 @@ const HealthPage = (props) => { } else { const dateParts = value.date.split('/'); // Assuming the date format is "DD/MM/YYYY" - const date = new Date(`${dateParts[2]}-${dateParts[1]}-${dateParts[0]}`); // Reformat the date string to "YYYY-MM-DD" + const date = new Date(`${dateParts[2]}-${dateParts[0]}-${dateParts[1]}`); // Reformat the date string to "YYYY-MM-DD" + return `${date.toLocaleDateString('en-US', { month: 'short', day: '2-digit', year: 'numeric' })} \nUptime: ${value.avgRunFinished.toFixed(2)}%`; } }); - if (selectedRange === '24hr') { barThickness = 35; } @@ -200,12 +290,6 @@ const HealthPage = (props) => { else if (selectedRange === '30d') { barThickness = 25; } - // let width = 800; // Default chart width - // if (selectedRange === '7day') { - // width = 400; // Adjust chart width for 7 days - // } else if (selectedRange === '30d') { - // width = 600; // Adjust chart width for 30 days - // } const datasets = [{ label: "", @@ -213,14 +297,11 @@ const HealthPage = (props) => { backgroundColor: filteredData.map(item => item.color), borderWidth: 1, barThickness: barThickness, - // barPercentage: barPercentage, }]; return { labels, datasets }; }; - console.log("HEALTHPAGE 2") - const options = { legend: { display: false @@ -253,12 +334,11 @@ const HealthPage = (props) => { }, afterLabel: function (tooltipItem, data) { const label = data.labels[tooltipItem.index]; - // console.log(label) const uptime = label.match(/Uptime:\s*(\d+(?:\.\d+)?)/)[1]; // Extract uptime value using regex - return `Success Rate: ${uptime}%`; // Customize the uptime display + return `Test-Workflow Health: ${uptime}%`; // Customize the uptime display }, title: function () { - return 'Fully Oprational'; // Hide the tooltip title + return 'Fully Operational'; // Hide the tooltip title } } } @@ -284,32 +364,118 @@ const HealthPage = (props) => { } }; + const healthBarData = updateChartData() + return ( -
- +
+ {/* Health Bar Chart Section */} - - - - - + + + + + -
-
- -
- Workflow Health - Operational + {/* Loading Bar for HealthBarChart */} + {isHealthLoading && ( + + )} + +
+
+
+ +
+ Workflow Health + Operational +
-
- {averageUptime.toFixed(2)}% - Success Rate +
+ {averageUptime.toFixed(2)}% + Success Rate
- +
+ + {userdata.support_access && ( +
+
+ Live Executions + + + + + + {/* */} + +
+ {/* Loading Bar for LiveExecutionsChart */} + {isLiveExecutionsLoading && ( + + )} + +
+ )} + +
); }; From 3bf5ac7400c20207265508c76abbf7b7d0e8af66 Mon Sep 17 00:00:00 2001 From: Frikky Date: Mon, 18 Aug 2025 17:20:51 +0200 Subject: [PATCH 27/59] Fix backend failure again --- backend/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 9e27ffbe..16ffeff0 100755 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -27,7 +27,7 @@ FROM alpine:latest as certs RUN apk add --update ca-certificates # Sets up the final image -FROM alpine:3.21.2 +FROM alpine:3.22.1 # FIXME: Install cgo because CGO_ENABLED=1 during build RUN apk add --no-cache libc6-compat From 0bf0736d07fdf16803c9a17308dc961d05a94bc4 Mon Sep 17 00:00:00 2001 From: Frikky Date: Mon, 18 Aug 2025 19:15:47 +0200 Subject: [PATCH 28/59] Rebuild with all components --- frontend/src/components/LiveExecutionsGraph.jsx | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 frontend/src/components/LiveExecutionsGraph.jsx diff --git a/frontend/src/components/LiveExecutionsGraph.jsx b/frontend/src/components/LiveExecutionsGraph.jsx new file mode 100644 index 00000000..ac572e84 --- /dev/null +++ b/frontend/src/components/LiveExecutionsGraph.jsx @@ -0,0 +1,7 @@ +import React from 'react'; + +const LiveExecutionsGraph = ({ executions }) => { + return null +} + +export default LiveExecutionsGraph From feaee0833359fccf06242699229bccb51ecd0c87 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Wed, 20 Aug 2025 11:21:00 +0200 Subject: [PATCH 29/59] fix(k8s): pass SHUFFLE_LOGS_DISABLED env var from orborus -> worker -> app Signed-off-by: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> --- functions/onprem/orborus/orborus.go | 39 ++++++++++++++++------------- functions/onprem/worker/worker.go | 4 +++ 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index de74c974..ce7318d1 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -734,7 +734,7 @@ func deployServiceWorkers(image string) { var updatedNetworks []swarm.NetworkAttachmentConfig for _, net := range serviceSpec.Networks { if net.Target != "shuffle_shuffle" { - updatedNetworks = append(updatedNetworks, net) + updatedNetworks = append(updatedNetworks, net) } } serviceSpec.Networks = updatedNetworks @@ -813,21 +813,21 @@ func handleBackendImageDownload(ctx context.Context, images string) error { newImages = append(newImages, curimage) // Force remove the current image to avoid cached layers -// if swarmConfig == "run" || swarmConfig == "swarm" { -// _, err := dockercli.ImageRemove(ctx, curimage, image.RemoveOptions{ -// Force: true, -// PruneChildren: true, -// }) -// -// if err != nil { -// log.Printf("[ERROR] Failed removing image for re-download: %s", err) -// } else { -// log.Printf("[DEBUG] Removed image: %s", curimage) -// } -// } else { -// //log.Printf("[DEBUG] Skipping image removal for %s as swarmConfig is not set to run or swarm. Value: %#v", curimage, swarmConfig) -// } -// + // if swarmConfig == "run" || swarmConfig == "swarm" { + // _, err := dockercli.ImageRemove(ctx, curimage, image.RemoveOptions{ + // Force: true, + // PruneChildren: true, + // }) + + // if err != nil { + // log.Printf("[ERROR] Failed removing image for re-download: %s", err) + // } else { + // log.Printf("[DEBUG] Removed image: %s", curimage) + // } + // } else { + // //log.Printf("[DEBUG] Skipping image removal for %s as swarmConfig is not set to run or swarm. Value: %#v", curimage, swarmConfig) + // } + err := shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, curimage) if err != nil { //log.Printf("[ERROR] Failed downloading image: %s", err) @@ -887,7 +887,7 @@ func handleBackendImageDownload(ctx context.Context, images string) error { log.Printf("[ERROR] Failed updating service %s with the new image %s: %s. Resp: %#v", service.Spec.Annotations.Name, image, err, resp) } else { log.Printf("[DEBUG] Updated service %s with the new image %s. Resp: %#v", service.Spec.Annotations.Name, image, resp) - + found = true if !strings.Contains(fmt.Sprintf("%s", resp), "error") { @@ -1065,6 +1065,10 @@ func deployK8sWorker(image string, identifier string, env []string) error { env = append(env, fmt.Sprintf("SHUFFLE_APP_CONTAINER_SECURITY_CONTEXT=%s", appContainerSecurityContext)) } + if len(os.Getenv("SHUFFLE_LOGS_DISABLED")) > 0 { + env = append(env, fmt.Sprintf("SHUFFLE_LOGS_DISABLED=%s", os.Getenv("SHUFFLE_LOGS_DISABLED"))) + } + clientset, _, err := shuffle.GetKubernetesClient() if err != nil { log.Printf("[ERROR] Error getting kubernetes client:", err) @@ -2214,7 +2218,6 @@ func main() { log.Printf("[INFO] Waiting for executions at %s with Environment %#v", fullUrl, environment) - hasStarted := false for { if req.Method == "POST" { diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 5c31f7e3..86f72e00 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -422,6 +422,10 @@ func deployk8sApp(image string, identifier string, env []string) error { envMap["SHUFFLE_SWARM_CONFIG"] = os.Getenv("SHUFFLE_SWARM_CONFIG") envMap["BASE_URL"] = "http://shuffle-workers:33333" + if len(os.Getenv("SHUFFLE_LOGS_DISABLED")) > 0 { + envMap["SHUFFLE_LOGS_DISABLED"] = os.Getenv("SHUFFLE_LOGS_DISABLED") + } + clientset, _, err := shuffle.GetKubernetesClient() if err != nil { log.Printf("[ERROR] Failed getting kubernetes: %s", err) From fcf36f1b866f5333afa504f959370cb3ce3a2c64 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Thu, 21 Aug 2025 16:34:51 +0530 Subject: [PATCH 30/59] chore: shuffle-shared bump --- functions/onprem/orborus/go.mod | 2 +- functions/onprem/orborus/orborus.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 8e32a56d..3865745d 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -10,7 +10,7 @@ require ( github.com/docker/docker v28.2.2+incompatible github.com/docker/go-connections v0.5.0 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.99 + github.com/shuffle/shuffle-shared v0.9.1 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 ) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index e350082b..1d13496b 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -2298,7 +2298,7 @@ func main() { } if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" && os.Getenv("SHUFFLE_SCALE_REPLICAS") == "" { - // go AutoScale(ctx) + go AutoScale(ctx) } hasStarted = true } From 3167756c610e0bf19505c0b83bd11fa60fa391c5 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Thu, 21 Aug 2025 16:36:16 +0530 Subject: [PATCH 31/59] comment autoscale back --- functions/onprem/orborus/orborus.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 1d13496b..35bd8f93 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -2298,7 +2298,7 @@ func main() { } if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" && os.Getenv("SHUFFLE_SCALE_REPLICAS") == "" { - go AutoScale(ctx) + //go AutoScale(ctx) } hasStarted = true } From 065dc160b3ee87474c88282e06b4a44ba024fec9 Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 22 Aug 2025 00:02:11 +0200 Subject: [PATCH 32/59] More delay and network management when issues occur --- functions/onprem/orborus/go.mod | 2 +- functions/onprem/orborus/go.sum | 8 +- functions/onprem/orborus/orborus.go | 2 +- functions/onprem/worker/go.mod | 2 +- functions/onprem/worker/go.sum | 8 +- functions/onprem/worker/worker.go | 148 +++++++++++++++++++++++++--- 6 files changed, 146 insertions(+), 24 deletions(-) diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 8e32a56d..c194ecdd 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -10,7 +10,7 @@ require ( github.com/docker/docker v28.2.2+incompatible github.com/docker/go-connections v0.5.0 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.99 + github.com/shuffle/shuffle-shared v0.9.2 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 ) diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index 13ae08b5..3944b4e5 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -142,8 +142,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0= github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4= -github.com/frikky/schemaless v0.0.16 h1:4d2ZktB9xGsAusbbKliOI8TuriSrdIMzD/6ToY3wkz8= -github.com/frikky/schemaless v0.0.16/go.mod h1:jT48kTcmr1q3o8i+8qe7g+eCsbwaz2Q9CjOJevQQzQs= +github.com/frikky/schemaless v0.0.17 h1:Tlg7td64r/EGAfQAL8LIlB8mOEnSRAgfJ8TERxzJSvM= +github.com/frikky/schemaless v0.0.17/go.mod h1:jT48kTcmr1q3o8i+8qe7g+eCsbwaz2Q9CjOJevQQzQs= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -324,8 +324,8 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.84 h1:ElIMQYjKBVOiadbiGkSzt/lPU5xqaQwRxQvk9wx/xYM= -github.com/shuffle/shuffle-shared v0.8.84/go.mod h1:RdfNxqCPI+zU4jQKy3E/p4Io2injm7LpSKQUCDHNtLk= +github.com/shuffle/shuffle-shared v0.9.2 h1:xl/dTNKWh9mol2XPaGXCUlUTqLuwrwLxlEuv7AINwkk= +github.com/shuffle/shuffle-shared v0.9.2/go.mod h1:PFLG4eaxz8zQc9Lbc0Ok/nG8rNsaBkZ0TNar+VU+J/g= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index ce7318d1..bf232b9d 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -90,7 +90,7 @@ var baseimageregistry = os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY") //var baseimagetagsuffix = os.Getenv("SHUFFLE_BASE_IMAGE_TAG_SUFFIX") -// Used for cloud with auth +// Used for cloud with auth. Onprem in certain cases too. var auth = os.Getenv("AUTH") var org = os.Getenv("ORG") diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 7b39554b..6cdc092d 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -10,7 +10,7 @@ require ( github.com/docker/docker v28.2.2+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.99 + github.com/shuffle/shuffle-shared v0.9.2 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 k8s.io/client-go v0.33.1 diff --git a/functions/onprem/worker/go.sum b/functions/onprem/worker/go.sum index 0ef2d3a6..b345e99e 100644 --- a/functions/onprem/worker/go.sum +++ b/functions/onprem/worker/go.sum @@ -142,8 +142,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0= github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4= -github.com/frikky/schemaless v0.0.16 h1:4d2ZktB9xGsAusbbKliOI8TuriSrdIMzD/6ToY3wkz8= -github.com/frikky/schemaless v0.0.16/go.mod h1:jT48kTcmr1q3o8i+8qe7g+eCsbwaz2Q9CjOJevQQzQs= +github.com/frikky/schemaless v0.0.17 h1:Tlg7td64r/EGAfQAL8LIlB8mOEnSRAgfJ8TERxzJSvM= +github.com/frikky/schemaless v0.0.17/go.mod h1:jT48kTcmr1q3o8i+8qe7g+eCsbwaz2Q9CjOJevQQzQs= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -326,8 +326,8 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.84 h1:ElIMQYjKBVOiadbiGkSzt/lPU5xqaQwRxQvk9wx/xYM= -github.com/shuffle/shuffle-shared v0.8.84/go.mod h1:RdfNxqCPI+zU4jQKy3E/p4Io2injm7LpSKQUCDHNtLk= +github.com/shuffle/shuffle-shared v0.9.2 h1:xl/dTNKWh9mol2XPaGXCUlUTqLuwrwLxlEuv7AINwkk= +github.com/shuffle/shuffle-shared v0.9.2/go.mod h1:PFLG4eaxz8zQc9Lbc0Ok/nG8rNsaBkZ0TNar+VU+J/g= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 86f72e00..ba81eeab 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -683,6 +683,10 @@ func deployk8sApp(image string, identifier string, env []string) error { return err } + // Giving the service time to start before we contineu anything + log.Printf("[DEBUG] Waiting 20 seconds before moving on to let app '%s' start properly. Service: %s (k8s)", name, image) + time.Sleep(20 * time.Second) + return nil } @@ -3135,7 +3139,7 @@ func findActiveSwarmNodes(dockercli *dockerclient.Client) (int64, error) { } /*** STARTREMOVE ***/ -func deploySwarmService(dockercli *dockerclient.Client, name, image string, deployport int) error { +func deploySwarmService(dockercli *dockerclient.Client, name, image string, deployport int, retry bool) error { log.Printf("[DEBUG] Deploying service for %s to swarm on port %d", name, deployport) //containerName := fmt.Sprintf("shuffle-worker-%s", parsedUuid) @@ -3296,6 +3300,21 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl _ = service if err != nil { + + if strings.Contains(fmt.Sprintf("%s", err), "network") && strings.Contains(fmt.Sprintf("%s", err), "not found") { + log.Printf("[DEBUG] Network %s not found. Trying to initialize it.", networkName) + networkErr := initSwarmNetwork() + if networkErr != nil { + log.Printf("[ERROR] Failed initializing swarm network: %s", err) + //return err + } + + // Retry deploying the service + if !retry { + return deploySwarmService(dockercli, name, image, deployport, true) + } + } + log.Printf("[DEBUG] Failed deploying %s with image %s: %s", name, image, err) return err } @@ -3351,13 +3370,108 @@ func findAppInfoKubernetes(image, name string, env []string) error { return err } -func findAppInfo(image, name string) (int, error) { +// Backups in case networks are removed +func initSwarmNetwork() error { + ctx := context.Background() dockercli, err := dockerclient.NewEnvClient() if err != nil { log.Printf("[ERROR] Unable to create docker client (2): %s", err) - return -1, err + return err } + // Create the network options with the specified MTU + options := make(map[string]string) + mtu := 1500 + options["com.docker.network.driver.mtu"] = fmt.Sprintf("%d", mtu) + + + ingressOptions := network.CreateOptions{ + Driver: "overlay", + Attachable: false, + Ingress: true, + IPAM: &network.IPAM{ + Driver: "default", + Config: []network.IPAMConfig{ + network.IPAMConfig{ + Subnet: "10.225.225.0/24", + Gateway: "10.225.225.1", + }, + }, + }, + } + + _, err = dockercli.NetworkCreate( + ctx, + "ingress", + ingressOptions, + ) + + if err != nil { + log.Printf("[WARNING] Ingress network may already exist: %s", err) + } + + //docker network create --driver=overlay workers + // Specific subnet? + networkName := "shuffle_swarm_executions" + if len(swarmNetworkName) > 0 { + networkName = swarmNetworkName + } + + networkCreateOptions := network.CreateOptions{ + Driver: "overlay", + Options: options, + Attachable: true, + Ingress: false, + IPAM: &network.IPAM{ + Driver: "default", + Config: []network.IPAMConfig{ + network.IPAMConfig{ + Subnet: "10.224.224.0/24", + Gateway: "10.224.224.1", + }, + }, + }, + } + _, err = dockercli.NetworkCreate( + ctx, + networkName, + networkCreateOptions, + ) + + if err != nil { + log.Printf("[WARNING] Swarm Executions network may already exist: %s", err) + } + + networkName = "shuffle-executions" + networkCreateOptions = network.CreateOptions{ + Driver: "overlay", + Options: options, + Attachable: true, + Ingress: false, + IPAM: &network.IPAM{ + Driver: "default", + Config: []network.IPAMConfig{ + network.IPAMConfig{ + Subnet: "10.223.223.0/24", + Gateway: "10.223.223.1", + }, + }, + }, + } + _, err = dockercli.NetworkCreate( + ctx, + networkName, + networkCreateOptions, + ) + + if err != nil { + log.Printf("[WARNING] Swarm Executions network may already exist: %s", err) + } + + return nil +} + +func findAppInfo(image, name string) (int, error) { highest := baseport exposedPort := -1 @@ -3379,6 +3493,12 @@ func findAppInfo(image, name string) (int, error) { //Filters: if exposedPort == -1 { + dockercli, err := dockerclient.NewEnvClient() + if err != nil { + log.Printf("[ERROR] Unable to create docker client (2): %s", err) + return -1, err + } + serviceListOptions := types.ServiceListOptions{} services, err := dockercli.ServiceList( context.Background(), @@ -3443,26 +3563,29 @@ func findAppInfo(image, name string) (int, error) { if exposedPort >= 0 { //log.Printf("[INFO] Found service %s on port %d - no need to deploy another", name, exposedPort) } else { + dockercli, err := dockerclient.NewEnvClient() + if err != nil { + log.Printf("[ERROR] Unable to create docker client (2): %s", err) + return -1, err + } + // Increment by 1 for highest port if highest <= baseport { highest = baseport } highest += 1 - err = deploySwarmService(dockercli, name, image, highest) + err = deploySwarmService(dockercli, name, image, highest, false) if err != nil { log.Printf("[WARNING] NOT Found service: %s. error: %s", name, err) return highest, err } else { - log.Printf("[DEBUG] Deployed app with name %s", name) + log.Printf("[DEBUG] Waiting 20 seconds before moving on to let app '%s' start properly. Service: %s", name, image) + time.Sleep(time.Duration(20) * time.Second) } exposedPort = highest - - if appsInitialized { - log.Printf("[DEBUG] Waiting 30 seconds before moving on to let app start") - time.Sleep(time.Duration(30) * time.Second) - } + //return exposedPort, errors.New("Deployed app %s") } return exposedPort, nil @@ -3652,7 +3775,7 @@ func sendAppRequest(ctx context.Context, incomingUrl, appName string, port int, func baseDeploy() { var cli *dockerclient.Client - var err error + //var err error if isKubernetes != "true" { cli, err := dockerclient.NewEnvClient() @@ -3714,8 +3837,7 @@ func baseDeploy() { //deployApp(cli, value, identifier, env, workflowExecution, action) log.Printf("[DEBUG] Deploying app with identifier %s to ensure basic apps are available from the get-go", identifier) - err = deployApp(cli, value, identifier, env, workflowExecution, action) - _ = err + go deployApp(cli, value, identifier, env, workflowExecution, action) //err := deployApp(cli, value, identifier, env, workflowExecution, action) //if err != nil { // log.Printf("[DEBUG] Failed deploying app %s: %s", value, err) From 85357f1b9b8096582d956d33420f4ee31380c18c Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Fri, 22 Aug 2025 12:40:03 +0000 Subject: [PATCH 33/59] simplified AI enable check for self hosted models --- backend/go-app/main.go | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index f57dcdcd..51a1a856 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -1135,6 +1135,8 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { } } + aiEnabled := os.Getenv("OPENAI_API_URL") != "" && os.Getenv("AI_MODEL") != "" + returnValue := shuffle.HandleInfo{ Success: true, Username: userInfo.Username, @@ -1159,7 +1161,7 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { ActiveApps: activatedAppIds, Theme: userInfo.Theme, OrgStatus: parsedStatus, - AIEnabled: org.LocalAIEnabled, + AIEnabled: aiEnabled, } returnData, err := json.Marshal(returnValue) @@ -5481,23 +5483,6 @@ func initHandlers() { r.Use(shuffle.RequestMiddleware) http.Handle("/", r) - - go func() { - log.Printf("[DEBUG] Starting AI availability check for all organizations") - activeOrgs, err := shuffle.GetAllOrgs(ctx) - if err != nil { - log.Printf("[ERROR] Error getting organizations for AI availability check: %s", err) - return - } - - for _, org := range activeOrgs { - if len(org.Id) == 0 { - continue - } - - go shuffle.CheckAndUpdateAIAvailability(ctx, org.Id) - } - }() } // Had to move away from mux, which means Method is fucked up right now. From c729baedc6bcce7199a9ad51005adac9f54eb8ef Mon Sep 17 00:00:00 2001 From: Frikky Date: Mon, 25 Aug 2025 21:41:17 +0200 Subject: [PATCH 34/59] Made sure SHUFFLE_BASE_IMAGE_REGISTRY is passed for k8s, and that registry/basepath/imagename:version now works --- functions/onprem/orborus/orborus.go | 25 +++++++++++++++++-------- functions/onprem/worker/worker.go | 17 +++++++++++++++++ 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index f04b72c9..1eee1411 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -85,9 +85,8 @@ var appContainerSecurityContext = os.Getenv("SHUFFLE_APP_CONTAINER_SECURITY_CONT // var baseimagename = "docker.pkg.github.com/shuffle/shuffle" // var baseimagename = "ghcr.io/frikky" // var baseimagename = "shuffle/shuffle" -var baseimagename = os.Getenv("SHUFFLE_BASE_IMAGE_NAME") var baseimageregistry = os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY") - +var baseimagename = os.Getenv("SHUFFLE_BASE_IMAGE_NAME") //var baseimagetagsuffix = os.Getenv("SHUFFLE_BASE_IMAGE_TAG_SUFFIX") // Used for cloud with auth. Onprem in certain cases too. @@ -1041,6 +1040,10 @@ func deployK8sWorker(image string, identifier string, env []string) error { env = append(env, fmt.Sprintf("KUBERNETES_SERVICE_PORT=%s", os.Getenv("KUBERNETES_SERVICE_PORT"))) } + if len(os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")) > 0 { + env = append(env, fmt.Sprintf("SHUFFLE_BASE_IMAGE_REGISTRY=%s", os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY"))) + } + if len(os.Getenv("REGISTRY_URL")) > 0 { env = append(env, fmt.Sprintf("REGISTRY_URL=%s", os.Getenv("REGISTRY_URL"))) } @@ -1100,8 +1103,13 @@ func deployK8sWorker(image string, identifier string, env []string) error { } } + // Required format: + // url/org/repo/appname:tag + // url/org/repo/appname:tag + + //env = append(env, fmt.Sprintf("SHUFFLE_SWARM_CONFIG=%s", swarmConfig)) env = append(env, fmt.Sprintf("BASE_URL=%s", baseUrl)) - env = append(env, fmt.Sprintf("SHUFFLE_SWARM_CONFIG=%s", swarmConfig)) + env = append(env, fmt.Sprintf("SHUFFLE_SWARM_CONFIG=run") env = append(env, fmt.Sprintf("WORKER_HOSTNAME=%s", "shuffle-workers")) if len(kubernetesNamespace) == 0 { @@ -2137,16 +2145,17 @@ func main() { if isKubernetes != "true" { deployServiceWorkers(workerImage) + + err := setBackendToSwarmNetwork(ctx) + if err != nil { + log.Printf("[WARNING] Failed setting backend to swarm network: %s", err) + } + } else { deployK8sWorker(workerImage, "shuffle-workers", []string{}) runString = "Run: \"kubectl get pods\" for more info" } - err := setBackendToSwarmNetwork(ctx) - if err != nil { - log.Printf("[WARNING] Failed setting backend to swarm network: %s", err) - } - log.Printf("[DEBUG] Waiting 45 seconds to ensure workers are deployed. %s", runString) time.Sleep(time.Duration(45) * time.Second) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index ba81eeab..a038c4ca 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -404,6 +404,8 @@ func deployk8sApp(image string, identifier string, env []string) error { kubernetesNamespace = "default" } + log.Printf("[DEBUG] Deploying k8s app with identifier %s to namespace %s", identifier, kubernetesNamespace) + deployport, err := strconv.Atoi(os.Getenv("SHUFFLE_APP_EXPOSED_PORT")) if err != nil { deployport = 80 @@ -866,6 +868,11 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] } else { // ** STARTREMOVE ***/ exposedPort = 80 + deployport, err := strconv.Atoi(os.Getenv("SHUFFLE_APP_EXPOSED_PORT")) + if err == nil { + exposedPort = deployport + } + err = findAppInfoKubernetes(image, appName, env) if err != nil { log.Printf("[ERROR] Failed finding and creating port for %s: %s", appName, err) @@ -1432,6 +1439,16 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { imageName = strings.ReplaceAll(imageName, " ", "-") } + // Kubernetes specific. + // Should it be though? + if isKubernetes == "true" { + // Map it to: + // /baseimagename/: + if len(localRegistry) > 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")) > 0 { + imageName = fmt.Sprintf("%s/%s/%s:%s", os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY"), baseimagename, parsedAppname, action.AppVersion) + } + } + askOtherWorkersToDownloadImage(imageName) // Added UUID to identifier just in case From ca68f4635d2e41e29e9978adc9961087d4da4e60 Mon Sep 17 00:00:00 2001 From: Frikky Date: Mon, 25 Aug 2025 21:44:54 +0200 Subject: [PATCH 35/59] Cleaned up worker and orborus to point to url properly --- functions/onprem/orborus/orborus.go | 2 +- functions/onprem/worker/worker.go | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 1eee1411..cb02b199 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -1109,7 +1109,7 @@ func deployK8sWorker(image string, identifier string, env []string) error { //env = append(env, fmt.Sprintf("SHUFFLE_SWARM_CONFIG=%s", swarmConfig)) env = append(env, fmt.Sprintf("BASE_URL=%s", baseUrl)) - env = append(env, fmt.Sprintf("SHUFFLE_SWARM_CONFIG=run") + env = append(env, fmt.Sprintf("SHUFFLE_SWARM_CONFIG=run")) env = append(env, fmt.Sprintf("WORKER_HOSTNAME=%s", "shuffle-workers")) if len(kubernetesNamespace) == 0 { diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index a038c4ca..3be11b04 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -1444,8 +1444,15 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { if isKubernetes == "true" { // Map it to: // /baseimagename/: - if len(localRegistry) > 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")) > 0 { - imageName = fmt.Sprintf("%s/%s/%s:%s", os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY"), baseimagename, parsedAppname, action.AppVersion) + localRegistry := os.Getenv("REGISTRY_URL") + if len(localRegistry) > 0 && len(baseimagename) > 0 { + + newImageName := fmt.Sprintf("%s/%s/%s:%s", localRegistry, baseimagename, parsedAppname, action.AppVersion) + + log.Printf("[INFO] Remapping image name %s to %s due to registry+image name existing on k8s", imageName, newImageName) + + imageName = newImageName + } } From 15bf1011e7eec40c07c1cc1694383deb691e39a4 Mon Sep 17 00:00:00 2001 From: Frikky Date: Tue, 26 Aug 2025 14:03:30 +0200 Subject: [PATCH 36/59] Bumped versions to match shuffle-shared and singul --- backend/go-app/go.mod | 22 ++++++------ backend/go-app/go.sum | 48 +++++++++++++------------ functions/onprem/orborus/go.mod | 20 ++++++----- functions/onprem/orborus/go.sum | 44 ++++++++++++----------- functions/onprem/worker/README.md | 17 +++++++++ functions/onprem/worker/go.mod | 30 ++++++++-------- functions/onprem/worker/go.sum | 59 +++++++++++++++++-------------- functions/onprem/worker/worker.go | 21 ++++++++++- 8 files changed, 157 insertions(+), 104 deletions(-) create mode 100644 functions/onprem/worker/README.md diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 7c857d01..d946307b 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -24,9 +24,9 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.0 - github.com/shuffle/singul v0.0.15 - golang.org/x/crypto v0.38.0 + github.com/shuffle/shuffle-shared v0.9.4 + github.com/shuffle/singul v0.0.16 + golang.org/x/crypto v0.40.0 google.golang.org/api v0.236.0 google.golang.org/grpc v1.72.2 gopkg.in/yaml.v3 v3.0.1 @@ -73,7 +73,7 @@ require ( github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/frikky/schemaless v0.0.18 // indirect + github.com/frikky/schemaless v0.0.20 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-jose/go-jose/v4 v4.0.5 // indirect @@ -115,12 +115,14 @@ require ( github.com/opencontainers/image-spec v1.1.1 // indirect github.com/opensearch-project/opensearch-go v1.1.0 // indirect github.com/opensearch-project/opensearch-go/v2 v2.3.0 // indirect + github.com/osteele/liquid v1.7.0 // indirect + github.com/osteele/tuesday v1.0.3 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pjbgf/sha1cd v0.3.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect - github.com/sashabaranov/go-openai v1.40.1 // indirect + github.com/sashabaranov/go-openai v1.40.5 // indirect github.com/sendgrid/rest v2.6.9+incompatible // indirect github.com/sendgrid/sendgrid-go v3.16.1+incompatible // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect @@ -143,12 +145,12 @@ require ( go.opentelemetry.io/otel/trace v1.36.0 // indirect go.uber.org/atomic v1.9.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/net v0.40.0 // indirect + golang.org/x/net v0.41.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sync v0.14.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/term v0.32.0 // indirect - golang.org/x/text v0.25.0 // indirect + golang.org/x/sync v0.16.0 // indirect + golang.org/x/sys v0.34.0 // indirect + golang.org/x/term v0.33.0 // indirect + golang.org/x/text v0.27.0 // indirect golang.org/x/time v0.11.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 // indirect diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 163f7931..f5c3834e 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -152,8 +152,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0= github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4= -github.com/frikky/schemaless v0.0.18 h1:w2kMDLGZD+1sJOQx5EycW/8Z5wo1p3JlV4kEFp1kWTY= -github.com/frikky/schemaless v0.0.18/go.mod h1:jT48kTcmr1q3o8i+8qe7g+eCsbwaz2Q9CjOJevQQzQs= +github.com/frikky/schemaless v0.0.20 h1:S/A2pQcRN9qa2RnufvxwCeM06trjG0JLTF3urt1tFQI= +github.com/frikky/schemaless v0.0.20/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= github.com/fsouza/go-dockerclient v1.12.1 h1:FMoLq+Zhv9Oz/rFmu6JWkImfr6CBgZOPcL+bHW4gS0o= github.com/fsouza/go-dockerclient v1.12.1/go.mod h1:OqsgJJcpCwqyM3JED7TdfM9QVWS5O7jSYwXxYKmOooY= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= @@ -329,6 +329,10 @@ github.com/opensearch-project/opensearch-go v1.1.0 h1:eG5sh3843bbU1itPRjA9QXbxcg github.com/opensearch-project/opensearch-go v1.1.0/go.mod h1:+6/XHCuTH+fwsMJikZEWsucZ4eZMma3zNSeLrTtVGbo= github.com/opensearch-project/opensearch-go/v2 v2.3.0 h1:nQIEMr+A92CkhHrZgUhcfsrZjibvB3APXf2a1VwCmMQ= github.com/opensearch-project/opensearch-go/v2 v2.3.0/go.mod h1:8LDr9FCgUTVoT+5ESjc2+iaZuldqE+23Iq0r1XeNue8= +github.com/osteele/liquid v1.7.0 h1:VsbPSchE5D5S5scylAIvERET4dnCxsO6IDri2oSJ5Dk= +github.com/osteele/liquid v1.7.0/go.mod h1:xU0Z2dn2hOQIEFEWNmeltOmCtfhtoW/2fCyiNQeNG+U= +github.com/osteele/tuesday v1.0.3 h1:SrCmo6sWwSgnvs1bivmXLvD7Ko9+aJvvkmDjB5G4FTU= +github.com/osteele/tuesday v1.0.3/go.mod h1:pREKpE+L03UFuR+hiznj3q7j3qB1rUZ4XfKejwWFF2M= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= @@ -349,8 +353,8 @@ github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4 github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= -github.com/sashabaranov/go-openai v1.40.1 h1:bJ08Iwct5mHBVkuvG6FEcb9MDTfsXdTYPGjYLRdeTEU= -github.com/sashabaranov/go-openai v1.40.1/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= +github.com/sashabaranov/go-openai v1.40.5 h1:SwIlNdWflzR1Rxd1gv3pUg6pwPc6cQ2uMoHs8ai+/NY= +github.com/sashabaranov/go-openai v1.40.5/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sendgrid/rest v2.6.9+incompatible h1:1EyIcsNdn9KIisLW50MKwmSRSK+ekueiEMJ7NEoxJo0= @@ -359,10 +363,10 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.9.0 h1:hUNtW3qHkji2cUVyR1cIXAz6qxpAHHP5hBmSAe3pIhs= -github.com/shuffle/shuffle-shared v0.9.0/go.mod h1:PFLG4eaxz8zQc9Lbc0Ok/nG8rNsaBkZ0TNar+VU+J/g= -github.com/shuffle/singul v0.0.15 h1:a5Qro1BKs9a+mWE289oYTkDeVFoKEd/H9qu9XJGsVxo= -github.com/shuffle/singul v0.0.15/go.mod h1:z1RRZsymTYxsB8WMQwwswypruy4j5EugGPFEZspTEA4= +github.com/shuffle/shuffle-shared v0.9.4 h1:xg8qjPgc1tCdIoM218LtQywNSs791m+Sn8F30pFb0W4= +github.com/shuffle/shuffle-shared v0.9.4/go.mod h1:klSHfahYORgrQS4AdG5RIfTaL4lakbJ+R9EXfc7QQo0= +github.com/shuffle/singul v0.0.16 h1:dW+0Mln9R1aUJ0fjikpWcxbjoQWqJHxe4kSxh2tQN5E= +github.com/shuffle/singul v0.0.16/go.mod h1:LYkp320A6gsoPlYbXUM+WvEPUVAuutlSsqnVKyRy4gs= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -441,8 +445,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= -golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= +golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= +golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -491,8 +495,8 @@ golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= -golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -508,8 +512,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= -golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -534,14 +538,14 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= -golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= +golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= +golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -552,8 +556,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= -golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= @@ -585,8 +589,8 @@ golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= -golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= +golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index c194ecdd..ce6e698d 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -10,7 +10,7 @@ require ( github.com/docker/docker v28.2.2+incompatible github.com/docker/go-connections v0.5.0 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.2 + github.com/shuffle/shuffle-shared v0.9.4 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 ) @@ -58,7 +58,7 @@ require ( github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/frikky/kin-openapi v0.42.0 // indirect - github.com/frikky/schemaless v0.0.17 // indirect + github.com/frikky/schemaless v0.0.20 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect @@ -95,11 +95,13 @@ require ( github.com/opencontainers/image-spec v1.1.1 // indirect github.com/opensearch-project/opensearch-go v1.1.0 // indirect github.com/opensearch-project/opensearch-go/v2 v2.3.0 // indirect + github.com/osteele/liquid v1.7.0 // indirect + github.com/osteele/tuesday v1.0.3 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pjbgf/sha1cd v0.3.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect - github.com/sashabaranov/go-openai v1.40.1 // indirect + github.com/sashabaranov/go-openai v1.40.5 // indirect github.com/sendgrid/rest v2.6.9+incompatible // indirect github.com/sendgrid/sendgrid-go v3.16.1+incompatible // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect @@ -121,13 +123,13 @@ require ( go.opentelemetry.io/otel/trace v1.36.0 // indirect go.opentelemetry.io/proto/otlp v1.5.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/crypto v0.38.0 // indirect - golang.org/x/net v0.40.0 // indirect + golang.org/x/crypto v0.40.0 // indirect + golang.org/x/net v0.41.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sync v0.14.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/term v0.32.0 // indirect - golang.org/x/text v0.25.0 // indirect + golang.org/x/sync v0.16.0 // indirect + golang.org/x/sys v0.34.0 // indirect + golang.org/x/term v0.33.0 // indirect + golang.org/x/text v0.27.0 // indirect golang.org/x/time v0.11.0 // indirect google.golang.org/api v0.236.0 // indirect google.golang.org/appengine v1.6.8 // indirect diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index 3944b4e5..252f0afb 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -142,8 +142,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0= github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4= -github.com/frikky/schemaless v0.0.17 h1:Tlg7td64r/EGAfQAL8LIlB8mOEnSRAgfJ8TERxzJSvM= -github.com/frikky/schemaless v0.0.17/go.mod h1:jT48kTcmr1q3o8i+8qe7g+eCsbwaz2Q9CjOJevQQzQs= +github.com/frikky/schemaless v0.0.20 h1:S/A2pQcRN9qa2RnufvxwCeM06trjG0JLTF3urt1tFQI= +github.com/frikky/schemaless v0.0.20/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -299,6 +299,10 @@ github.com/opensearch-project/opensearch-go v1.1.0 h1:eG5sh3843bbU1itPRjA9QXbxcg github.com/opensearch-project/opensearch-go v1.1.0/go.mod h1:+6/XHCuTH+fwsMJikZEWsucZ4eZMma3zNSeLrTtVGbo= github.com/opensearch-project/opensearch-go/v2 v2.3.0 h1:nQIEMr+A92CkhHrZgUhcfsrZjibvB3APXf2a1VwCmMQ= github.com/opensearch-project/opensearch-go/v2 v2.3.0/go.mod h1:8LDr9FCgUTVoT+5ESjc2+iaZuldqE+23Iq0r1XeNue8= +github.com/osteele/liquid v1.7.0 h1:VsbPSchE5D5S5scylAIvERET4dnCxsO6IDri2oSJ5Dk= +github.com/osteele/liquid v1.7.0/go.mod h1:xU0Z2dn2hOQIEFEWNmeltOmCtfhtoW/2fCyiNQeNG+U= +github.com/osteele/tuesday v1.0.3 h1:SrCmo6sWwSgnvs1bivmXLvD7Ko9+aJvvkmDjB5G4FTU= +github.com/osteele/tuesday v1.0.3/go.mod h1:pREKpE+L03UFuR+hiznj3q7j3qB1rUZ4XfKejwWFF2M= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= @@ -314,8 +318,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= -github.com/sashabaranov/go-openai v1.40.1 h1:bJ08Iwct5mHBVkuvG6FEcb9MDTfsXdTYPGjYLRdeTEU= -github.com/sashabaranov/go-openai v1.40.1/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= +github.com/sashabaranov/go-openai v1.40.5 h1:SwIlNdWflzR1Rxd1gv3pUg6pwPc6cQ2uMoHs8ai+/NY= +github.com/sashabaranov/go-openai v1.40.5/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sendgrid/rest v2.6.9+incompatible h1:1EyIcsNdn9KIisLW50MKwmSRSK+ekueiEMJ7NEoxJo0= @@ -324,8 +328,8 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.9.2 h1:xl/dTNKWh9mol2XPaGXCUlUTqLuwrwLxlEuv7AINwkk= -github.com/shuffle/shuffle-shared v0.9.2/go.mod h1:PFLG4eaxz8zQc9Lbc0Ok/nG8rNsaBkZ0TNar+VU+J/g= +github.com/shuffle/shuffle-shared v0.9.4 h1:xg8qjPgc1tCdIoM218LtQywNSs791m+Sn8F30pFb0W4= +github.com/shuffle/shuffle-shared v0.9.4/go.mod h1:klSHfahYORgrQS4AdG5RIfTaL4lakbJ+R9EXfc7QQo0= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -402,8 +406,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= -golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= +golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= +golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -452,8 +456,8 @@ golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= -golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -469,8 +473,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= -golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -495,14 +499,14 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= -golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= +golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= +golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -513,8 +517,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= -golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= @@ -546,8 +550,8 @@ golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= -golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= +golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/functions/onprem/worker/README.md b/functions/onprem/worker/README.md new file mode 100644 index 00000000..ba6c2a9d --- /dev/null +++ b/functions/onprem/worker/README.md @@ -0,0 +1,17 @@ +# Worker +A worker implementation in Golang. This runs ALL Shuffle workflows onprem. In general receives jobs from Orborus. + +## Development +The ideal way to test the Worker is with a single workflow execution, standalone. Here are some environment variables you can use: + +``` +# Control what to run +export ENVIRONMENT_NAME="" +export SHUFFLE_CLOUDRUN_URL="https://shuffler.io" +export AUTHORIZATION="" +export EXECUTIONID="" + +# Control debugging and shutdown mechanisms +export DEBUG="true" +export SHUFFLE_WORKER_SHUTDOWN_DISABLED="true" +``` diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 6cdc092d..48b8cd12 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -5,12 +5,14 @@ go 1.24.0 toolchain go1.24.4 //replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared +//replace github.com/shuffle/singul => ../../../../singul require ( github.com/docker/docker v28.2.2+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.2 + github.com/shuffle/shuffle-shared v0.9.4 + github.com/shuffle/singul v0.0.16 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 k8s.io/client-go v0.33.1 @@ -32,7 +34,6 @@ require ( cloud.google.com/go/scheduler v1.11.7 // indirect cloud.google.com/go/storage v1.55.0 // indirect dario.cat/mergo v1.0.0 // indirect - github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect @@ -43,7 +44,6 @@ require ( github.com/algolia/algoliasearch-client-go/v3 v3.31.4 // indirect github.com/bradfitz/gomemcache v0.0.0-20250403215159-8d39553ac7cf // indirect github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 // indirect - github.com/cenkalti/backoff/v5 v5.0.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudflare/circl v1.6.1 // indirect github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 // indirect @@ -60,7 +60,7 @@ require ( github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/frikky/kin-openapi v0.42.0 // indirect - github.com/frikky/schemaless v0.0.17 // indirect + github.com/frikky/schemaless v0.0.20 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect @@ -83,13 +83,13 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect github.com/googleapis/gax-go/v2 v2.14.2 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect - github.com/moby/sys/sequential v0.6.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect @@ -97,17 +97,19 @@ require ( github.com/opencontainers/image-spec v1.1.1 // indirect github.com/opensearch-project/opensearch-go v1.1.0 // indirect github.com/opensearch-project/opensearch-go/v2 v2.3.0 // indirect + github.com/osteele/liquid v1.7.0 // indirect + github.com/osteele/tuesday v1.0.3 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pjbgf/sha1cd v0.3.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect - github.com/sashabaranov/go-openai v1.40.1 // indirect + github.com/sashabaranov/go-openai v1.40.5 // indirect github.com/sendgrid/rest v2.6.9+incompatible // indirect github.com/sendgrid/sendgrid-go v3.16.1+incompatible // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/skeema/knownhosts v1.3.1 // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/pflag v1.0.6 // indirect github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect @@ -116,20 +118,18 @@ require ( go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect go.opentelemetry.io/otel v1.36.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 // indirect go.opentelemetry.io/otel/metric v1.36.0 // indirect go.opentelemetry.io/otel/sdk v1.36.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect go.opentelemetry.io/otel/trace v1.36.0 // indirect - go.opentelemetry.io/proto/otlp v1.5.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/crypto v0.38.0 // indirect - golang.org/x/net v0.40.0 // indirect + golang.org/x/crypto v0.40.0 // indirect + golang.org/x/net v0.41.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sync v0.14.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/term v0.32.0 // indirect - golang.org/x/text v0.25.0 // indirect + golang.org/x/sync v0.16.0 // indirect + golang.org/x/sys v0.34.0 // indirect + golang.org/x/term v0.33.0 // indirect + golang.org/x/text v0.27.0 // indirect golang.org/x/time v0.11.0 // indirect google.golang.org/api v0.236.0 // indirect google.golang.org/appengine v1.6.8 // indirect diff --git a/functions/onprem/worker/go.sum b/functions/onprem/worker/go.sum index b345e99e..f118b94f 100644 --- a/functions/onprem/worker/go.sum +++ b/functions/onprem/worker/go.sum @@ -142,8 +142,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0= github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4= -github.com/frikky/schemaless v0.0.17 h1:Tlg7td64r/EGAfQAL8LIlB8mOEnSRAgfJ8TERxzJSvM= -github.com/frikky/schemaless v0.0.17/go.mod h1:jT48kTcmr1q3o8i+8qe7g+eCsbwaz2Q9CjOJevQQzQs= +github.com/frikky/schemaless v0.0.20 h1:S/A2pQcRN9qa2RnufvxwCeM06trjG0JLTF3urt1tFQI= +github.com/frikky/schemaless v0.0.20/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -241,8 +241,8 @@ github.com/googleapis/gax-go/v2 v2.14.2 h1:eBLnkZ9635krYIPD+ag1USrOAI0Nr0QYF3+/3 github.com/googleapis/gax-go/v2 v2.14.2/go.mod h1:ON64QhlJkhVtSqp4v1uaK92VyZ2gmvDQsweuyLV+8+w= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1 h1:VNqngBF40hVlDloBruUehVYC3ArSgIyScOAyMRqBxRg= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1/go.mod h1:RBRO7fro65R6tjKzYgLAFo0t1QEXY1Dp+i/bvpRiqiQ= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -301,6 +301,10 @@ github.com/opensearch-project/opensearch-go v1.1.0 h1:eG5sh3843bbU1itPRjA9QXbxcg github.com/opensearch-project/opensearch-go v1.1.0/go.mod h1:+6/XHCuTH+fwsMJikZEWsucZ4eZMma3zNSeLrTtVGbo= github.com/opensearch-project/opensearch-go/v2 v2.3.0 h1:nQIEMr+A92CkhHrZgUhcfsrZjibvB3APXf2a1VwCmMQ= github.com/opensearch-project/opensearch-go/v2 v2.3.0/go.mod h1:8LDr9FCgUTVoT+5ESjc2+iaZuldqE+23Iq0r1XeNue8= +github.com/osteele/liquid v1.7.0 h1:VsbPSchE5D5S5scylAIvERET4dnCxsO6IDri2oSJ5Dk= +github.com/osteele/liquid v1.7.0/go.mod h1:xU0Z2dn2hOQIEFEWNmeltOmCtfhtoW/2fCyiNQeNG+U= +github.com/osteele/tuesday v1.0.3 h1:SrCmo6sWwSgnvs1bivmXLvD7Ko9+aJvvkmDjB5G4FTU= +github.com/osteele/tuesday v1.0.3/go.mod h1:pREKpE+L03UFuR+hiznj3q7j3qB1rUZ4XfKejwWFF2M= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= @@ -316,8 +320,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= -github.com/sashabaranov/go-openai v1.40.1 h1:bJ08Iwct5mHBVkuvG6FEcb9MDTfsXdTYPGjYLRdeTEU= -github.com/sashabaranov/go-openai v1.40.1/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= +github.com/sashabaranov/go-openai v1.40.5 h1:SwIlNdWflzR1Rxd1gv3pUg6pwPc6cQ2uMoHs8ai+/NY= +github.com/sashabaranov/go-openai v1.40.5/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sendgrid/rest v2.6.9+incompatible h1:1EyIcsNdn9KIisLW50MKwmSRSK+ekueiEMJ7NEoxJo0= @@ -326,8 +330,10 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.9.2 h1:xl/dTNKWh9mol2XPaGXCUlUTqLuwrwLxlEuv7AINwkk= -github.com/shuffle/shuffle-shared v0.9.2/go.mod h1:PFLG4eaxz8zQc9Lbc0Ok/nG8rNsaBkZ0TNar+VU+J/g= +github.com/shuffle/shuffle-shared v0.9.4 h1:xg8qjPgc1tCdIoM218LtQywNSs791m+Sn8F30pFb0W4= +github.com/shuffle/shuffle-shared v0.9.4/go.mod h1:klSHfahYORgrQS4AdG5RIfTaL4lakbJ+R9EXfc7QQo0= +github.com/shuffle/singul v0.0.16 h1:dW+0Mln9R1aUJ0fjikpWcxbjoQWqJHxe4kSxh2tQN5E= +github.com/shuffle/singul v0.0.16/go.mod h1:LYkp320A6gsoPlYbXUM+WvEPUVAuutlSsqnVKyRy4gs= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -335,8 +341,8 @@ github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnB github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -393,8 +399,8 @@ go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFw go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= -go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= -go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= +go.opentelemetry.io/proto/otlp v1.6.0 h1:jQjP+AQyTf+Fe7OKj/MfkDrmK4MNVtw2NpXsf9fefDI= +go.opentelemetry.io/proto/otlp v1.6.0/go.mod h1:cicgGehlFuNdgZkcALOCh3VE6K/u2tAjzlRhDwmVpZc= go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -404,8 +410,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= -golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= +golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= +golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -454,8 +460,8 @@ golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= -golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -471,8 +477,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= -golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -491,20 +497,19 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= -golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= +golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= +golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -515,8 +520,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= -golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= @@ -548,8 +553,8 @@ golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= -golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= +golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 3be11b04..35314834 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -2,6 +2,7 @@ package main import ( "github.com/shuffle/shuffle-shared" + "github.com/shuffle/singul/pkg" "bytes" "context" @@ -57,6 +58,7 @@ var logsDisabled = os.Getenv("SHUFFLE_LOGS_DISABLED") var cleanupEnv = strings.ToLower(os.Getenv("CLEANUP")) var swarmNetworkName = os.Getenv("SHUFFLE_SWARM_NETWORK_NAME") var dockerApiVersion = strings.ToLower(os.Getenv("DOCKER_API_VERSION")) +var shutdownDisabled = strings.ToLower(os.Getenv("SHUFFLE_WORKER_SHUTDOWN_DISABLED")) // Kubernetes settings var appServiceAccountName = os.Getenv("SHUFFLE_APP_SERVICE_ACCOUNT_NAME") @@ -289,7 +291,13 @@ func setWorkflowExecution(ctx context.Context, workflowExecution shuffle.Workflo // removes every container except itself (worker) func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason string, handleResultSend bool) { log.Printf("[DEBUG][%s] Shutdown (%s) started with reason %#v. Result amount: %d. ResultsSent: %d, Send result: %#v, Parent: %#v", workflowExecution.ExecutionId, workflowExecution.Status, reason, len(workflowExecution.Results), requestsSent, handleResultSend, workflowExecution.ExecutionParent) - //reason := "Error in execution" + + // This is an escape hatch for development only + // Typically meant to be used when you aren't sure how to make a workflow run in bad scenarios, and want to rapidly debug it. + if shutdownDisabled == "true" { + log.Printf("[ERROR] Shutdown disabled: NOT shutting down. This should ONLY be used for development & debugging.") + os.Exit(3) + } sleepDuration := 1 if handleResultSend && requestsSent < 2 { @@ -1433,7 +1441,15 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { appname = strings.Replace(appname, ".", "-", -1) appversion = strings.Replace(appversion, ".", "-", -1) + action, _ = singul.HandleSingulStartnode(workflowExecution, action, []string{}) + parsedAppname := strings.Replace(strings.ToLower(action.AppName), " ", "-", -1) + //if strings.ToLower(parsedAppname) == "singul" { + // parsedAppname = "shuffle-ai" + // appversion = "1.0.0" + // appname = "shuffle-ai" + //} + imageName := fmt.Sprintf("%s:%s_%s", baseimagename, parsedAppname, action.AppVersion) if strings.Contains(imageName, " ") { imageName = strings.ReplaceAll(imageName, " ", "-") @@ -4181,6 +4197,9 @@ func main() { if os.Getenv("DEBUG") == "true" { debug = true + + log.Printf("[INFO] Disabled cleanup due to debug mode (DEBUG=true)") + cleanupEnv = "false" } /*** STARTREMOVE ***/ From 6c60dd3bccac42e3f6a6d84fe1d7a5a30ef5289f Mon Sep 17 00:00:00 2001 From: Frikky Date: Tue, 26 Aug 2025 16:07:44 +0200 Subject: [PATCH 37/59] Merging fixes for 2.1.0 --- frontend/src/components/Billing.jsx | 155 +++- frontend/src/components/BillingStats.jsx | 254 +++--- frontend/src/components/CacheView.jsx | 114 ++- .../src/components/CollectIngestModal.jsx | 185 +++-- frontend/src/components/EditWorkflow.jsx | 258 +++++- frontend/src/components/LicencePopup.jsx | 6 +- frontend/src/components/Navbar.jsx | 16 +- .../src/components/OrgHeaderexpandedNew.jsx | 8 +- frontend/src/components/ParsedAction.jsx | 72 +- frontend/src/components/PartnerDetails.jsx | 33 +- frontend/src/components/PartnerSettings.jsx | 83 +- frontend/src/components/PartnerTab.jsx | 5 +- .../src/components/ShuffleCodeEditor1.jsx | 230 +++++- frontend/src/views/AgentUI.jsx | 18 +- frontend/src/views/AngularWorkflow.jsx | 759 +++++++++++++++--- frontend/src/views/AppExplorer.jsx | 49 +- frontend/src/views/Apps2.jsx | 28 +- frontend/src/views/Docs.jsx | 23 +- frontend/src/views/LoginPage.jsx | 4 +- frontend/src/views/Workflows2.jsx | 413 +++++++++- 20 files changed, 2115 insertions(+), 598 deletions(-) diff --git a/frontend/src/components/Billing.jsx b/frontend/src/components/Billing.jsx index 19698a13..8b14eaf5 100644 --- a/frontend/src/components/Billing.jsx +++ b/frontend/src/components/Billing.jsx @@ -91,10 +91,13 @@ const Billing = memo((props) => { const [currentTab, setCurrentTab] = useState(0) const [allChildOrgs, setAllChildOrgs] = useState([]) const [allChildOrgsStats, setAllChildOrgsStats] = useState([]) + const [statistics, setStatistics] = useState([]) + const [monthlyAppRunsParent, setMonthlyAppRunsParent] = useState(0) + const [monthlyAllSuborgExecutions, setMonthlyAllSuborgExecutions] = useState(0) useEffect(() => { - if (userdata.app_execution_limit !== undefined && userdata.app_execution_usage !== undefined) { - const percentage = ((userdata.app_execution_usage + userdata.app_executions_suborgs) / userdata.app_execution_limit) * 100; + if (monthlyAppRunsParent > 0 || monthlyAllSuborgExecutions > 0) { + const percentage = ((monthlyAppRunsParent + monthlyAllSuborgExecutions) / userdata.app_execution_limit) * 100; setCurrentAppRunsInPercentage(Math.round(percentage)); setCurrentAppRunsInNumber(userdata.app_execution_limit - userdata.app_execution_usage - userdata.app_executions_suborgs); } @@ -102,7 +105,7 @@ const Billing = memo((props) => { if (userdata?.id?.length > 0 && isLoggedIn === false){ setIsLoggedIn(true) } - }, [userdata]); + }, [monthlyAppRunsParent, monthlyAllSuborgExecutions, userdata]); const [BillingEmail, setBillingEmail] = useState(selectedOrganization?.Billing?.Email); @@ -199,6 +202,48 @@ const Billing = memo((props) => { } }, []) + + const getStats = (orgid) => { + + if (orgid === undefined || orgid === null) { + return + } + + fetch(`${globalUrl}/api/v1/orgs/${orgid}/stats`, { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for workflows :O!: ", response.status); + return; + } + + return response.json(); + }) + .then((responseJson) => { + if (responseJson["success"] === false) { + return + } + + setStatistics(responseJson); + }) + .catch((error) => { + console.log("error: ", error) + }); + } + + + useEffect(() => { + if (selectedOrganization && selectedOrganization?.id?.length > 0) { + getStats(selectedOrganization.id); + } + }, [selectedOrganization]); + const paperStyle = { padding: 20, // maxWidth: 400, @@ -1973,7 +2018,13 @@ const Billing = memo((props) => { }; const isChildOrg = userdata?.active_org?.creator_org !== "" && userdata?.active_org?.creator_org !== undefined && userdata?.active_org?.creator_org !== null - + + useEffect(() => { + if (isChildOrg && currentTab === 0) { + setCurrentTab(1); + } + }, [isChildOrg, currentTab]); + return (
@@ -2373,8 +2424,8 @@ const Billing = memo((props) => { We offer priority support through consultations and training to help you make the most of our product. If you have any questions, please reach out to us at {supportEmail}. We offer priority support through consultations and training to help you make the most of our product. If you have any questions, please reach out to us at -
- {billingInfo.subscription !== undefined && billingInfo.subscription !== null ? ( +
+ {/* {billingInfo.subscription !== undefined && billingInfo.subscription !== null ? ( isChildOrg ? null : ( { selectedOrganization={selectedOrganization} /> ) - ) : null} + ) : null} */}
@@ -2410,17 +2461,17 @@ const Billing = memo((props) => { }} /> - You have used {currentAppRunsInPercentage}% of total app execution limit or {userdata.app_execution_usage + userdata.app_executions_suborgs} app runs out of {userdata.app_execution_limit} app runs. + You have used {currentAppRunsInPercentage}% of total app execution limit or {Number(monthlyAppRunsParent ?? 0) + Number(monthlyAllSuborgExecutions ?? 0)} app runs out of {userdata.app_execution_limit} app runs this month. {userdata?.active_org?.creator_org?.length > 0 ? null : ( <> - Parent Organization App Executions: {userdata.app_execution_usage} + Parent Organization App Executions: {monthlyAppRunsParent} - Sub-Organization App Executions: {userdata.app_executions_suborgs || "N/A"} + Sub-Organization App Executions: {monthlyAllSuborgExecutions || "N/A"} )} @@ -2602,12 +2653,7 @@ const Billing = memo((props) => { { - setCurrentTab(-1) - - // Force re-render - setTimeout(() => { - setCurrentTab(newValue) - }, 100); + setCurrentTab(newValue) }} style={{ marginTop: 20 }} TabIndicatorProps={{ @@ -2619,52 +2665,66 @@ const Billing = memo((props) => { } }} > - } + - - {isCloud ? - - : null} - + {isChildOrg ? null : + />} + {isCloud ? + + : null}
- {currentTab === 0 ? -
- -
+ { + currentTab === 0 ? + : currentTab === 1 ? -
+
- : + : currentTab === 2 ? { setAllChildOrgs={setAllChildOrgs} allChildOrgsStats={allChildOrgsStats} setAllChildOrgsStats={setAllChildOrgsStats} + currentTab={currentTab} /> + : + }
diff --git a/frontend/src/components/BillingStats.jsx b/frontend/src/components/BillingStats.jsx index 7bd33476..8e011d42 100644 --- a/frontend/src/components/BillingStats.jsx +++ b/frontend/src/components/BillingStats.jsx @@ -45,6 +45,12 @@ const AppStats = (defaultprops) => { inputWorkflows, clickedFromOrgTab, syncStats, + statistics, + monthlyAppRunsParent, + setMonthlyAppRunsParent, + monthlyAllSuborgExecutions, + setMonthlyAllSuborgExecutions, + currentTab } = defaultprops; const [keys, setKeys] = useState([]) @@ -57,7 +63,6 @@ const AppStats = (defaultprops) => { const [endTime, setEndTime] = useState("") const [startTime, setStartTime] = useState("") - const [statistics, setStatistics] = useState(undefined); const [filteredStatistics, setFilteredStatistics] = useState(undefined); const [apprunCost, setApprunCost] = useState(0) @@ -78,6 +83,11 @@ const AppStats = (defaultprops) => { } }, []) + useEffect(() => { + if (statistics && statistics?.org_id?.length > 0) { + handleDataSetting(statistics, "day") + } +}, [statistics]) const getWorkflowStats = async (workflow, startTime, endTime) => { @@ -219,6 +229,7 @@ const AppStats = (defaultprops) => { const statKey = syncStats === true ? "onprem_stats" : "daily_statistics" if (statistics[statKey] === undefined || statistics[statKey] === null) { setFilteredStatistics(statistics) + setMonthlyAppRunsParent(statistics["monthly_app_executions"] ?? 0) return } @@ -265,20 +276,27 @@ const AppStats = (defaultprops) => { } } - // Make a date at the 1st of the current month + // Make a date at the 1st of the current month - only when no start time is selected var foundstarttime = (new Date()) - foundstarttime.setDate(1) if (startTime !== "" && startTime !== undefined && startTime !== null) { - foundstarttime = startTime + foundstarttime = new Date(startTime) + // Set to start of day to include the entire start date + foundstarttime.setHours(0, 0, 0, 0) + } else { + // Default to 1st of current month when no start time is selected + foundstarttime.setDate(1) + foundstarttime.setHours(0, 0, 0, 0) } - // Set to tomorrow by default + // Set end time properly var foundendtime = (new Date()) - foundendtime.setDate(foundendtime.getDate() + 1) - - // Check if endtime is after the daily statistics["date"] string if (endTime !== "" && endTime !== undefined && endTime !== null) { - foundendtime = endTime + foundendtime = new Date(endTime) + // Set to end of day to include the entire end date + foundendtime.setHours(23, 59, 59, 999) + } else { + // Default to current date when no end time is selected + foundendtime.setHours(23, 59, 59, 999) } // Check if start time is before the daily statistics["date"] string @@ -290,8 +308,20 @@ const AppStats = (defaultprops) => { } const date = new Date(item["date"]) - if (date >= foundstarttime) { - if (date <= foundendtime) { + // Normalize the date to start of day for comparison + const normalizedDate = new Date(date) + normalizedDate.setHours(0, 0, 0, 0) + + // Normalize foundstarttime for comparison + const normalizedStartTime = new Date(foundstarttime) + normalizedStartTime.setHours(0, 0, 0, 0) + + // Normalize foundendtime for comparison + const normalizedEndTime = new Date(foundendtime) + normalizedEndTime.setHours(0, 0, 0, 0) + + if (normalizedDate >= normalizedStartTime) { + if (normalizedDate <= normalizedEndTime) { newlist.push(item) } } @@ -326,6 +356,10 @@ const AppStats = (defaultprops) => { workflowexecutions += item["workflow_executions"] appexecutions += item["app_executions"] + if (currentTab === 0) { + appexecutions += (item["child_app_executions"] ?? 0) + } + estimatedcost += (item["app_executions"] * invocationCost) } @@ -343,7 +377,16 @@ const AppStats = (defaultprops) => { } setFilteredStatistics(tmpstats) - handleDataSetting(tmpstats, "day") + handleDataSetting(tmpstats, "day") + // if we have done monthly reset than only show monthly app runs as current month app run + const currentMonth = new Date().getMonth() + 1 + if (!monthlyAppRunsParent && statistics["monthly_app_executions"] > 0 && currentMonth === statistics["last_monthly_reset_month"]) { + setMonthlyAppRunsParent(statistics["monthly_app_executions"]) + } + + if (!monthlyAllSuborgExecutions && statistics["monthly_child_app_executions"]> 0 && currentMonth === statistics["last_monthly_reset_month"]) { + setMonthlyAllSuborgExecutions(statistics["monthly_child_app_executions"]) + } if (workflows !== undefined && workflows !== null && workflows.length > 0) { @@ -429,7 +472,7 @@ const AppStats = (defaultprops) => { if (item["child_app_executions"] !== undefined && item["child_app_executions"] !== null) { childorgappRuns["data"].push({ key: new Date(item["date"]), - data: inputdata["child_app_executions"] + data: item["child_app_executions"] }) } @@ -449,40 +492,45 @@ const AppStats = (defaultprops) => { } } - // Adds data for today - if (inputdata["daily_app_executions"] !== undefined && inputdata["daily_app_executions"] !== null) { - appRuns["data"].push({ - key: new Date(), - data: inputdata["daily_app_executions"] - }) + // Only add today's data if endTime is not set or if today falls within the selected date range + const today = new Date() + const shouldAddTodayData = endTime === "" || endTime === undefined || endTime === null || + (new Date(endTime) >= today.setHours(0, 0, 0, 0)) - appcostRuns["data"].push({ - key: new Date(), - data: (inputdata["daily_app_executions"] * invocationCost).toFixed(2) - }) - } + if (shouldAddTodayData) { + // Adds data for today + if (inputdata["daily_app_executions"] !== undefined && inputdata["daily_app_executions"] !== null) { + appRuns["data"].push({ + key: new Date(), + data: inputdata["daily_app_executions"] + }) - if (inputdata["daily_child_app_executions"] !== undefined && inputdata["daily_app_executions"] !== null) { - childorgappRuns["data"].push({ - key: new Date(), - data: inputdata["daily_child_app_executions"] - }) + appcostRuns["data"].push({ + key: new Date(), + data: (inputdata["daily_app_executions"] * invocationCost).toFixed(2) + }) + } - //setApprunCosts(appcostRuns) - } + if (inputdata["daily_child_app_executions"] !== undefined && inputdata["daily_app_executions"] !== null) { + childorgappRuns["data"].push({ + key: new Date(), + data: inputdata["daily_child_app_executions"] + }) + } - if (inputdata["daily_workflow_executions"] !== undefined && inputdata["daily_workflow_executions"] !== null) { - workflowRuns["data"].push({ - key: new Date(), - data: inputdata["daily_workflow_executions"] - }) - } + if (inputdata["daily_workflow_executions"] !== undefined && inputdata["daily_workflow_executions"] !== null) { + workflowRuns["data"].push({ + key: new Date(), + data: inputdata["daily_workflow_executions"] + }) + } - if (inputdata["daily_subflow_executions"] !== undefined && inputdata["daily_subflow_executions"] !== null) { - subflowRuns["data"].push({ - key: new Date(), - data: inputdata["daily_subflow_executions"] - }) + if (inputdata["daily_subflow_executions"] !== undefined && inputdata["daily_subflow_executions"] !== null) { + subflowRuns["data"].push({ + key: new Date(), + data: inputdata["daily_subflow_executions"] + }) + } } // Only for parent orgs @@ -494,49 +542,8 @@ const AppStats = (defaultprops) => { setWorkflowRuns(workflowRuns) setAppruns(appRuns) setApprunCosts(appcostRuns) - } - - const getStats = (orgid) => { - - if (orgid === undefined || orgid === null) { - return - } - - fetch(`${globalUrl}/api/v1/orgs/${orgid}/stats`, { - method: "GET", - headers: { - "Content-Type": "application/json", - Accept: "application/json", - }, - credentials: "include", - }) - .then((response) => { - if (response.status !== 200) { - console.log("Status not 200 for workflows :O!: ", response.status); - return; - } - - return response.json(); - }) - .then((responseJson) => { - if (responseJson["success"] === false) { - return - } - - setStatistics(responseJson) - handleDataSetting(responseJson, "day") - }) - .catch((error) => { - console.log("error: ", error) - }); } - useEffect(() => { - if(selectedOrganization?.id?.length > 0) { - getStats(selectedOrganization.id) - } - }, [selectedOrganization]) - const paperStyle = { textAlign: "center", padding: "40px", @@ -656,15 +663,19 @@ const AppStats = (defaultprops) => { ] const data = ( -
+
All shown statistics are gathered from Your Organisation Statistics. - It exists to give you more insight into your workflows, and to understand your utilization of the Shuffle platform. The billing tracker is in Beta, and is always calculated manually before being invoiced. + {currentTab === 0 ? + + All Organization app runs are calculated base on addition of parent org app runs + all child org app runs. + : It exists to give you more insight into your workflows, and to + understand your utilization of the Shuffle platform.{" "}}
{syncStats !== true ? null : "PS: You are currently looking at data from your onprem synced org"} @@ -675,7 +686,7 @@ const AppStats = (defaultprops) => { {filteredStatistics !== undefined ?
- {syncStats == true ? null : + {/* {syncStats == true ? null : The cost of app runs in the selected period based on {filteredStatistics.monthly_app_executions} App Runs. These numbers do not exclude your included 10.000/month or {includedExecutions} App Runs per month. App Run cost: ${invocationCost}. @@ -695,7 +706,7 @@ const AppStats = (defaultprops) => { - } + } */} {syncStats === true ? null : { } - {syncStats === true ? null : + {syncStats === true || currentTab === 0 ? null : Workflow runs in the selected period @@ -731,7 +742,7 @@ const AppStats = (defaultprops) => { } - {syncStats === true ? null : + {/* {syncStats === true ? null : Estimated cost to be billed at the end of the current month. Subtracted contractually included app runs. Actual cost month to date: ${monthToDateCost}. App Run cost: ${invocationCost}. @@ -746,15 +757,9 @@ const AppStats = (defaultprops) => { - } -
- : null} -
- - - {clickedFromOrgTab? ( - -
+ } */} + +
{
- ):( - -
- } - /> - } - />
- - )} + : null} +
{appRuns === undefined ? null : - + } - {childOrgsAppRuns === undefined ? + {childOrgsAppRuns === undefined || currentTab === 1 ? null : - + } - {workflowRuns === undefined ? + {workflowRuns === undefined || currentTab === 0? null : - + } - {subflowRuns === undefined ? + {subflowRuns === undefined || currentTab === 0 ? null : - + } {/*appRunCosts === undefined ? @@ -944,7 +918,7 @@ const AppStats = (defaultprops) => { */} - {syncStats === true ? null : + {syncStats === true || currentTab === 0 ? null :
{resultLoading ?
@@ -1000,7 +974,7 @@ const AppStats = (defaultprops) => { ) const dataWrapper = ( -
{data}
+
{data}
); return dataWrapper; diff --git a/frontend/src/components/CacheView.jsx b/frontend/src/components/CacheView.jsx index e9670baf..4da13127 100644 --- a/frontend/src/components/CacheView.jsx +++ b/frontend/src/components/CacheView.jsx @@ -35,6 +35,7 @@ import { InputLabel, Pagination, PaginationItem, + Avatar, } from "@mui/material"; import { @@ -134,7 +135,7 @@ const CacheView = memo((props) => { // Direct category migration from ../components/Files.jsx const [selectAllChecked, setSelectAllChecked] = React.useState(false) const [renderTextBox, setRenderTextBox] = React.useState(false); - const [datastoreCategories, setDatastoreCategories] = React.useState(["default"]); + const [datastoreCategories, setDatastoreCategories] = React.useState(["default", "protected"]); const [selectedCategory, setSelectedCategory] = React.useState("default"); const [selectedFileId, setSelectedFileId] = React.useState(""); const [updateToThisCategory, setUpdateToThisCategory] = useState("") @@ -322,7 +323,6 @@ const CacheView = memo((props) => { } var url = `${globalUrl}/api/v1/orgs/${orgId}/list_cache` - if (category !== undefined && category !== null && category !== "default" && category !== "") { url += "?category=" + category.replaceAll(" ", "_") } else { @@ -398,7 +398,7 @@ const CacheView = memo((props) => { } } - if ((category === undefined || category === "default" || category === "") && datastoreCategories.length === 1 && datastoreCategories[0] === "default") { + if ((category === undefined || category === "default" || category === "") && datastoreCategories.length === 2 && datastoreCategories[0] === "default") { var newcategories = ["default"] for (var key in responseJson.keys) { var foundcategory = responseJson.keys[key].category @@ -585,7 +585,7 @@ const CacheView = memo((props) => { }) .then((responseJson) => { setAddCache(responseJson); - toast("New key added Successfully!"); + toast.success("New key added!"); listOrgCache(orgId, selectedCategory, 0, pageSize, page); setModalOpen(false); }) @@ -699,6 +699,7 @@ const CacheView = memo((props) => {
+ { sortable: true, }, { - width: 600, + width: 540, field: 'value', filterable: true, headerName: 'Value', renderCell: (props) => { const data = props.row + + if (data?.category?.toLowerCase() === "protected") { + return ( + + *************** + + ) + } + const validate = validateJson(data.value) return ( @@ -1485,8 +1499,8 @@ const CacheView = memo((props) => { backgroundColor: theme.palette.platformColor, border: theme.palette.defaultBorder, padding: 5, - minWidth: 600, - maxHeight: 600, + minWidth: 500, + maxHeight: 500, overflowY: "auto", }} collapsed={true} @@ -1507,6 +1521,74 @@ const CacheView = memo((props) => { ) } }, + { + field: 'category', + headerName: 'Category', + description: 'Category for this key.', + width: 75, + filterable: false, + sortable: true, + renderCell: (props) => { + // Return avatar with hover for the category + const data = props.row + const clickCategory = (e) => { + setCategoryConfig(undefined) + setCategoryAutomations(defaultAutomation) + + if (selectAllChecked || selectedFiles.length > 0) { + setUpdateToThisCategory(data.category) + return + } + + setSelectedCategory(data.category) + if (data.category === "all" || data.category === "default") { + listOrgCache(orgId, "", 0, pageSize, page) + } else { + listOrgCache(orgId, data.category, 0, pageSize, page) + } + + // Add it to the url as a query + if (window.location.search.includes("category=")) { + const newurl = window.location.href.replace(/category=[^&]+/, `category=${data.category}`) + window.history.pushState({ path: newurl }, "", newurl) + } else { + window.history.pushState({ path: window.location.href }, "", `${window.location.href}&category=${data.category}`) + } + } + + + const iconDetails = GetIconInfo({ + "app_name": data.category, + "name": data.category, + }) + + const avatarLetter = (data.category === "" || data.category === "default" ? " " : data.category.charAt(0).toUpperCase())[0] + return ( + + { + clickCategory(e) + }} + style={{ + color: "white", + backgroundColor: iconDetails?.iconBackgroundColor || theme.palette.primary.secondary, + marginLeft: 15, + height: 30, + width: 30, + cursor: data.category !== "" && data.category !== "default" ? "pointer" : "default", + }} + variant="rounded" + > + {iconDetails?.originalIcon ? + iconDetails?.originalIcon + : + avatarLetter + } + + + ) + } + }, { field: 'actions', headerName: 'Actions', @@ -1520,7 +1602,7 @@ const CacheView = memo((props) => { return ( - {data?.workflow_id === "" || data?.workflow_id === null || data?.workflow_id === undefined ? + {data?.workflow_id === "" || data?.workflow_id === null || data?.workflow_id === undefined || data?.workflow_id?.length !== 36 ? { { title={"Go to workflow"} style={{}} aria-label={"Download"} + placement="left" > { > { > Learn more + + {selectedCategory === "protected" ? +
+ Protected keys are encrypted, only available to admins, and will be masked when used in workflows. This is a basic protection, and is NOT bulletproof. +
+ : null} +
@@ -2244,7 +2334,7 @@ const CacheView = memo((props) => { setSelectedRows(newSelection); }} keepNonExistentRowsSelected={false} - getRowId={(row) => row.key} + getRowId={(row) => `${row?.key}_${row?.category}`} autoHeight={true} sx={{ diff --git a/frontend/src/components/CollectIngestModal.jsx b/frontend/src/components/CollectIngestModal.jsx index 499b2fc3..d00faffd 100644 --- a/frontend/src/components/CollectIngestModal.jsx +++ b/frontend/src/components/CollectIngestModal.jsx @@ -23,8 +23,12 @@ import { import { Rocket as RocketIcon, FilterAlt as FilterAltIcon, + Add as AddIcon, } from '@mui/icons-material'; +import algoliasearch from 'algoliasearch/lite'; +const searchClient = algoliasearch("JNSS5CFDZZ", "c8f882473ff42d41158430be09ec2b4e") + const CollectIngestModal = (props) => { const { globalUrl, open, setOpen, workflows, getWorkflows, apps, } = props; @@ -86,11 +90,13 @@ const CollectIngestModal = (props) => { } const IngestItem = (props) => { - const { type, appCategory, index } = props + const { type, appCategory, index, webhook } = props const [hovering, setHovering] = useState(false); const [selectedApps, setSelectedApps] = useState([]); - //const [isFinished, setIsFinished] = useState(false); + + const [showAppsearch, setShowAppsearch] = useState(false); + const [algoliaOptions, setAlgoliaOptions] = useState([]); const appname = type const ingestedAmount = 20 @@ -167,7 +173,7 @@ const CollectIngestModal = (props) => { // { borderRadius: theme.palette.borderRadius, border: hovering ? `2px solid ${theme.palette.primary.main}` : foundMatchingWorkflow !== null ? `2px solid ${theme.palette.success.main}` : `2px solid ${theme.palette.secondary.main}`, textAlign: "center", - marginBottom: 5, + marginBottom: 10, overflow: "hidden", }} @@ -189,86 +195,116 @@ const CollectIngestModal = (props) => { }} onMouseLeave={() => setHovering(false)} > -
- {iconDetails?.originalIcon && ( - iconDetails?.originalIcon - )} +
- +
- {appname} - -
+
+ {selectedApps.map((app, index) => { + // Show image of each one + return ( +
+ + + +
+ ) + })} -
- {matchingapps.length > 0 ? - + { + setShowAppsearch(!showAppsearch) + }} + > + + + +
- value={selectedApps} - onChange={(event, value) => { - setSelectedApps(value) - }} + {showAppsearch ? + { - const parsedname = option.name.replaceAll("_", " ") + value={selectedApps} + onChange={(event, value) => { + setSelectedApps(value) + }} - return ( -
- {option.name} - - {parsedname} - -
- ) - }} - renderInput={(params) => { - return ( - - ) - }} - /> - : null} + getOptionLabel={(option) => { + const parsedname = option.name.replaceAll("_", " ") + return ( +
+ {option.name} + + {parsedname} + +
+ ) + }} + renderInput={(params) => { + return ( + + ) + }} + /> + : + } +
- startIngestion(appname, newapps, appCategory, index) - //} - }}> - {foundMatchingWorkflow !== null ? - "Re-Create Ingestion" - : - "Start Ingestion" - } - +
+ {iconDetails?.originalIcon && ( + iconDetails?.originalIcon + )} + + + + {appname} + +
{foundMatchingWorkflow !== null ? @@ -319,7 +355,7 @@ const CollectIngestModal = (props) => { sx: { borderRadius: theme?.palette?.DialogStyle?.borderRadius, border: theme?.palette?.DialogStyle?.border, - minWidth: 500, + minWidth: 850, minHeight: 700, fontFamily: theme?.typography?.fontFamily, backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, @@ -350,13 +386,14 @@ const CollectIngestModal = (props) => { - + + + - diff --git a/frontend/src/components/EditWorkflow.jsx b/frontend/src/components/EditWorkflow.jsx index 00b9608d..d227a2bd 100644 --- a/frontend/src/components/EditWorkflow.jsx +++ b/frontend/src/components/EditWorkflow.jsx @@ -3,6 +3,7 @@ import { getTheme } from '../theme.jsx'; import { isMobile } from "react-device-detect" import { MuiChipsInput } from "mui-chips-input"; import { toast } from "react-toastify" +import ReactGA from 'react-ga4'; import UsecaseSearch from "../components/UsecaseSearch.jsx" import WorkflowGrid from "../components/WorkflowGrid.jsx" import dayjs from 'dayjs'; @@ -63,7 +64,10 @@ import { Add as AddIcon, Remove as RemoveIcon, EditNote as EditNoteIcon, - AutoAwesome as AutoAwesomeIcon + AutoAwesome as AutoAwesomeIcon, + CloudUpload as CloudUploadIcon, + CheckCircle as CheckCircleIcon, + Close as CloseIcon } from "@mui/icons-material"; const EditWorkflow = (props) => { @@ -94,6 +98,11 @@ const EditWorkflow = (props) => { const [selectedCleanupActions, setSelectedCleanupActions] = React.useState(workflow?.form_control?.cleanup_actions !== undefined && workflow?.form_control?.cleanup_actions !== null ? JSON.parse(JSON.stringify(workflow?.form_control?.cleanup_actions)) : []) const [formWidth, setFormWidth] = React.useState(boxWidth === undefined || boxWidth === null ? 500 : boxWidth) + + // Flowchart upload states + const [uploadedImage, setUploadedImage] = React.useState(null) + const [imageBase64, setImageBase64] = React.useState("") + const [imageUploading, setImageUploading] = React.useState(false) const classes = useStyles(); @@ -103,6 +112,59 @@ const EditWorkflow = (props) => { } }, [formWidth]) + // Handle file upload and base64 conversion + const handleImageUpload = (file) => { + const allowedTypes = ['image/png', 'image/jpeg', 'image/jpg'] + if (!allowedTypes.includes(file.type)) { + toast.error("Please upload a PNG, JPG, or JPEG image") + return + } + + const maxSize = 5 * 1024 * 1024 // 5MB in bytes + if (file.size > maxSize) { + toast.error("Image must be less than 5MB") + return + } + + setImageUploading(true) + + const reader = new FileReader() + reader.onload = (e) => { + const base64 = e.target.result + setImageBase64(base64) + setUploadedImage({ + name: file.name, + size: file.size, + type: file.type + }) + setImageUploading(false) + + // Disable "Create from scratch" when image is uploaded + if (newWorkflow) { + setWorkflowAsCode(false) + } + } + reader.onerror = () => { + toast.error("Failed to read image file") + setImageUploading(false) + } + reader.readAsDataURL(file) + } + + const removeUploadedImage = () => { + setUploadedImage(null) + setImageBase64("") + setImageUploading(false) + } + + const formatFileSize = (bytes) => { + if (bytes === 0) return '0 Bytes' + const k = 1024 + const sizes = ['Bytes', 'KB', 'MB', 'GB'] + const i = Math.floor(Math.log(bytes) / Math.log(k)) + return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + ' ' + sizes[i] + } + if (scrollTo !== undefined && scrollTo !== null && scrollTo.length > 0 && scrollDone === false) { setTimeout(() => { const foundScroll = document.getElementById(scrollTo) @@ -308,7 +370,7 @@ const EditWorkflow = (props) => { variant="contained" style={{}} id="save_workflow_button" - disabled={name.length === 0 || submitLoading === true || aiGenerateLoading === true} + disabled={name.length === 0 || submitLoading === true || aiGenerateLoading === true || uploadedImage !== null} onClick={() => { setSubmitLoading(true) @@ -395,24 +457,46 @@ const EditWorkflow = (props) => { - Generate using the name, description, usecases and tags provided. Required: Name + Description + Generate using the name, description, usecases and tags provided. Required: Name + (Description OR Flowchart Image) } >
+ {/* Flowchart Upload Section - Only for new workflows */} + {newWorkflow === true ? ( +
+ {!uploadedImage ? ( +
{ + const input = document.createElement('input') + input.type = 'file' + input.accept = 'image/png,image/jpeg,image/jpg' + input.onchange = (e) => { + if (e.target.files.length > 0) { + handleImageUpload(e.target.files[0]) + } + } + input.click() + }} + > + {imageUploading ? ( +
+ + + Processing image... + +
+ ) : ( +
+ + + Generate Workflow from Flowchart + + + Click to upload your flowchart - AI will convert it to a workflow + + + PNG, JPG, JPEG • Max 5MB + +
+ )} +
+ ) : ( +
+
+ +
+ + {uploadedImage.name} + + + {formatFileSize(uploadedImage.size)} + +
+
+ + + +
+ )} +
+ ) : null} + {showMoreClicked === true ?
{ - Beta Feature: When a workflow run is done, the data from the selected actions will be removed by replacing it with a default value. This is useful for cleaning up sensitive data, or data that is no longer needed. This is done after a workflow run is finished or aborted, and is not reversible. Data will remain in the workflow run result (last node value) even if the action result itself is cleaned up. + When a workflow run is done, the data from the selected actions will be removed by replacing it with a default value. This is useful for cleaning up sensitive data, or data that is no longer needed. This is done after a workflow run is finished or aborted, and is not reversible. Data will remain in the workflow run result (last node value) even if the action result itself is cleaned up. diff --git a/frontend/src/components/LicencePopup.jsx b/frontend/src/components/LicencePopup.jsx index 0267267d..70e4cc6e 100644 --- a/frontend/src/components/LicencePopup.jsx +++ b/frontend/src/components/LicencePopup.jsx @@ -418,6 +418,10 @@ const LicencePopup = (props) => { showSupport = true } + if (userdata?.app_execution_limit >= 300000) { + top_text = "Enterprise Plan" + } + if (subscription.name.includes("Open Source")) { top_text = "Open Source" showSupport = true @@ -761,7 +765,7 @@ const LicencePopup = (props) => { { isCloud ? userdata?.app_execution_limit && userdata?.app_execution_limit !== 10000 ? - "You have already subscribed to the Scale plan, which includes " + (userdata?.app_execution_limit/1000) + "K app runs/month. You can increase the limit by upgrading current plan. Contact support@shuffler.io for more information." : + `You have already subscribed to the ${top_text}, which includes ${userdata?.app_execution_limit/1000}K app runs/month. You can increase the limit by upgrading current plan. Contact support@shuffler.io for more information.` : `You are using free Starter plan with max ${userdata?.app_execution_limit === 10000 ? "10,000" : "2,000"} runs per month. Upgrade to increase this limit.` : diff --git a/frontend/src/components/Navbar.jsx b/frontend/src/components/Navbar.jsx index 41b80efe..63efe7a9 100644 --- a/frontend/src/components/Navbar.jsx +++ b/frontend/src/components/Navbar.jsx @@ -87,15 +87,15 @@ const menuData = { ], Services: [ { - title: "Professional Services", + title: "Proof of Concept", description: - "Professional Services help you solve problems at your convenience.", + "Register for POC to test the best of Shuffle for free for 30 days.", icon: "/images/ProfessionalServices.svg", - path: "/professional-services", + path: "/poc", gaData: { category: "navbar", action: "services_click", - label: "professional_services_click" + label: "proof_of_concept_click" } }, { @@ -103,7 +103,7 @@ const menuData = { description: "Support to help you build automations with confidence.", icon: "/images/Support.svg", - path: "/contact?category=support", + path: "/support", gaData: { category: "navbar", action: "services_click", @@ -1486,7 +1486,7 @@ const Navbar = (props) => { } }} > - Become a partner + Become a Partner @@ -1661,7 +1661,7 @@ const Navbar = (props) => { const topbarHeight = showTopbar ? 40 : 0 const topbar = !isCloud || !showTopbar ? null : - curpath === "/" || curpath.includes("/docs") || curpath === "/pricing" || curpath === "/contact" || curpath === "/search" || curpath === "/usecases" || curpath === "/training" || curpath === "/professional-services" ? + curpath === "/" || curpath.includes("/docs") || curpath === "/pricing" || curpath === "/contact" || curpath === "/search" || curpath === "/usecases" || curpath === "/training" || curpath === "/poc" || curpath === "/professional-services" ? {/* uncommit this to show topbar for release */} {/*
diff --git a/frontend/src/components/OrgHeaderexpandedNew.jsx b/frontend/src/components/OrgHeaderexpandedNew.jsx index 18d65423..d0cd4952 100644 --- a/frontend/src/components/OrgHeaderexpandedNew.jsx +++ b/frontend/src/components/OrgHeaderexpandedNew.jsx @@ -419,7 +419,7 @@ const OrgHeaderexpandedNew = (props) => {
- Name + Name {
{userdata?.support ? (
-
Status
+
Status