From 9be293ac2a50f548b2f80f4f485723594f4874c8 Mon Sep 17 00:00:00 2001 From: Frikky Date: Wed, 11 Feb 2026 14:27:20 +0100 Subject: [PATCH 01/61] New dockerbuild --- .github/workflows/dockerbuild.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dockerbuild.yaml b/.github/workflows/dockerbuild.yaml index 9f948ad4..d5fb8d78 100644 --- a/.github/workflows/dockerbuild.yaml +++ b/.github/workflows/dockerbuild.yaml @@ -20,19 +20,19 @@ jobs: include: - app: frontend path: frontend - version: 2.1.3 + version: 2.2.0 experimental: true - app: backend path: backend - version: 2.1.3 + version: 2.2.0 experimental: true - app: orborus path: functions/onprem/orborus - version: 2.1.3 + version: 2.2.0 experimental: true - app: worker path: functions/onprem/worker - version: 2.1.3 + version: 2.2.0 experimental: true steps: - name: Checkout From 6bf70ca89d1b93268d6ddf6f4e2ff38b8ccba7ef Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Thu, 12 Feb 2026 21:40:15 +0530 Subject: [PATCH 02/61] shuffle-shared bump --- backend/go-app/go.mod | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index a3aceeb0..5e9068e0 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -26,7 +26,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.87 + github.com/shuffle/shuffle-shared v0.9.93 github.com/shuffle/singul v0.0.26 golang.org/x/crypto v0.45.0 google.golang.org/api v0.236.0 @@ -65,7 +65,6 @@ require ( github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect - github.com/coreos/go-oidc/v3 v3.17.0 // indirect github.com/cyphar/filepath-securejoin v0.4.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.6.0 // indirect From bd8eb55e9fdc75989a5a00812d7844f628b36940 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Mon, 23 Feb 2026 23:25:17 +0530 Subject: [PATCH 03/61] worker: reduce subflow polling load and app retry pressure --- functions/onprem/worker/worker.go | 135 +++++++++++++++++++++++++++--- 1 file changed, 123 insertions(+), 12 deletions(-) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 83e49d85..370ce2bf 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -19,6 +19,7 @@ import ( "os" "strconv" "strings" + "sync" "time" "github.com/docker/docker/api/types" @@ -110,6 +111,8 @@ var finishedExecutions []string var imagesDistributed []string var imagedownloadTimeout = time.Second * 300 +var subflowPollBackoff sync.Map + var window = shuffle.NewTimeWindow(10 * time.Second) // Images to be autodeployed in the latest version of Shuffle. @@ -275,8 +278,6 @@ func setWorkflowExecution(ctx context.Context, workflowExecution shuffle.Workflo break } - // Sleep for 1 second - time.Sleep(1 * time.Second) } } else { log.Printf("[DEBUG][%s] No need to poll for results. Not polling", workflowExecution.ExecutionId) @@ -2176,6 +2177,58 @@ func handleSubflowPoller(ctx context.Context, workflowExecution shuffle.Workflow data = fmt.Sprintf(`{"execution_id": "%s", "authorization": "%s"}`, workflowExecution.ExecutionId, workflowExecution.Authorization) } + key := fmt.Sprintf("%s:%s", workflowExecution.ExecutionId, subflowId) + cacheKey := fmt.Sprintf("workflowexecution_%s", workflowExecution.ExecutionId) + usedCache := false + if cacheData, err := shuffle.GetCache(ctx, cacheKey); err == nil { + cachedBytes, ok := cacheData.([]uint8) + if ok { + cacheWorkflow := shuffle.WorkflowExecution{} + if jsonErr := json.Unmarshal([]byte(cachedBytes), &cacheWorkflow); jsonErr == nil { + workflowExecution = cacheWorkflow + usedCache = true + log.Printf("[DEBUG][%s] Using cached workflow execution for subflow poll", workflowExecution.ExecutionId) + + if workflowExecution.Status == "FINISHED" || workflowExecution.Status == "SUCCESS" { + log.Printf("[INFO][%s] Workflow execution is finished (cache). Exiting worker.", workflowExecution.ExecutionId) + log.Printf("[DEBUG] Shutting down (20)") + resetSubflowPollDelay(key) + if isKubernetes == "true" { + clientset, _, err := shuffle.GetKubernetesClient() + if err != nil { + log.Println("[ERROR] Error getting kubernetes client (2):", err) + os.Exit(1) + } + + cleanupKubernetesExecution(clientset, workflowExecution, kubernetesNamespace) + } else { + shutdown(workflowExecution, "", "", true) + } + } + + for _, result := range workflowExecution.Results { + if result.Action.ID != subflowId { + continue + } + + if result.Status == "SUCCESS" || result.Status == "FINISHED" || result.Status == "FAILURE" || result.Status == "ABORTED" { + resetSubflowPollDelay(key) + setWorkflowExecution(ctx, workflowExecution, false) + return nil + } + } + } + } + } + + if usedCache { + delay := nextSubflowPollDelay(key) + attempt := getSubflowPollAttempt(key) + log.Printf("[DEBUG][%s] Subflow poll backoff attempt %d for %s (cache hit), sleeping %s", workflowExecution.ExecutionId, attempt, subflowId, delay) + time.Sleep(delay) + return errors.New("Subflow status not found yet (cache)") + } + req, err := http.NewRequest( "POST", streamResultUrl, @@ -2186,7 +2239,7 @@ func handleSubflowPoller(ctx context.Context, workflowExecution shuffle.Workflow newresp, err := client.Do(req) if err != nil { log.Printf("[ERROR] Failed making request (1): %s", err) - time.Sleep(time.Duration(sleepTime) * time.Second) + time.Sleep(nextSubflowPollDelay(key)) return err } @@ -2194,7 +2247,7 @@ func handleSubflowPoller(ctx context.Context, workflowExecution shuffle.Workflow body, err := ioutil.ReadAll(newresp.Body) if err != nil { log.Printf("[ERROR] Failed reading body (1): %s", err) - time.Sleep(time.Duration(sleepTime) * time.Second) + time.Sleep(nextSubflowPollDelay(key)) return err } @@ -2206,20 +2259,21 @@ func handleSubflowPoller(ctx context.Context, workflowExecution shuffle.Workflow shutdown(workflowExecution, "", "", true) } - time.Sleep(time.Duration(sleepTime) * time.Second) + time.Sleep(nextSubflowPollDelay(key)) return errors.New(fmt.Sprintf("Bad statuscode: %d", newresp.StatusCode)) } err = json.Unmarshal(body, &workflowExecution) if err != nil { log.Printf("[ERROR] Failed workflowExecution unmarshal: %s", err) - time.Sleep(time.Duration(sleepTime) * time.Second) + time.Sleep(nextSubflowPollDelay(key)) return err } if workflowExecution.Status == "FINISHED" || workflowExecution.Status == "SUCCESS" { log.Printf("[INFO][%s] Workflow execution is finished. Exiting worker.", workflowExecution.ExecutionId) log.Printf("[DEBUG] Shutting down (20)") + resetSubflowPollDelay(key) if isKubernetes == "true" { // log.Printf("workflow execution: %#v", workflowExecution) clientset, _, err := shuffle.GetKubernetesClient() @@ -2235,11 +2289,14 @@ func handleSubflowPoller(ctx context.Context, workflowExecution shuffle.Workflow } hasUserinput := false + foundSubflow := false for _, result := range workflowExecution.Results { if result.Action.ID != subflowId { continue } + foundSubflow = true + if result.Action.AppName == "User Input" { hasUserinput = true } @@ -2247,21 +2304,66 @@ func handleSubflowPoller(ctx context.Context, workflowExecution shuffle.Workflow log.Printf("[DEBUG][%s] Found subflow to handle: %s (%s)", workflowExecution.ExecutionId, result.Action.AppName, result.Status) if result.Status == "SUCCESS" || result.Status == "FINISHED" || result.Status == "FAILURE" || result.Status == "ABORTED" { // Check for results - + resetSubflowPollDelay(key) setWorkflowExecution(ctx, workflowExecution, false) return nil } } - if workflowExecution.Status == "WAITING" && workflowExecution.ExecutionSource != "default" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "swarm" { - log.Printf("[INFO][%s] Workflow execution is waiting. Exiting worker, as backend will restart it.", workflowExecution.ExecutionId) + if workflowExecution.Status == "WAITING" && !foundSubflow && workflowExecution.ExecutionSource != "default" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "swarm" { + log.Printf("[INFO][%s] Workflow execution is waiting without subflow progress. Exiting worker, as backend will restart it.", workflowExecution.ExecutionId) + resetSubflowPollDelay(key) shutdown(workflowExecution, "", "", true) + return nil } log.Printf("[INFO][%s] (2) Status: %s, Results: %d, actions: %d. Userinput: %#v", workflowExecution.ExecutionId, workflowExecution.Status, len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)+extra, hasUserinput) + delay := nextSubflowPollDelay(key) + attempt := getSubflowPollAttempt(key) + log.Printf("[DEBUG][%s] Subflow poll backoff attempt %d for %s, sleeping %s", workflowExecution.ExecutionId, attempt, subflowId, delay) + time.Sleep(delay) return errors.New("Subflow status not found yet") } +func nextSubflowPollDelay(key string) time.Duration { + baseDelay := 250 * time.Millisecond + maxDelay := 5 * time.Second + + current := 0 + if raw, ok := subflowPollBackoff.Load(key); ok { + if value, ok := raw.(int); ok { + current = value + } + } + + if current < 8 { + current += 1 + } + + subflowPollBackoff.Store(key, current) + + delay := baseDelay * time.Duration(1< maxDelay { + delay = maxDelay + } + + return delay +} + +func resetSubflowPollDelay(key string) { + subflowPollBackoff.Delete(key) +} + +func getSubflowPollAttempt(key string) int { + if raw, ok := subflowPollBackoff.Load(key); ok { + if value, ok := raw.(int); ok { + return value + } + } + + return 0 +} + func handleDefaultExecutionWrapper(ctx context.Context, workflowExecution shuffle.WorkflowExecution, streamResultUrl string, extra int) error { if extra == -1 { extra = 0 @@ -3479,6 +3581,8 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl fmt.Sprintf("SHUFFLE_APP_EXPOSED_PORT=%d", deployport), fmt.Sprintf("SHUFFLE_SWARM_CONFIG=%s", os.Getenv("SHUFFLE_SWARM_CONFIG")), fmt.Sprintf("SHUFFLE_LOGS_DISABLED=%s", logsDisabled), + fmt.Sprintf("CALLBACK_URL=%s", baseUrl), + fmt.Sprintf("BASE_URL=%s", appCallbackUrl), }, Hosts: []string{ containerName, @@ -3736,9 +3840,10 @@ func findAppInfo(image, name string, redeploy bool) (int, error) { // Remove the service and redeploy it. // There are cases where the service doesn't update properly // Check when the last update happened. If it was within the last few minutes, skip - if int(time.Since(service.UpdatedAt).Seconds()) > 60 { + updateAgeSeconds := int(time.Since(service.UpdatedAt).Seconds()) + if updateAgeSeconds > 60 { - log.Printf("[INFO] Attempting redeploy of app %s with image %s since it is more than 10 minutes since last attempt with failure.", name, image) + log.Printf("[INFO] Attempting redeploy of app %s with image %s since last update was %d seconds ago.", name, image, updateAgeSeconds) err = dockercli.ServiceRemove( context.Background(), @@ -3767,7 +3872,7 @@ func findAppInfo(image, name string, redeploy bool) (int, error) { } } } else { - //log.Printf("[INFO] NOT redeploying service %s since it was updated less than 3 minutes ago.", name) + log.Printf("[INFO] Skipping redeploy of app %s since last update was %d seconds ago.", name, updateAgeSeconds) } } @@ -4095,6 +4200,12 @@ func sendAppRequest(ctx context.Context, incomingUrl, appName string, port int, log.Printf("[DEBUG] Setting client timeout to %d seconds for app request", timeoutInt) client.Timeout = time.Duration(timeoutInt) * time.Second } + } else if len(os.Getenv("SHUFFLE_APP_SDK_TIMEOUT")) > 0 { + timeoutInt, err := strconv.Atoi(os.Getenv("SHUFFLE_APP_SDK_TIMEOUT")) + if err == nil && timeoutInt > 0 { + log.Printf("[DEBUG] Using SHUFFLE_APP_SDK_TIMEOUT=%d seconds for app request", timeoutInt) + client.Timeout = time.Duration(timeoutInt) * time.Second + } } // Content type required From 434e51240504d7dc936d899184309be36819f96c Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Tue, 24 Feb 2026 20:03:50 +0530 Subject: [PATCH 04/61] worker: stabilize app redeploys and callbacks --- functions/onprem/worker/worker.go | 42 ++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 370ce2bf..af2ca8cd 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -1316,6 +1316,27 @@ func removeIndex(s []string, i int) []string { return s[:len(s)-1] } +func countHealthyServiceTasks(dockercli *dockerclient.Client, serviceID string) (int, error) { + ctx := context.Background() + filters := filters.NewArgs() + filters.Add("service", serviceID) + filters.Add("desired-state", "running") + + tasks, err := dockercli.TaskList(ctx, types.TaskListOptions{Filters: filters}) + if err != nil { + return 0, err + } + + count := 0 + for _, task := range tasks { + if task.Status.State == swarm.TaskStateRunning { + count += 1 + } + } + + return count, nil +} + func getWorkerURLs() ([]string, error) { workerUrls := []string{} @@ -3836,12 +3857,22 @@ func findAppInfo(image, name string, redeploy bool) (int, error) { } if redeploy { + healthyTasks, taskErr := countHealthyServiceTasks(dockercli, service.ID) + if taskErr != nil { + log.Printf("[WARNING] Failed checking tasks for service %s: %s", name, taskErr) + } + + if healthyTasks > 0 { + log.Printf("[INFO] Skipping redeploy of app %s since %d task(s) are running", name, healthyTasks) + break + } + log.Printf("[INFO] Found to redeploy! Service: %s with image %s on port %d", name, image, exposedPort) // Remove the service and redeploy it. // There are cases where the service doesn't update properly // Check when the last update happened. If it was within the last few minutes, skip updateAgeSeconds := int(time.Since(service.UpdatedAt).Seconds()) - if updateAgeSeconds > 60 { + if updateAgeSeconds > 300 { log.Printf("[INFO] Attempting redeploy of app %s with image %s since last update was %d seconds ago.", name, image, updateAgeSeconds) @@ -4115,9 +4146,12 @@ func sendAppRequest(ctx context.Context, incomingUrl, appName string, port int, parsedRequest.Url = parsedRequest.BaseUrl parsedRequest.BaseUrl = tmp - // Run with proper hostname, but set to shuffle-worker to avoid specific host target. - // This means running with VIP instead. - if len(hostname) > 0 { + callbackUrl := os.Getenv("SHUFFLE_WORKER_SERVER_URL") + if len(callbackUrl) > 0 { + parsedRequest.BaseUrl = callbackUrl + } else if len(hostname) > 0 { + // Run with proper hostname, but set to shuffle-worker to avoid specific host target. + // This means running with VIP instead. parsedRequest.BaseUrl = fmt.Sprintf("http://%s:%d", hostname, baseport) //parsedRequest.BaseUrl = fmt.Sprintf("http://shuffle-workers:%d", baseport) //log.Printf("[DEBUG][%s] Changing hostname to local hostname in Docker network for WORKER URL: %s", workflowExecution.ExecutionId, parsedRequest.BaseUrl) From b9032efc2efbb1eb2c60374bcacdf73a834f39d2 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Tue, 24 Feb 2026 23:04:02 +0530 Subject: [PATCH 05/61] shuffle-shared bump --- backend/go-app/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 5e9068e0..66610b9d 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -26,7 +26,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.93 + github.com/shuffle/shuffle-shared v0.9.98 github.com/shuffle/singul v0.0.26 golang.org/x/crypto v0.45.0 google.golang.org/api v0.236.0 From c1fdda6fe100e79fcc06fa402998b90067a05a4f Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Tue, 24 Feb 2026 23:08:04 +0530 Subject: [PATCH 06/61] fix SSO arguments --- backend/go-app/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 540d79db..98d42dd0 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -1263,7 +1263,7 @@ func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { // Should run calculations if len(org.SSOConfig.OpenIdAuthorization) > 0 { - baseSSOUrl, err = shuffle.GetOpenIdUrl(request, *org, user, "") + baseSSOUrl = shuffle.GetOpenIdUrl(request, *org) if err != nil { log.Printf("[ERROR] Failed getting OpenID URL for org %s: %s", org.Name, err) } From aa533cd162941a0727c46a99edf52dcbda755016 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Tue, 24 Feb 2026 23:40:01 +0530 Subject: [PATCH 07/61] shuffle-shared bump --- functions/onprem/worker/go.mod | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 0b9c303c..48194082 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -11,7 +11,7 @@ require ( github.com/docker/docker v28.3.3+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.87 + github.com/shuffle/shuffle-shared v0.9.98 github.com/shuffle/singul v0.0.26 k8s.io/api v0.34.2 k8s.io/apimachinery v0.34.2 @@ -49,7 +49,6 @@ require ( github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect - github.com/coreos/go-oidc/v3 v3.17.0 // indirect github.com/cyphar/filepath-securejoin v0.4.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.6.0 // indirect From 4df45844a82f031a65d52ed718b0fa9a883be141 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Thu, 26 Feb 2026 11:22:12 +0530 Subject: [PATCH 08/61] shuffle-shared bump --- backend/go-app/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 66610b9d..4bbbbdf4 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -26,7 +26,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.98 + github.com/shuffle/shuffle-shared v0.9.99 github.com/shuffle/singul v0.0.26 golang.org/x/crypto v0.45.0 google.golang.org/api v0.236.0 From e38a54303d5fc639100aaf83f6016529d94fc42d Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Tue, 3 Mar 2026 14:29:56 +0530 Subject: [PATCH 09/61] shuffle-shared bump --- backend/go-app/go.mod | 2 +- functions/onprem/orborus/go.mod | 3 +-- functions/onprem/worker/go.mod | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 4bbbbdf4..bcc96f11 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -26,7 +26,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.99 + github.com/shuffle/shuffle-shared v1.0.0 github.com/shuffle/singul v0.0.26 golang.org/x/crypto v0.45.0 google.golang.org/api v0.236.0 diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 7fa5bdc4..f4a37b8b 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -10,7 +10,7 @@ require ( github.com/docker/docker v28.3.3+incompatible github.com/docker/go-connections v0.5.0 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.87 + github.com/shuffle/shuffle-shared v1.0.0 k8s.io/api v0.34.2 k8s.io/apimachinery v0.34.2 ) @@ -48,7 +48,6 @@ require ( github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect - github.com/coreos/go-oidc/v3 v3.17.0 // indirect github.com/cyphar/filepath-securejoin v0.4.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.6.0 // indirect diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 48194082..7b8bfefa 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -11,7 +11,7 @@ require ( github.com/docker/docker v28.3.3+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.98 + github.com/shuffle/shuffle-shared v1.0.0 github.com/shuffle/singul v0.0.26 k8s.io/api v0.34.2 k8s.io/apimachinery v0.34.2 From 56dc036fb2125075e5fec3b026a1855847a71c4f Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Fri, 6 Mar 2026 21:19:00 +0530 Subject: [PATCH 10/61] updated shuffle-shared --- functions/onprem/worker/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 7b8bfefa..18446248 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -11,7 +11,7 @@ require ( github.com/docker/docker v28.3.3+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v1.0.0 + github.com/shuffle/shuffle-shared v1.0.3 github.com/shuffle/singul v0.0.26 k8s.io/api v0.34.2 k8s.io/apimachinery v0.34.2 From b1f1c071f8aea479cb31be0a5ea3f9f3a67bada6 Mon Sep 17 00:00:00 2001 From: Frikky Date: Tue, 10 Mar 2026 12:12:36 +0100 Subject: [PATCH 11/61] Re-added sync from openapi repos as well if available. Agnostic app download --- backend/go-app/go.sum | 6 ++---- backend/go-app/walkoff.go | 3 +++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 0e38239a..82e794b4 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -101,8 +101,6 @@ github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151X github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/coreos/go-oidc/v3 v3.17.0 h1:hWBGaQfbi0iVviX4ibC7bk8OKT5qNr4klBaCHVNvehc= -github.com/coreos/go-oidc/v3 v3.17.0/go.mod h1:wqPbKFrVnE90vty060SB40FCJ8fTHTxSwyXJqZH+sI8= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= @@ -350,8 +348,8 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shuffle/opensearch-go/v4 v4.0.0 h1:Mh85CD1MwOgXiFFYlzS1llnvdqL3CztRdR1ZT/SLIjU= github.com/shuffle/opensearch-go/v4 v4.0.0/go.mod h1:gVLZKQE5khQWMb68XBtgKrhu78oLGL2zHwAGnFMDwC0= -github.com/shuffle/shuffle-shared v0.9.87 h1:INA1cZ18MKcMs8kaVBJoQqrdNuqLHfge9elJ17hZfVc= -github.com/shuffle/shuffle-shared v0.9.87/go.mod h1:AkXajlWWB16WfWjCw9K7y38L8JKABJzVxcX6KI/J1H4= +github.com/shuffle/shuffle-shared v1.0.0 h1:hNNsv8uS/LxxT87NGpq/tQR1ERtgYb67SG7bmZyWeiI= +github.com/shuffle/shuffle-shared v1.0.0/go.mod h1:AkXajlWWB16WfWjCw9K7y38L8JKABJzVxcX6KI/J1H4= github.com/shuffle/singul v0.0.26 h1:P2uZ8YIYQUN4qNfQujoW3Lhod91X6TAfrRHGWiUPNI8= github.com/shuffle/singul v0.0.26/go.mod h1:S8GszXL+fT2mTnh7j57V0r/tY5Y/mSW8QowQEnADs3k= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 288fedc0..465e093c 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -2143,6 +2143,7 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } + func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error { ctx := context.Background() @@ -2211,6 +2212,7 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, } // 1. This parses OpenAPI v2 to v3 etc, for use. + // Always returns an ID as well parsedOpenApi, err := handleSwaggerValidation(readFile) if err != nil { log.Printf("[WARNING] Validation error for %s: %s", filename, err) @@ -3322,6 +3324,7 @@ func LoadSpecificApps(resp http.ResponseWriter, request *http.Request) { } IterateAppGithubFolders(ctx, fs, dir, "", "", tmpBody.ForceUpdate, false) + iterateOpenApiGithub(fs, dir, "", "") } else if strings.Contains(tmpBody.URL, "s3") { //https://docs.aws.amazon.com/sdk-for-go/api/service/s3/ From cd4786326402ed9f82fd412f3a239a37d4737f7e Mon Sep 17 00:00:00 2001 From: Frikky Date: Tue, 10 Mar 2026 12:53:37 +0100 Subject: [PATCH 12/61] Remapped tenzir to default to their main image --- functions/onprem/orborus/orborus.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 99639e72..6aadedfe 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -3183,7 +3183,8 @@ func deployTenzirNode() error { return nil } - imageName := "frikky/shuffle:tenzir" + //imageName := "frikky/shuffle:tenzir" + imageName := "tenzir/tenzir:main" if os.Getenv("TENZIR_IMAGE_NAME") != "" { imageName = os.Getenv("TENZIR_IMAGE_NAME") log.Printf("[INFO] Using custom Tenzir image name: %s", imageName) From 25920b36b6a182eb694c3bc9f38e0766229ca61b Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Wed, 11 Mar 2026 11:23:50 +0530 Subject: [PATCH 13/61] shuffle-shared bump --- backend/go-app/go.mod | 4 ++-- functions/onprem/orborus/go.mod | 2 +- functions/onprem/worker/go.mod | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index bcc96f11..180aa924 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -26,7 +26,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v1.0.0 + github.com/shuffle/shuffle-shared v1.0.6 github.com/shuffle/singul v0.0.26 golang.org/x/crypto v0.45.0 google.golang.org/api v0.236.0 @@ -75,7 +75,7 @@ require ( github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/frikky/schemaless v0.0.28 // indirect + github.com/frikky/schemaless v0.0.31 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-jose/go-jose/v4 v4.1.3 // indirect diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index f4a37b8b..b131c52d 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -10,7 +10,7 @@ require ( github.com/docker/docker v28.3.3+incompatible github.com/docker/go-connections v0.5.0 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v1.0.0 + github.com/shuffle/shuffle-shared v1.0.6 k8s.io/api v0.34.2 k8s.io/apimachinery v0.34.2 ) diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 18446248..b2151de9 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -11,7 +11,7 @@ require ( github.com/docker/docker v28.3.3+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v1.0.3 + github.com/shuffle/shuffle-shared v1.0.6 github.com/shuffle/singul v0.0.26 k8s.io/api v0.34.2 k8s.io/apimachinery v0.34.2 From c5015c799431f1a583124baebe102f6f9a0b6912 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Wed, 11 Mar 2026 13:34:00 +0530 Subject: [PATCH 14/61] fix: shuffle-shared bump issues code removal and stuff --- backend/go-app/go.mod | 3 ++- backend/go-app/main.go | 18 +++++++++--------- backend/go-app/walkoff.go | 16 ++++++++-------- functions/onprem/worker/go.mod | 5 +++-- 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 180aa924..688eac00 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -27,7 +27,7 @@ require ( github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 github.com/shuffle/shuffle-shared v1.0.6 - github.com/shuffle/singul v0.0.26 + github.com/shuffle/singul v0.0.28 golang.org/x/crypto v0.45.0 google.golang.org/api v0.236.0 google.golang.org/grpc v1.72.2 @@ -56,6 +56,7 @@ require ( github.com/ProtonMail/go-crypto v1.1.6 // indirect github.com/adrg/strutil v0.3.1 // indirect github.com/algolia/algoliasearch-client-go/v3 v3.31.4 // indirect + github.com/andybalholm/brotli v1.2.0 // indirect github.com/bitly/go-simplejson v0.5.1 // indirect github.com/bradfitz/gomemcache v0.0.0-20250403215159-8d39553ac7cf // indirect github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 // indirect diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 98d42dd0..48fe90e1 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5999,13 +5999,13 @@ func initHandlers() { r.HandleFunc("/api/v1/get_openapi/{key}", getOpenapi).Methods("GET", "OPTIONS") // Specific triggers - r.HandleFunc("/api/v1/workflows/{key}/outlook", shuffle.HandleCreateOutlookSub).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/workflows/{key}/outlook/{triggerId}", shuffle.HandleDeleteOutlookSub).Methods("DELETE", "OPTIONS") - r.HandleFunc("/api/v1/triggers/outlook/register", shuffle.HandleNewOutlookRegister).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/triggers/outlook/getFolders", shuffle.HandleGetOutlookFolders).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/triggers/outlook/{key}", shuffle.HandleGetSpecificTrigger).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/triggers/gmail/register", shuffle.HandleNewGmailRegister).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/triggers/gmail/getFolders", shuffle.HandleGetGmailFolders).Methods("GET", "OPTIONS") +// r.HandleFunc("/api/v1/workflows/{key}/outlook", shuffle.HandleCreateOutlookSub).Methods("POST", "OPTIONS") +// r.HandleFunc("/api/v1/workflows/{key}/outlook/{triggerId}", shuffle.HandleDeleteOutlookSub).Methods("DELETE", "OPTIONS") +// r.HandleFunc("/api/v1/triggers/outlook/register", shuffle.HandleNewOutlookRegister).Methods("GET", "OPTIONS") +// r.HandleFunc("/api/v1/triggers/outlook/getFolders", shuffle.HandleGetOutlookFolders).Methods("GET", "OPTIONS") +// r.HandleFunc("/api/v1/triggers/outlook/{key}", shuffle.HandleGetSpecificTrigger).Methods("GET", "OPTIONS") +// r.HandleFunc("/api/v1/triggers/gmail/register", shuffle.HandleNewGmailRegister).Methods("GET", "OPTIONS") +// r.HandleFunc("/api/v1/triggers/gmail/getFolders", shuffle.HandleGetGmailFolders).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/triggers/pipeline", shuffle.HandleNewPipelineRegister).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/triggers/github/register", shuffle.HandleNewGithubRegister).Methods("PUT", "OPTIONS") //r.HandleFunc("/api/v1/triggers/pipeline/save", shuffle.HandleSavePipelineInfo).Methods("PUT", "OPTIONS") @@ -6015,8 +6015,8 @@ func initHandlers() { //r.HandleFunc("/api/v1/triggers/gmail/routing", handleGmailRouting).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/triggers/gmail/{key}", shuffle.HandleGetSpecificTrigger).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/workflows/{key}/gmail", shuffle.HandleCreateGmailSub).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/workflows/{key}/gmail/{triggerId}", shuffle.HandleDeleteGmailSub).Methods("DELETE", "OPTIONS") +// r.HandleFunc("/api/v1/workflows/{key}/gmail", shuffle.HandleCreateGmailSub).Methods("POST", "OPTIONS") +// r.HandleFunc("/api/v1/workflows/{key}/gmail/{triggerId}", shuffle.HandleDeleteGmailSub).Methods("DELETE", "OPTIONS") //r.HandleFunc("/api/v1/triggers/gmail/{key}", handleGetSpecificGmailTrigger).Methods("GET", "OPTIONS") //r.HandleFunc("/api/v1/triggers/outlook/getFolders", shuffle.HandleGetOutlookFolders).Methods("GET", "OPTIONS") diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 465e093c..be8175ee 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -813,15 +813,15 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { // log.Printf("Failed to delete webhook: %s", err) //} } else if item.TriggerType == "EMAIL" { - err = shuffle.HandleOutlookSubRemoval(ctx, user, workflow.ID, item.ID) - if err != nil { - log.Printf("[DEBUG] Failed to delete OUTLOOK email sub (checking gmail after): %s", err) - } + // err = shuffle.HandleOutlookSubRemoval(ctx, user, workflow.ID, item.ID) + // if err != nil { + // log.Printf("[DEBUG] Failed to delete OUTLOOK email sub (checking gmail after): %s", err) + // } - err = shuffle.HandleGmailSubRemoval(ctx, user, workflow.ID, item.ID) - if err != nil { - log.Printf("Failed to delete gmail email sub: %s", err) - } + // err = shuffle.HandleGmailSubRemoval(ctx, user, workflow.ID, item.ID) + // if err != nil { + // log.Printf("Failed to delete gmail email sub: %s", err) + // } } } diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index b2151de9..ec3ce491 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 github.com/shuffle/shuffle-shared v1.0.6 - github.com/shuffle/singul v0.0.26 + github.com/shuffle/singul v0.0.28 k8s.io/api v0.34.2 k8s.io/apimachinery v0.34.2 k8s.io/client-go v0.34.2 @@ -42,6 +42,7 @@ require ( github.com/ProtonMail/go-crypto v1.1.6 // indirect github.com/adrg/strutil v0.3.1 // indirect github.com/algolia/algoliasearch-client-go/v3 v3.31.4 // indirect + github.com/andybalholm/brotli v1.2.0 // indirect github.com/bradfitz/gomemcache v0.0.0-20250403215159-8d39553ac7cf // indirect github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect @@ -60,7 +61,7 @@ require ( github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/frikky/kin-openapi v0.42.0 // indirect - github.com/frikky/schemaless v0.0.28 // indirect + github.com/frikky/schemaless v0.0.31 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect From 39a57672e459551d857f8b039df6487395ed5c3a Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Wed, 18 Mar 2026 17:55:19 +0530 Subject: [PATCH 15/61] shuffle-shared bump --- backend/go-app/go.mod | 4 ++-- functions/onprem/orborus/go.sum | 10 ++++------ functions/onprem/worker/go.mod | 4 ++-- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 688eac00..d44f674e 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -26,7 +26,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v1.0.6 + github.com/shuffle/shuffle-shared v1.1.4 github.com/shuffle/singul v0.0.28 golang.org/x/crypto v0.45.0 google.golang.org/api v0.236.0 @@ -76,7 +76,7 @@ require ( github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/frikky/schemaless v0.0.31 // indirect + github.com/frikky/schemaless v0.0.32 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-jose/go-jose/v4 v4.1.3 // indirect diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index 5f00a945..85701632 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -93,8 +93,6 @@ github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151X github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/coreos/go-oidc/v3 v3.17.0 h1:hWBGaQfbi0iVviX4ibC7bk8OKT5qNr4klBaCHVNvehc= -github.com/coreos/go-oidc/v3 v3.17.0/go.mod h1:wqPbKFrVnE90vty060SB40FCJ8fTHTxSwyXJqZH+sI8= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s= github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= @@ -130,8 +128,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0= github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4= -github.com/frikky/schemaless v0.0.28 h1:gdurMqBwtvY4Y/5pcxn8bdGCJn/eolKGz+c5DcidLkI= -github.com/frikky/schemaless v0.0.28/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= +github.com/frikky/schemaless v0.0.32 h1:tbLxdi3GIJZaQDbfiCGEOFDogeBYaHn+IF+Vp9Xzqv4= +github.com/frikky/schemaless v0.0.32/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -315,8 +313,8 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shuffle/opensearch-go/v4 v4.0.0 h1:Mh85CD1MwOgXiFFYlzS1llnvdqL3CztRdR1ZT/SLIjU= github.com/shuffle/opensearch-go/v4 v4.0.0/go.mod h1:gVLZKQE5khQWMb68XBtgKrhu78oLGL2zHwAGnFMDwC0= -github.com/shuffle/shuffle-shared v0.9.87 h1:INA1cZ18MKcMs8kaVBJoQqrdNuqLHfge9elJ17hZfVc= -github.com/shuffle/shuffle-shared v0.9.87/go.mod h1:AkXajlWWB16WfWjCw9K7y38L8JKABJzVxcX6KI/J1H4= +github.com/shuffle/shuffle-shared v1.1.4 h1:TfeP9yslJfajdS2Sk/I81xU+R6ZxkyC49h9hc7oPBv4= +github.com/shuffle/shuffle-shared v1.1.4/go.mod h1:ycTfvlAIXv78Vxu+fHtPPIEP8mK9Pwtfk/yw2uMiAB0= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index ec3ce491..98432766 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -11,7 +11,7 @@ require ( github.com/docker/docker v28.3.3+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v1.0.6 + github.com/shuffle/shuffle-shared v1.1.4 github.com/shuffle/singul v0.0.28 k8s.io/api v0.34.2 k8s.io/apimachinery v0.34.2 @@ -61,7 +61,7 @@ require ( github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/frikky/kin-openapi v0.42.0 // indirect - github.com/frikky/schemaless v0.0.31 // indirect + github.com/frikky/schemaless v0.0.32 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect From d36f2fcffdcc3c2db4f88d953e83f1f498ef4b49 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Wed, 18 Mar 2026 18:58:09 +0530 Subject: [PATCH 16/61] singul version bump --- backend/go-app/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index d44f674e..4b21cd7d 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -27,7 +27,7 @@ require ( github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 github.com/shuffle/shuffle-shared v1.1.4 - github.com/shuffle/singul v0.0.28 + github.com/shuffle/singul v0.0.29 golang.org/x/crypto v0.45.0 google.golang.org/api v0.236.0 google.golang.org/grpc v1.72.2 From b631abfb771a8aad3e8620a2679dd1a40cfa5195 Mon Sep 17 00:00:00 2001 From: Lalit Deore Date: Wed, 18 Mar 2026 19:09:41 +0530 Subject: [PATCH 17/61] add azure devops sync feature --- frontend/src/views/Workflows2.jsx | 208 +++++++++++++++++++++++++++++- 1 file changed, 204 insertions(+), 4 deletions(-) diff --git a/frontend/src/views/Workflows2.jsx b/frontend/src/views/Workflows2.jsx index 443181b5..f2b5e2f2 100644 --- a/frontend/src/views/Workflows2.jsx +++ b/frontend/src/views/Workflows2.jsx @@ -36,6 +36,8 @@ import { Button, TextField, FormControl, + FormControlLabel, + Switch, IconButton, Menu, MenuItem, @@ -53,6 +55,12 @@ import { Zoom, Collapse, Skeleton, + Table, + TableBody, + TableCell, + TableContainer, + TableHead, + TableRow, } from "@mui/material"; // Material UI Icons @@ -1075,6 +1083,10 @@ const Workflows2 = (props) => { const [downloadBranch, setDownloadBranch] = React.useState("main"); const [loadWorkflowsModalOpen, setLoadWorkflowsModalOpen] = React.useState(false); + const [syncMode, setSyncMode] = React.useState(false); + const [remoteWorkflows, setRemoteWorkflows] = React.useState([]); + const [remoteWorkflowsLoading, setRemoteWorkflowsLoading] = React.useState(false); + const [syncListModalOpen, setSyncListModalOpen] = React.useState(false); const [exportModalOpen, setExportModalOpen] = React.useState(false); const [exportData, setExportData] = React.useState(""); const [dialogModalOpen, setDialogModalOpen] = React.useState(false); @@ -5967,6 +5979,78 @@ const Workflows2 = (props) => { ); }); + const listRemoteWorkflows = () => { + const parsedData = { + url: downloadUrl, + branch: downloadBranch || "master", + list_only: true, + }; + if (field1.length > 0) parsedData["username"] = field1; + if (field2.length > 0) parsedData["password"] = field2; + + setRemoteWorkflowsLoading(true); + fetch(globalUrl + "/api/v1/workflows/download_remote", { + method: "POST", + mode: "cors", + headers: { Accept: "application/json" }, + body: JSON.stringify(parsedData), + credentials: "include", + }) + .then((r) => r.json()) + .then((responseJson) => { + setRemoteWorkflowsLoading(false); + if (responseJson.success && Array.isArray(responseJson.workflows)) { + setRemoteWorkflows(responseJson.workflows); + setLoadWorkflowsModalOpen(false); + setSyncListModalOpen(true); + } else { + toast("Failed to list remote workflows: " + (responseJson.reason || "Unknown error")); + } + }) + .catch((err) => { + setRemoteWorkflowsLoading(false); + toast(err.toString()); + }); + }; + + const handleRemoteWorkflowAction = (remoteWf, action) => { + const parsedData = { + url: downloadUrl, + branch: downloadBranch || "master", + original_workflow_id: remoteWf.id, + }; + if (action === "sync") { + parsedData["sync_to_id"] = remoteWf.org_workflow_id || remoteWf.id; + } + if (field1.length > 0) parsedData["username"] = field1; + if (field2.length > 0) parsedData["password"] = field2; + + toast(action === "sync" ? `Syncing "${remoteWf.name}"...` : `Importing "${remoteWf.name}"...`); + fetch(globalUrl + "/api/v1/workflows/download_remote", { + method: "POST", + mode: "cors", + headers: { Accept: "application/json" }, + body: JSON.stringify(parsedData), + credentials: "include", + }) + .then((r) => r.json()) + .then((responseJson) => { + if (responseJson.success) { + toast.success(action === "sync" ? `"${remoteWf.name}" synced successfully` : `"${remoteWf.name}" imported successfully`); + // Refresh org workflow list and update local exists_in_org state + getAvailableWorkflows(); + setRemoteWorkflows((prev) => + prev.map((w) => + w.id === remoteWf.id ? { ...w, exists_in_org: true, org_workflow_id: w.id } : w + ) + ); + } else { + toast("Failed: " + (responseJson.reason || "Unknown error")); + } + }) + .catch((err) => toast(err.toString())); + }; + const importWorkflowsFromUrl = (url) => { console.log("IMPORT WORKFLOWS FROM ", downloadUrl); @@ -6018,8 +6102,12 @@ const Workflows2 = (props) => { }; const handleGithubValidation = () => { - importWorkflowsFromUrl(downloadUrl); - setLoadWorkflowsModalOpen(false); + if (syncMode) { + listRemoteWorkflows(); + } else { + importWorkflowsFromUrl(downloadUrl); + setLoadWorkflowsModalOpen(false); + } } const workflowDownloadModalOpen = loadWorkflowsModalOpen ? ( @@ -6141,6 +6229,22 @@ const Workflows2 = (props) => { fullWidth /> +
+ setSyncMode(e.target.checked)} + color="primary" + /> + } + label={ + + Sync — list workflows before importing + + } + /> +
+ + + ) : null; + + const syncListModal = syncListModalOpen ? ( + setSyncListModalOpen(false)} + PaperProps={{ + sx: { + borderRadius: theme?.palette?.DialogStyle?.borderRadius, + border: theme?.palette?.DialogStyle?.border, + minWidth: "700px", + maxWidth: "900px", + fontFamily: theme?.typography?.fontFamily, + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + zIndex: 1000, + '& .MuiDialogContent-root': { + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + }, + '& .MuiDialogTitle-root': { + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + }, + '& .MuiDialogActions-root': { + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + }, + } + }} + > + +
+ Workflows from remote repository + setSyncListModalOpen(false)}> + + +
+ + + {remoteWorkflows.length === 0 ? ( + No workflow JSON files found in the repository. + ) : ( + + + + + Name + Folder + Last Updated + Action + + + + {remoteWorkflows.map((wf) => ( + + {wf.name} + {wf.folder_name || "—"} + + {wf.updated_at && wf.updated_at > 0 + ? new Date(wf.updated_at * 1000).toLocaleDateString() + : "—"} + + + {wf.exists_in_org ? ( + + ) : ( + + )} + + + ))} + +
+ + )} + + + @@ -6330,6 +6529,7 @@ const Workflows2 = (props) => { {publishModal} {aiAnnouncementModal} {workflowDownloadModalOpen} + {syncListModal} {/*!drawerOpen ?
From ee0efb26223ce32cd77415537f31f583095b4307 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Wed, 18 Mar 2026 22:14:53 +0530 Subject: [PATCH 18/61] new azure sync feature pre-alpha --- backend/go-app/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 4b21cd7d..fa62722c 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -26,7 +26,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v1.1.4 + github.com/shuffle/shuffle-shared v1.1.4-experimental github.com/shuffle/singul v0.0.29 golang.org/x/crypto v0.45.0 google.golang.org/api v0.236.0 From 368b5f63933c93c563a7db1a16d9ef5817966a18 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Thu, 19 Mar 2026 09:18:22 +0100 Subject: [PATCH 19/61] allow to enable debug mode for shuffle components Signed-off-by: Pascal Sthamer --- functions/kubernetes/charts/shuffle/README.md | 131 ++++++++++++++++-- .../shuffle/templates/backend/_helpers.tpl | 3 + .../shuffle/templates/orborus/_helpers.tpl | 3 + .../templates/shuffle-app/_helpers.tpl | 11 +- .../templates/shuffle-app/shuffle-apps.yaml | 2 +- .../templates/shuffle-worker/_helpers.tpl | 5 +- .../charts/shuffle/values.schema.json | 20 +++ .../kubernetes/charts/shuffle/values.yaml | 15 ++ 8 files changed, 176 insertions(+), 14 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/README.md b/functions/kubernetes/charts/shuffle/README.md index 7c1315b9..a9aede23 100644 --- a/functions/kubernetes/charts/shuffle/README.md +++ b/functions/kubernetes/charts/shuffle/README.md @@ -215,7 +215,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia ## Parameters -##### Global parameters +###### Global parameters | Name | Description | Value | | ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | @@ -225,7 +225,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `global.compatibility.openshift.adaptSecurityContext` | Adapt the securityContext sections of the deployment to make them compatible with Openshift restricted-v2 SCC: remove runAsUser, runAsGroup and fsGroup and let the platform use their allowed default IDs. Possible values: auto (apply if the detected running cluster is Openshift), force (perform the adaptation always), disabled (do not perform adaptation) | `auto` | | `global.compatibility.omitEmptySeLinuxOptions` | If set to true, removes the seLinuxOptions from the securityContexts when it is set to an empty object | `false` | -##### Common parameters +###### Common parameters | Name | Description | Value | | ------------------------ | --------------------------------------------------------------------------------------- | --------------- | @@ -241,7 +241,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `diagnosticMode.command` | Command to override all containers in the chart release | `["sleep"]` | | `diagnosticMode.args` | Args to override all containers in the chart release | `["infinity"]` | -##### Shared Shuffle Parameters +###### Shared Shuffle Parameters | Name | Description | Value | | -------------------------- | -------------------------------------------------------------------------------------------------------------------------- | --------------- | @@ -251,7 +251,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `shuffle.appBaseImageName` | The base image used for shuffle apps. The final image for an app is //: | `frikky` | | `shuffle.timezone` | The timezone used by Shuffle | `Europe/Berlin` | -##### backend Parameters +###### backend Parameters | Name | Description | Value | | ----------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------- | @@ -356,6 +356,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `backend.networkPolicy.allowExternalEgress` | Allow the pod to access any range of port and all destinations. | `true` | | `backend.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` | | `backend.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | +| `backend.debug` | Enable debug mode for backend, which enables extra logging. orborus.debug, worker.debug, and app.debug should be set to identical values, if shuffle manages these deployments. | `false` | | `backend.cleanupSchedule` | The interval in seconds at which the cleanup job runs | `300` | | `backend.openSearch.url` | The URL at which OpenSearch is available | `http://{{ .Release.Name }}-opensearch:9200` | | `backend.openSearch.username` | The username that is used for authenticating with OpenSearch | `admin` | @@ -366,7 +367,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `backend.apps.downloadBranch` | The branch from which apps should be downloaded on startup. | `master` | | `backend.apps.forceUpdate` | Force an update of apps on startup. | `false` | -##### frontend Parameters +###### frontend Parameters | Name | Description | Value | | ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------- | @@ -472,7 +473,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `frontend.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` | | `frontend.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | -##### orborus Parameters +###### orborus Parameters | Name | Description | Value | | ----------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------- | @@ -575,10 +576,11 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `orborus.networkPolicy.allowExternalEgress` | Allow the pod to access any range of port and all destinations. | `true` | | `orborus.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` | | `orborus.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | +| `orborus.debug` | Enable debug mode for orborus, which enables extra logging. orborus.debug, worker.debug, and app.debug should be set to identical values, if shuffle manages these deployments. | `false` | | `orborus.executionConcurrency` | The maximum amount of concurrent workflow executions per worker | `25` | | `orborus.manageWorkerDeployments` | Whether workers are deployed and managed by orborus. When disabled, every worker is expected to be already deployed (see worker.enableHelmDeployment). | `true` | -##### worker Parameters +###### worker Parameters | Name | Description | Value | | ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------ | @@ -684,9 +686,10 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `worker.networkPolicy.allowExternalEgress` | Allow the pod to access any range of port and all destinations. | `true` | | `worker.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` | | `worker.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | +| `worker.debug` | Enable debug mode for worker, which enables extra logging. orborus.debug, worker.debug, and app.debug should be set to identical values, if shuffle manages these deployments. | `false` | | `worker.manageAppDeployments` | Whether apps are deployed and managed by worker. When disabled, every used app is expected to to be already deployed (see apps.enabled). | `true` | -##### app Parameters +###### app Parameters | Name | Description | Value | | ------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | @@ -789,12 +792,122 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `app.networkPolicy.allowExternalEgress` | Allow the pod to access any range of port and all destinations. | `true` | | `app.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` | | `app.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | +| `app.debug` | Enable debug mode for app, which enables extra logging | `false` | | `app.mountTmpVolume` | Whether a writable /tmp emptyDir volume should be mounted to the app. | `true` | | `app.exposedContainerPort` | The port that shuffle app containers will listen on for new requests. | `80` | | `app.sdkTimeout` | The timeout in seconds for app actions. | `300` | | `app.disableLogs` | Do not capture app logs. By default, app logs are captured, so that they are visible in the frontend. | `false` | -##### Parameters to deploy apps using helm +###### Parameters to deploy apps using helm + +| Name | Description | Value | +| ----------------------------- | -------------------------------------------------- | ------- | +| `apps.enabled` | Whether apps should be deployed using helm. | `false` | +| `apps.shuffleTools.enabled` | Whether the shuffle-tools app is enabled | `true` | +| `apps.shuffleTools.version` | The version of the shuffle-tools app to deploy. | `1.2.0` | +| `apps.shuffleSubflow.enabled` | Whether the shuffle-subflow app is enabled | `true` | +| `apps.shuffleSubflow.version` | The version of the shuffle-subflow app to deploy. | `1.1.0` | +| `apps.http.enabled` | Whether the http app is enabled | `true` | +| `apps.http.version` | The version of the http app to deploy. | `1.4.0` | +| `apps.MY_APP.app` | The name of the app (required, e.g. shuffle-tools) | | +| `apps.MY_APP.version` | The version of the app (required, e.g. 1.2.0) | | + +###### Traffic Exposure Parameters + +| Name | Description | Value | +| -------------------------- | ----------------------------------------------------------------------------------------------------- | --------------- | +| `ingress.enabled` | Enable ingress record generation for frontend and backend | `false` | +| `ingress.pathType` | Ingress path type for the frontend path | `Prefix` | +| `ingress.backendPathType` | Ingress path type for the backend path | `Prefix` | +| `ingress.apiVersion` | Force Ingress API version (automatically detected if not set) | `""` | +| `ingress.hostname` | Default host for the ingress record | `shuffle.local` | +| `ingress.ingressClassName` | IngressClass that will be be used to implement the Ingress (Kubernetes 1.18+) | `nginx` | +| `ingress.path` | Ingress path for Shuffle frontend | `"/"` | +| `ingress.backendPath` | Ingress path for Shuffle backend | `"/api/"` | +| `ingress.annotations` | Additional annotations for the Ingress resource. | `{}` | +| `ingress.tls` | Enable TLS configuration for the host defined at `ingress.hostname` parameter | `false` | +| `ingress.selfSigned` | Create a TLS secret for this ingress record using self-signed certificates generated by Helm | `false` | +| `ingress.extraHosts` | An array with additional hostname(s) to be covered with the ingress record | `[]` | +| `ingress.extraPaths` | An array with additional arbitrary paths that may need to be added to the ingress under the main host | `[]` | +| `ingress.extraTls` | TLS configuration for additional hostname(s) to be covered with this ingress record | `[]` | +| `ingress.secrets` | Custom TLS certificates as secrets | `[]` | +| `ingress.extraRules` | Additional rules to be covered with this ingress record | `[]` | + +###### Istio Parameters + +| Name | Description | Value | +| --------------------------------------- | ------------------------------------------------------------------------------- | ------------------------ | +| `istio.enabled` | Enable creation of an Istio Gateway and VirtualService for frontend and backend | `false` | +| `istio.apiVersion` | The istio apiVersion to use for Gateway and VirtualService resources | `networking.istio.io/v1` | +| `istio.hosts` | One or more hosts exposed by Istio | `[]` | +| `istio.gateway.annotations` | Additional annotations for the Gateway resource | `{}` | +| `istio.gateway.selector` | The selector matches the ingress gateway pod labels | `{ istio: ingress }` | +| `istio.gateway.http.enabled` | Enable HTTP server port 80 | `true` | +| `istio.gateway.http.httpsRedirect` | If set to true, a 301 redirect is send for all HTTP connections | `false` | +| `istio.gateway.https.enabled` | Enable HTTPS server on port 443 | `false` | +| `istio.gateway.https.tlsCredentialName` | The name of the secret that holds the TLS certs including the CA certificates. | `""` | +| `istio.gateway.https.tlsCipherSuites` | If specified, only support the specified cipher list. | `[]` | +| `istio.gateway.extraServers` | Additional servers for the Gateway resource | `[]` | +| `istio.virtualService.annotations` | Additional annotations for the VirtualService resource. | `{}` | +| `istio.virtualService.backendHeaders` | Header manipulation rules for backend traffic | `{}` | +| `istio.virtualService.frontendHeaders` | Header manipulation rules for frontend traffic | `{}` | + +###### Persistence Parameters + +| Name | Description | Value | +| ------------------------------------- | ------------------------------------------------- | ------------------- | +| `persistence.enabled` | Enable persistence using Persistent Volume Claims | `true` | +| `persistence.apps.existingClaim` | Name of an existing PVC to use | `""` | +| `persistence.apps.storageClass` | PVC Storage Class for shuffle-apps volume | `""` | +| `persistence.apps.subPath` | The sub path used in the volume | `""` | +| `persistence.apps.accessModes` | The access mode of the volume | `["ReadWriteOnce"]` | +| `persistence.apps.size` | The size of the volume | `5Gi` | +| `persistence.apps.annotations` | Annotations for the PVC | `{}` | +| `persistence.apps.selector` | Selector to match an existing Persistent Volume | `{}` | +| `persistence.appBuilder.storageClass` | PVC Storage Class for backend-apps-claim volume | `""` | +| `persistence.appBuilder.accessModes` | The access mode of the volume | `["ReadWriteOnce"]` | +| `persistence.appBuilder.size` | The size of the volume | `5Gi` | +| `persistence.appBuilder.annotations` | Annotations for the PVC | `{}` | +| `persistence.appBuilder.selector` | Selector to match an existing Persistent Volume | `{}` | +| `persistence.files.existingClaim` | Name of an existing PVC to use | `""` | +| `persistence.files.storageClass` | PVC Storage Class for shuffle-files volume | `""` | +| `persistence.files.subPath` | The sub path used in the volume | `""` | +| `persistence.files.accessModes` | The access mode of the volume | `["ReadWriteOnce"]` | +| `persistence.files.size` | The size of the volume | `5Gi` | +| `persistence.files.annotations` | Annotations for the PVC | `{}` | +| `persistence.files.selector` | Selector to match an existing Persistent Volume | `{}` | + +###### Init Container Parameters + +| Name | Description | Value | +| ----------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------ | +| `volumePermissions.enabled` | Enable init container that changes the owner/group of the PV mount point to `runAsUser:fsGroup` | `false` | +| `volumePermissions.image.registry` | OS Shell + Utility image registry | `docker.io` | +| `volumePermissions.image.repository` | OS Shell + Utility image repository | `bitnamilegacy/os-shell` | +| `volumePermissions.image.tag` | OS Shell + Utility image tag (immutable tags are recommended) | `12-debian-12-r30` | +| `volumePermissions.image.pullPolicy` | OS Shell + Utility image pull policy | `IfNotPresent` | +| `volumePermissions.image.pullSecrets` | OS Shell + Utility image pull secrets | `[]` | +| `volumePermissions.resourcesPreset` | Set init container resources according to one common preset (allowed values: none, nano, small, medium, large, xlarge, 2xlarge). This is ignored if volumePermissions.resources is set (volumePermissions.resources is recommended for production). | `nano` | +| `volumePermissions.resources` | Set init container requests and limits for different resources like CPU or memory (essential for production workloads) | `{}` | +| `volumePermissions.containerSecurityContext.enabled` | Enabled init container' Security Context | `true` | +| `volumePermissions.containerSecurityContext.seLinuxOptions` | Set SELinux options in init container | `{}` | +| `volumePermissions.containerSecurityContext.runAsUser` | Set init container's Security Context runAsUser | `0` | + +###### OpenSearch Parameters + +| Name | Description | Value | +| -------------------- | ----------------------------------------------------- | ------ | +| `opensearch.enabled` | Switch to enable or disable the opensearch helm chart | `true` | + +###### Vault Parameters + +| Name | Description | Value | +| --------------- | -------------------------------------------------------------------------- | ----- | +| `vault.role` | Specify the Vault role, which should be used to get the secret from Vault. | `""` | +| `vault.secrets` | A list of VaultSecrets to create | `[]` | + +###### Other Parameters + | Name | Description | Value | | ----------------------------- | -------------------------------------------------- | ------- | diff --git a/functions/kubernetes/charts/shuffle/templates/backend/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/backend/_helpers.tpl index 48bd6543..1d61b12e 100644 --- a/functions/kubernetes/charts/shuffle/templates/backend/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/backend/_helpers.tpl @@ -113,4 +113,7 @@ SHUFFLE_OPENSEARCH_INDEX_PREFIX: "{{ .Values.backend.openSearch.indexPrefix }}" SHUFFLE_RERUN_SCHEDULE: "{{ .Values.backend.cleanupSchedule }}" TZ: "{{ .Values.shuffle.timezone }}" REGISTRY_URL: "{{ .Values.shuffle.appRegistry }}" # Used by app builder +{{- if .Values.backend.debug }} +DEBUG: "true" +{{- end }} {{- end -}} \ No newline at end of file diff --git a/functions/kubernetes/charts/shuffle/templates/orborus/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/orborus/_helpers.tpl index 3ced6821..c69c84f3 100644 --- a/functions/kubernetes/charts/shuffle/templates/orborus/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/orborus/_helpers.tpl @@ -89,6 +89,9 @@ TZ: "{{ .Values.shuffle.timezone }}" BASE_URL: {{ include "shuffle.backend.baseUrl" . | quote }} KUBERNETES_NAMESPACE: "{{ .Release.Namespace }}" SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY: {{ .Values.orborus.executionConcurrency | quote }} +{{- if .Values.orborus.debug }} +DEBUG: "true" +{{- end }} {{- if .Values.orborus.manageWorkerDeployments }} # Shuffle worker configuration diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl index 5963f9c1..bcce6348 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl @@ -121,9 +121,14 @@ Usage: {{/* Return the environment variables of shuffle apps in the format KEY: VALUE +Usage: +{{- include "shuffle.appInstance.env" (dict "app" $appValues "context" $) -}} */}} {{- define "shuffle.appInstance.env" -}} -SHUFFLE_APP_SDK_TIMEOUT: {{ .Values.app.sdkTimeout | quote }} -SHUFFLE_APP_EXPOSED_PORT: {{ .Values.app.exposedContainerPort | quote }} -SHUFFLE_LOGS_DISABLED: {{ .Values.app.disableLogs | quote }} +SHUFFLE_APP_SDK_TIMEOUT: {{ .app.sdkTimeout | quote }} +SHUFFLE_APP_EXPOSED_PORT: {{ .app.exposedContainerPort | quote }} +SHUFFLE_LOGS_DISABLED: {{ .app.disableLogs | quote }} +{{- if .app.debug }} +DEBUG: "true" +{{- end }} {{- end -}} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml index af04851f..6ee29a84 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml @@ -168,7 +168,7 @@ spec: value: {{ include "shuffle.backend.baseUrl" $ | quote }} - name: SHUFFLE_SWARM_CONFIG value: run # Shuffle Worker requires this to be set even when using K8s instead of swarm - {{- $env := include "shuffle.appInstance.env" $ | fromYaml }} + {{- $env := include "shuffle.appInstance.env" (dict "app" $appValues "context" $) | fromYaml }} {{- range $key, $val := $env }} - name: {{ $key | quote }} value: {{ $val | quote }} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl index 8feafa76..2e57fe04 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl @@ -139,6 +139,9 @@ SHUFFLE_SWARM_CONFIG: "run" # Shuffle Worker requires this to be set even when u BASE_URL: {{ include "shuffle.backend.baseUrl" . | quote }} SHUFFLE_APP_EXPOSED_PORT: {{ .Values.app.exposedContainerPort | quote }} WORKER_HOSTNAME: {{ include "shuffle.worker.hostname" . }} +{{- if .Values.worker.debug }} +DEBUG: "true" +{{- end }} {{- if .Values.worker.manageAppDeployments }} # Shuffle app images @@ -177,6 +180,6 @@ SHUFFLE_APP_EPHEMERAL_STORAGE_LIMIT: {{ (index $appResources.limits "ephemeral-s {{- end }} # Include shuffle app environment variables. Worker passes them down to apps, when creating their deployment. -{{ include "shuffle.appInstance.env" . }} +{{ include "shuffle.appInstance.env" (dict "app" .Values.app "context" $) -}} {{- end }} {{- end -}} diff --git a/functions/kubernetes/charts/shuffle/values.schema.json b/functions/kubernetes/charts/shuffle/values.schema.json index ca262866..5ac3da8e 100644 --- a/functions/kubernetes/charts/shuffle/values.schema.json +++ b/functions/kubernetes/charts/shuffle/values.schema.json @@ -771,6 +771,11 @@ } } }, + "debug": { + "type": "boolean", + "description": "Enable debug mode for backend, which enables extra logging. orborus.debug, worker.debug, and app.debug should be set to identical values, if shuffle manages these deployments.", + "default": false + }, "cleanupSchedule": { "type": "number", "description": "The interval in seconds at which the cleanup job runs", @@ -2059,6 +2064,11 @@ } } }, + "debug": { + "type": "boolean", + "description": "Enable debug mode for orborus, which enables extra logging. orborus.debug, worker.debug, and app.debug should be set to identical values, if shuffle manages these deployments.", + "default": false + }, "executionConcurrency": { "type": "number", "description": "The maximum amount of concurrent workflow executions per worker", @@ -2703,6 +2713,11 @@ } } }, + "debug": { + "type": "boolean", + "description": "Enable debug mode for worker, which enables extra logging. orborus.debug, worker.debug, and app.debug should be set to identical values, if shuffle manages these deployments.", + "default": false + }, "manageAppDeployments": { "type": "boolean", "description": "Whether apps are deployed and managed by worker. When disabled, every used app is expected to to be already deployed (see apps.enabled).", @@ -3322,6 +3337,11 @@ } } }, + "debug": { + "type": "boolean", + "description": "Enable debug mode for app, which enables extra logging", + "default": false + }, "mountTmpVolume": { "type": "boolean", "description": "Whether a writable /tmp emptyDir volume should be mounted to the app.", diff --git a/functions/kubernetes/charts/shuffle/values.yaml b/functions/kubernetes/charts/shuffle/values.yaml index 590d2c10..27e04aa2 100644 --- a/functions/kubernetes/charts/shuffle/values.yaml +++ b/functions/kubernetes/charts/shuffle/values.yaml @@ -516,6 +516,10 @@ backend: ## extraEgress: [] + ## @param backend.debug Enable debug mode for backend, which enables extra logging. orborus.debug, worker.debug, and app.debug should be set to identical values, if shuffle manages these deployments. + ## + debug: false + ## @param backend.cleanupSchedule The interval in seconds at which the cleanup job runs ## cleanupSchedule: 300 @@ -1329,6 +1333,10 @@ orborus: ## extraEgress: [] + ## @param orborus.debug Enable debug mode for orborus, which enables extra logging. orborus.debug, worker.debug, and app.debug should be set to identical values, if shuffle manages these deployments. + ## + debug: false + ## @param orborus.executionConcurrency The maximum amount of concurrent workflow executions per worker ## executionConcurrency: 25 @@ -1739,6 +1747,10 @@ worker: ## extraEgress: [] + ## @param worker.debug Enable debug mode for worker, which enables extra logging. orborus.debug, worker.debug, and app.debug should be set to identical values, if shuffle manages these deployments. + ## + debug: false + ## @param worker.manageAppDeployments Whether apps are deployed and managed by worker. When disabled, every used app is expected to to be already deployed (see apps.enabled). ## This effectively removes required RBAC permissions from the shuffle-worker service account to create deployments and services. ## The worker might still attempt to create kubernetes objects, resulting in an error. There is currently no way to tell the worker, that it should not manage k8s resources. @@ -2124,6 +2136,9 @@ app: ## extraEgress: [] + ## @param app.debug Enable debug mode for app, which enables extra logging + ## + debug: false ## @param app.mountTmpVolume Whether a writable /tmp emptyDir volume should be mounted to the app. ## mountTmpVolume: true From 1279bb09494910a339f7ff70c70bd29f0d16058e Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Thu, 19 Mar 2026 09:21:47 +0100 Subject: [PATCH 20/61] fix readme Signed-off-by: Pascal Sthamer --- functions/kubernetes/charts/shuffle/README.md | 344 +----------------- 1 file changed, 10 insertions(+), 334 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/README.md b/functions/kubernetes/charts/shuffle/README.md index a9aede23..fa20d558 100644 --- a/functions/kubernetes/charts/shuffle/README.md +++ b/functions/kubernetes/charts/shuffle/README.md @@ -215,7 +215,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia ## Parameters -###### Global parameters +### Global parameters | Name | Description | Value | | ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | @@ -225,7 +225,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `global.compatibility.openshift.adaptSecurityContext` | Adapt the securityContext sections of the deployment to make them compatible with Openshift restricted-v2 SCC: remove runAsUser, runAsGroup and fsGroup and let the platform use their allowed default IDs. Possible values: auto (apply if the detected running cluster is Openshift), force (perform the adaptation always), disabled (do not perform adaptation) | `auto` | | `global.compatibility.omitEmptySeLinuxOptions` | If set to true, removes the seLinuxOptions from the securityContexts when it is set to an empty object | `false` | -###### Common parameters +### Common parameters | Name | Description | Value | | ------------------------ | --------------------------------------------------------------------------------------- | --------------- | @@ -241,7 +241,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `diagnosticMode.command` | Command to override all containers in the chart release | `["sleep"]` | | `diagnosticMode.args` | Args to override all containers in the chart release | `["infinity"]` | -###### Shared Shuffle Parameters +### Shared Shuffle Parameters | Name | Description | Value | | -------------------------- | -------------------------------------------------------------------------------------------------------------------------- | --------------- | @@ -251,7 +251,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `shuffle.appBaseImageName` | The base image used for shuffle apps. The final image for an app is //: | `frikky` | | `shuffle.timezone` | The timezone used by Shuffle | `Europe/Berlin` | -###### backend Parameters +### backend Parameters | Name | Description | Value | | ----------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------- | @@ -367,7 +367,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `backend.apps.downloadBranch` | The branch from which apps should be downloaded on startup. | `master` | | `backend.apps.forceUpdate` | Force an update of apps on startup. | `false` | -###### frontend Parameters +### frontend Parameters | Name | Description | Value | | ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------- | @@ -473,7 +473,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `frontend.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` | | `frontend.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | -###### orborus Parameters +### orborus Parameters | Name | Description | Value | | ----------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------- | @@ -580,7 +580,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `orborus.executionConcurrency` | The maximum amount of concurrent workflow executions per worker | `25` | | `orborus.manageWorkerDeployments` | Whether workers are deployed and managed by orborus. When disabled, every worker is expected to be already deployed (see worker.enableHelmDeployment). | `true` | -###### worker Parameters +### worker Parameters | Name | Description | Value | | ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------ | @@ -689,7 +689,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `worker.debug` | Enable debug mode for worker, which enables extra logging. orborus.debug, worker.debug, and app.debug should be set to identical values, if shuffle manages these deployments. | `false` | | `worker.manageAppDeployments` | Whether apps are deployed and managed by worker. When disabled, every used app is expected to to be already deployed (see apps.enabled). | `true` | -###### app Parameters +### app Parameters | Name | Description | Value | | ------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | @@ -798,332 +798,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `app.sdkTimeout` | The timeout in seconds for app actions. | `300` | | `app.disableLogs` | Do not capture app logs. By default, app logs are captured, so that they are visible in the frontend. | `false` | -###### Parameters to deploy apps using helm - -| Name | Description | Value | -| ----------------------------- | -------------------------------------------------- | ------- | -| `apps.enabled` | Whether apps should be deployed using helm. | `false` | -| `apps.shuffleTools.enabled` | Whether the shuffle-tools app is enabled | `true` | -| `apps.shuffleTools.version` | The version of the shuffle-tools app to deploy. | `1.2.0` | -| `apps.shuffleSubflow.enabled` | Whether the shuffle-subflow app is enabled | `true` | -| `apps.shuffleSubflow.version` | The version of the shuffle-subflow app to deploy. | `1.1.0` | -| `apps.http.enabled` | Whether the http app is enabled | `true` | -| `apps.http.version` | The version of the http app to deploy. | `1.4.0` | -| `apps.MY_APP.app` | The name of the app (required, e.g. shuffle-tools) | | -| `apps.MY_APP.version` | The version of the app (required, e.g. 1.2.0) | | - -###### Traffic Exposure Parameters - -| Name | Description | Value | -| -------------------------- | ----------------------------------------------------------------------------------------------------- | --------------- | -| `ingress.enabled` | Enable ingress record generation for frontend and backend | `false` | -| `ingress.pathType` | Ingress path type for the frontend path | `Prefix` | -| `ingress.backendPathType` | Ingress path type for the backend path | `Prefix` | -| `ingress.apiVersion` | Force Ingress API version (automatically detected if not set) | `""` | -| `ingress.hostname` | Default host for the ingress record | `shuffle.local` | -| `ingress.ingressClassName` | IngressClass that will be be used to implement the Ingress (Kubernetes 1.18+) | `nginx` | -| `ingress.path` | Ingress path for Shuffle frontend | `"/"` | -| `ingress.backendPath` | Ingress path for Shuffle backend | `"/api/"` | -| `ingress.annotations` | Additional annotations for the Ingress resource. | `{}` | -| `ingress.tls` | Enable TLS configuration for the host defined at `ingress.hostname` parameter | `false` | -| `ingress.selfSigned` | Create a TLS secret for this ingress record using self-signed certificates generated by Helm | `false` | -| `ingress.extraHosts` | An array with additional hostname(s) to be covered with the ingress record | `[]` | -| `ingress.extraPaths` | An array with additional arbitrary paths that may need to be added to the ingress under the main host | `[]` | -| `ingress.extraTls` | TLS configuration for additional hostname(s) to be covered with this ingress record | `[]` | -| `ingress.secrets` | Custom TLS certificates as secrets | `[]` | -| `ingress.extraRules` | Additional rules to be covered with this ingress record | `[]` | - -###### Istio Parameters - -| Name | Description | Value | -| --------------------------------------- | ------------------------------------------------------------------------------- | ------------------------ | -| `istio.enabled` | Enable creation of an Istio Gateway and VirtualService for frontend and backend | `false` | -| `istio.apiVersion` | The istio apiVersion to use for Gateway and VirtualService resources | `networking.istio.io/v1` | -| `istio.hosts` | One or more hosts exposed by Istio | `[]` | -| `istio.gateway.annotations` | Additional annotations for the Gateway resource | `{}` | -| `istio.gateway.selector` | The selector matches the ingress gateway pod labels | `{ istio: ingress }` | -| `istio.gateway.http.enabled` | Enable HTTP server port 80 | `true` | -| `istio.gateway.http.httpsRedirect` | If set to true, a 301 redirect is send for all HTTP connections | `false` | -| `istio.gateway.https.enabled` | Enable HTTPS server on port 443 | `false` | -| `istio.gateway.https.tlsCredentialName` | The name of the secret that holds the TLS certs including the CA certificates. | `""` | -| `istio.gateway.https.tlsCipherSuites` | If specified, only support the specified cipher list. | `[]` | -| `istio.gateway.extraServers` | Additional servers for the Gateway resource | `[]` | -| `istio.virtualService.annotations` | Additional annotations for the VirtualService resource. | `{}` | -| `istio.virtualService.backendHeaders` | Header manipulation rules for backend traffic | `{}` | -| `istio.virtualService.frontendHeaders` | Header manipulation rules for frontend traffic | `{}` | - -###### Persistence Parameters - -| Name | Description | Value | -| ------------------------------------- | ------------------------------------------------- | ------------------- | -| `persistence.enabled` | Enable persistence using Persistent Volume Claims | `true` | -| `persistence.apps.existingClaim` | Name of an existing PVC to use | `""` | -| `persistence.apps.storageClass` | PVC Storage Class for shuffle-apps volume | `""` | -| `persistence.apps.subPath` | The sub path used in the volume | `""` | -| `persistence.apps.accessModes` | The access mode of the volume | `["ReadWriteOnce"]` | -| `persistence.apps.size` | The size of the volume | `5Gi` | -| `persistence.apps.annotations` | Annotations for the PVC | `{}` | -| `persistence.apps.selector` | Selector to match an existing Persistent Volume | `{}` | -| `persistence.appBuilder.storageClass` | PVC Storage Class for backend-apps-claim volume | `""` | -| `persistence.appBuilder.accessModes` | The access mode of the volume | `["ReadWriteOnce"]` | -| `persistence.appBuilder.size` | The size of the volume | `5Gi` | -| `persistence.appBuilder.annotations` | Annotations for the PVC | `{}` | -| `persistence.appBuilder.selector` | Selector to match an existing Persistent Volume | `{}` | -| `persistence.files.existingClaim` | Name of an existing PVC to use | `""` | -| `persistence.files.storageClass` | PVC Storage Class for shuffle-files volume | `""` | -| `persistence.files.subPath` | The sub path used in the volume | `""` | -| `persistence.files.accessModes` | The access mode of the volume | `["ReadWriteOnce"]` | -| `persistence.files.size` | The size of the volume | `5Gi` | -| `persistence.files.annotations` | Annotations for the PVC | `{}` | -| `persistence.files.selector` | Selector to match an existing Persistent Volume | `{}` | - -###### Init Container Parameters - -| Name | Description | Value | -| ----------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------ | -| `volumePermissions.enabled` | Enable init container that changes the owner/group of the PV mount point to `runAsUser:fsGroup` | `false` | -| `volumePermissions.image.registry` | OS Shell + Utility image registry | `docker.io` | -| `volumePermissions.image.repository` | OS Shell + Utility image repository | `bitnamilegacy/os-shell` | -| `volumePermissions.image.tag` | OS Shell + Utility image tag (immutable tags are recommended) | `12-debian-12-r30` | -| `volumePermissions.image.pullPolicy` | OS Shell + Utility image pull policy | `IfNotPresent` | -| `volumePermissions.image.pullSecrets` | OS Shell + Utility image pull secrets | `[]` | -| `volumePermissions.resourcesPreset` | Set init container resources according to one common preset (allowed values: none, nano, small, medium, large, xlarge, 2xlarge). This is ignored if volumePermissions.resources is set (volumePermissions.resources is recommended for production). | `nano` | -| `volumePermissions.resources` | Set init container requests and limits for different resources like CPU or memory (essential for production workloads) | `{}` | -| `volumePermissions.containerSecurityContext.enabled` | Enabled init container' Security Context | `true` | -| `volumePermissions.containerSecurityContext.seLinuxOptions` | Set SELinux options in init container | `{}` | -| `volumePermissions.containerSecurityContext.runAsUser` | Set init container's Security Context runAsUser | `0` | - -###### OpenSearch Parameters - -| Name | Description | Value | -| -------------------- | ----------------------------------------------------- | ------ | -| `opensearch.enabled` | Switch to enable or disable the opensearch helm chart | `true` | - -###### Vault Parameters - -| Name | Description | Value | -| --------------- | -------------------------------------------------------------------------- | ----- | -| `vault.role` | Specify the Vault role, which should be used to get the secret from Vault. | `""` | -| `vault.secrets` | A list of VaultSecrets to create | `[]` | - -###### Other Parameters - - -| Name | Description | Value | -| ----------------------------- | -------------------------------------------------- | ------- | -| `apps.enabled` | Whether apps should be deployed using helm. | `false` | -| `apps.shuffleTools.enabled` | Whether the shuffle-tools app is enabled | `true` | -| `apps.shuffleTools.version` | The version of the shuffle-tools app to deploy. | `1.2.0` | -| `apps.shuffleSubflow.enabled` | Whether the shuffle-subflow app is enabled | `true` | -| `apps.shuffleSubflow.version` | The version of the shuffle-subflow app to deploy. | `1.1.0` | -| `apps.http.enabled` | Whether the http app is enabled | `true` | -| `apps.http.version` | The version of the http app to deploy. | `1.4.0` | -| `apps.MY_APP.app` | The name of the app (required, e.g. shuffle-tools) | | -| `apps.MY_APP.version` | The version of the app (required, e.g. 1.2.0) | | - -##### Traffic Exposure Parameters - -| Name | Description | Value | -| -------------------------- | ----------------------------------------------------------------------------------------------------- | --------------- | -| `ingress.enabled` | Enable ingress record generation for frontend and backend | `false` | -| `ingress.pathType` | Ingress path type for the frontend path | `Prefix` | -| `ingress.backendPathType` | Ingress path type for the backend path | `Prefix` | -| `ingress.apiVersion` | Force Ingress API version (automatically detected if not set) | `""` | -| `ingress.hostname` | Default host for the ingress record | `shuffle.local` | -| `ingress.ingressClassName` | IngressClass that will be be used to implement the Ingress (Kubernetes 1.18+) | `nginx` | -| `ingress.path` | Ingress path for Shuffle frontend | `"/"` | -| `ingress.backendPath` | Ingress path for Shuffle backend | `"/api/"` | -| `ingress.annotations` | Additional annotations for the Ingress resource. | `{}` | -| `ingress.tls` | Enable TLS configuration for the host defined at `ingress.hostname` parameter | `false` | -| `ingress.selfSigned` | Create a TLS secret for this ingress record using self-signed certificates generated by Helm | `false` | -| `ingress.extraHosts` | An array with additional hostname(s) to be covered with the ingress record | `[]` | -| `ingress.extraPaths` | An array with additional arbitrary paths that may need to be added to the ingress under the main host | `[]` | -| `ingress.extraTls` | TLS configuration for additional hostname(s) to be covered with this ingress record | `[]` | -| `ingress.secrets` | Custom TLS certificates as secrets | `[]` | -| `ingress.extraRules` | Additional rules to be covered with this ingress record | `[]` | - -##### Istio Parameters - -| Name | Description | Value | -| --------------------------------------- | ------------------------------------------------------------------------------- | ------------------------ | -| `istio.enabled` | Enable creation of an Istio Gateway and VirtualService for frontend and backend | `false` | -| `istio.apiVersion` | The istio apiVersion to use for Gateway and VirtualService resources | `networking.istio.io/v1` | -| `istio.hosts` | One or more hosts exposed by Istio | `[]` | -| `istio.gateway.annotations` | Additional annotations for the Gateway resource | `{}` | -| `istio.gateway.selector` | The selector matches the ingress gateway pod labels | `{ istio: ingress }` | -| `istio.gateway.http.enabled` | Enable HTTP server port 80 | `true` | -| `istio.gateway.http.httpsRedirect` | If set to true, a 301 redirect is send for all HTTP connections | `false` | -| `istio.gateway.https.enabled` | Enable HTTPS server on port 443 | `false` | -| `istio.gateway.https.tlsCredentialName` | The name of the secret that holds the TLS certs including the CA certificates. | `""` | -| `istio.gateway.https.tlsCipherSuites` | If specified, only support the specified cipher list. | `[]` | -| `istio.gateway.extraServers` | Additional servers for the Gateway resource | `[]` | -| `istio.virtualService.annotations` | Additional annotations for the VirtualService resource. | `{}` | -| `istio.virtualService.backendHeaders` | Header manipulation rules for backend traffic | `{}` | -| `istio.virtualService.frontendHeaders` | Header manipulation rules for frontend traffic | `{}` | - -##### Persistence Parameters - -| Name | Description | Value | -| ------------------------------------- | ------------------------------------------------- | ------------------- | -| `persistence.enabled` | Enable persistence using Persistent Volume Claims | `true` | -| `persistence.apps.existingClaim` | Name of an existing PVC to use | `""` | -| `persistence.apps.storageClass` | PVC Storage Class for shuffle-apps volume | `""` | -| `persistence.apps.subPath` | The sub path used in the volume | `""` | -| `persistence.apps.accessModes` | The access mode of the volume | `["ReadWriteOnce"]` | -| `persistence.apps.size` | The size of the volume | `5Gi` | -| `persistence.apps.annotations` | Annotations for the PVC | `{}` | -| `persistence.apps.selector` | Selector to match an existing Persistent Volume | `{}` | -| `persistence.appBuilder.storageClass` | PVC Storage Class for backend-apps-claim volume | `""` | -| `persistence.appBuilder.accessModes` | The access mode of the volume | `["ReadWriteOnce"]` | -| `persistence.appBuilder.size` | The size of the volume | `5Gi` | -| `persistence.appBuilder.annotations` | Annotations for the PVC | `{}` | -| `persistence.appBuilder.selector` | Selector to match an existing Persistent Volume | `{}` | -| `persistence.files.existingClaim` | Name of an existing PVC to use | `""` | -| `persistence.files.storageClass` | PVC Storage Class for shuffle-files volume | `""` | -| `persistence.files.subPath` | The sub path used in the volume | `""` | -| `persistence.files.accessModes` | The access mode of the volume | `["ReadWriteOnce"]` | -| `persistence.files.size` | The size of the volume | `5Gi` | -| `persistence.files.annotations` | Annotations for the PVC | `{}` | -| `persistence.files.selector` | Selector to match an existing Persistent Volume | `{}` | - -##### Init Container Parameters - -| Name | Description | Value | -| ----------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------ | -| `volumePermissions.enabled` | Enable init container that changes the owner/group of the PV mount point to `runAsUser:fsGroup` | `false` | -| `volumePermissions.image.registry` | OS Shell + Utility image registry | `docker.io` | -| `volumePermissions.image.repository` | OS Shell + Utility image repository | `bitnamilegacy/os-shell` | -| `volumePermissions.image.tag` | OS Shell + Utility image tag (immutable tags are recommended) | `12-debian-12-r30` | -| `volumePermissions.image.pullPolicy` | OS Shell + Utility image pull policy | `IfNotPresent` | -| `volumePermissions.image.pullSecrets` | OS Shell + Utility image pull secrets | `[]` | -| `volumePermissions.resourcesPreset` | Set init container resources according to one common preset (allowed values: none, nano, small, medium, large, xlarge, 2xlarge). This is ignored if volumePermissions.resources is set (volumePermissions.resources is recommended for production). | `nano` | -| `volumePermissions.resources` | Set init container requests and limits for different resources like CPU or memory (essential for production workloads) | `{}` | -| `volumePermissions.containerSecurityContext.enabled` | Enabled init container' Security Context | `true` | -| `volumePermissions.containerSecurityContext.seLinuxOptions` | Set SELinux options in init container | `{}` | -| `volumePermissions.containerSecurityContext.runAsUser` | Set init container's Security Context runAsUser | `0` | - -##### OpenSearch Parameters - -| Name | Description | Value | -| -------------------- | ----------------------------------------------------- | ------ | -| `opensearch.enabled` | Switch to enable or disable the opensearch helm chart | `true` | - -##### Vault Parameters - -| Name | Description | Value | -| --------------- | -------------------------------------------------------------------------- | ----- | -| `vault.role` | Specify the Vault role, which should be used to get the secret from Vault. | `""` | -| `vault.secrets` | A list of VaultSecrets to create | `[]` | - -##### Other Parameters - -| Name | Description | Value | -| ----------------------------- | -------------------------------------------------- | ------- | -| `apps.enabled` | Whether apps should be deployed using helm. | `false` | -| `apps.shuffleTools.enabled` | Whether the shuffle-tools app is enabled | `true` | -| `apps.shuffleTools.version` | The version of the shuffle-tools app to deploy. | `1.2.0` | -| `apps.shuffleSubflow.enabled` | Whether the shuffle-subflow app is enabled | `true` | -| `apps.shuffleSubflow.version` | The version of the shuffle-subflow app to deploy. | `1.1.0` | -| `apps.http.enabled` | Whether the http app is enabled | `true` | -| `apps.http.version` | The version of the http app to deploy. | `1.4.0` | -| `apps.MY_APP.app` | The name of the app (required, e.g. shuffle-tools) | | -| `apps.MY_APP.version` | The version of the app (required, e.g. 1.2.0) | | - -#### Traffic Exposure Parameters - -| Name | Description | Value | -| -------------------------- | ----------------------------------------------------------------------------------------------------- | --------------- | -| `ingress.enabled` | Enable ingress record generation for frontend and backend | `false` | -| `ingress.pathType` | Ingress path type for the frontend path | `Prefix` | -| `ingress.backendPathType` | Ingress path type for the backend path | `Prefix` | -| `ingress.apiVersion` | Force Ingress API version (automatically detected if not set) | `""` | -| `ingress.hostname` | Default host for the ingress record | `shuffle.local` | -| `ingress.ingressClassName` | IngressClass that will be be used to implement the Ingress (Kubernetes 1.18+) | `nginx` | -| `ingress.path` | Ingress path for Shuffle frontend | `"/"` | -| `ingress.backendPath` | Ingress path for Shuffle backend | `"/api/"` | -| `ingress.annotations` | Additional annotations for the Ingress resource. | `{}` | -| `ingress.tls` | Enable TLS configuration for the host defined at `ingress.hostname` parameter | `false` | -| `ingress.selfSigned` | Create a TLS secret for this ingress record using self-signed certificates generated by Helm | `false` | -| `ingress.extraHosts` | An array with additional hostname(s) to be covered with the ingress record | `[]` | -| `ingress.extraPaths` | An array with additional arbitrary paths that may need to be added to the ingress under the main host | `[]` | -| `ingress.extraTls` | TLS configuration for additional hostname(s) to be covered with this ingress record | `[]` | -| `ingress.secrets` | Custom TLS certificates as secrets | `[]` | -| `ingress.extraRules` | Additional rules to be covered with this ingress record | `[]` | - -#### Istio Parameters - -| Name | Description | Value | -| --------------------------------------- | ------------------------------------------------------------------------------- | ------------------------ | -| `istio.enabled` | Enable creation of an Istio Gateway and VirtualService for frontend and backend | `false` | -| `istio.apiVersion` | The istio apiVersion to use for Gateway and VirtualService resources | `networking.istio.io/v1` | -| `istio.hosts` | One or more hosts exposed by Istio | `[]` | -| `istio.gateway.annotations` | Additional annotations for the Gateway resource | `{}` | -| `istio.gateway.selector` | The selector matches the ingress gateway pod labels | `{ istio: ingress }` | -| `istio.gateway.http.enabled` | Enable HTTP server port 80 | `true` | -| `istio.gateway.http.httpsRedirect` | If set to true, a 301 redirect is send for all HTTP connections | `false` | -| `istio.gateway.https.enabled` | Enable HTTPS server on port 443 | `false` | -| `istio.gateway.https.tlsCredentialName` | The name of the secret that holds the TLS certs including the CA certificates. | `""` | -| `istio.gateway.https.tlsCipherSuites` | If specified, only support the specified cipher list. | `[]` | -| `istio.gateway.extraServers` | Additional servers for the Gateway resource | `[]` | -| `istio.virtualService.annotations` | Additional annotations for the VirtualService resource. | `{}` | -| `istio.virtualService.backendHeaders` | Header manipulation rules for backend traffic | `{}` | -| `istio.virtualService.frontendHeaders` | Header manipulation rules for frontend traffic | `{}` | - -#### Persistence Parameters - -| Name | Description | Value | -| ------------------------------------- | ------------------------------------------------- | ------------------- | -| `persistence.enabled` | Enable persistence using Persistent Volume Claims | `true` | -| `persistence.apps.existingClaim` | Name of an existing PVC to use | `""` | -| `persistence.apps.storageClass` | PVC Storage Class for shuffle-apps volume | `""` | -| `persistence.apps.subPath` | The sub path used in the volume | `""` | -| `persistence.apps.accessModes` | The access mode of the volume | `["ReadWriteOnce"]` | -| `persistence.apps.size` | The size of the volume | `5Gi` | -| `persistence.apps.annotations` | Annotations for the PVC | `{}` | -| `persistence.apps.selector` | Selector to match an existing Persistent Volume | `{}` | -| `persistence.appBuilder.storageClass` | PVC Storage Class for backend-apps-claim volume | `""` | -| `persistence.appBuilder.accessModes` | The access mode of the volume | `["ReadWriteOnce"]` | -| `persistence.appBuilder.size` | The size of the volume | `5Gi` | -| `persistence.appBuilder.annotations` | Annotations for the PVC | `{}` | -| `persistence.appBuilder.selector` | Selector to match an existing Persistent Volume | `{}` | -| `persistence.files.existingClaim` | Name of an existing PVC to use | `""` | -| `persistence.files.storageClass` | PVC Storage Class for shuffle-files volume | `""` | -| `persistence.files.subPath` | The sub path used in the volume | `""` | -| `persistence.files.accessModes` | The access mode of the volume | `["ReadWriteOnce"]` | -| `persistence.files.size` | The size of the volume | `5Gi` | -| `persistence.files.annotations` | Annotations for the PVC | `{}` | -| `persistence.files.selector` | Selector to match an existing Persistent Volume | `{}` | - -#### Init Container Parameters - -| Name | Description | Value | -| ----------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------ | -| `volumePermissions.enabled` | Enable init container that changes the owner/group of the PV mount point to `runAsUser:fsGroup` | `false` | -| `volumePermissions.image.registry` | OS Shell + Utility image registry | `docker.io` | -| `volumePermissions.image.repository` | OS Shell + Utility image repository | `bitnamilegacy/os-shell` | -| `volumePermissions.image.tag` | OS Shell + Utility image tag (immutable tags are recommended) | `12-debian-12-r30` | -| `volumePermissions.image.pullPolicy` | OS Shell + Utility image pull policy | `IfNotPresent` | -| `volumePermissions.image.pullSecrets` | OS Shell + Utility image pull secrets | `[]` | -| `volumePermissions.resourcesPreset` | Set init container resources according to one common preset (allowed values: none, nano, small, medium, large, xlarge, 2xlarge). This is ignored if volumePermissions.resources is set (volumePermissions.resources is recommended for production). | `nano` | -| `volumePermissions.resources` | Set init container requests and limits for different resources like CPU or memory (essential for production workloads) | `{}` | -| `volumePermissions.containerSecurityContext.enabled` | Enabled init container' Security Context | `true` | -| `volumePermissions.containerSecurityContext.seLinuxOptions` | Set SELinux options in init container | `{}` | -| `volumePermissions.containerSecurityContext.runAsUser` | Set init container's Security Context runAsUser | `0` | - -#### OpenSearch Parameters - -| Name | Description | Value | -| -------------------- | ----------------------------------------------------- | ------ | -| `opensearch.enabled` | Switch to enable or disable the opensearch helm chart | `true` | - -#### Vault Parameters - -| Name | Description | Value | -| --------------- | -------------------------------------------------------------------------- | ----- | -| `vault.role` | Specify the Vault role, which should be used to get the secret from Vault. | `""` | -| `vault.secrets` | A list of VaultSecrets to create | `[]` | - -#### Other Parameters +### Parameters to deploy apps using helm | Name | Description | Value | | ----------------------------- | -------------------------------------------------- | ------- | @@ -1232,3 +907,4 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `vault.secrets` | A list of VaultSecrets to create | `[]` | ### Other Parameters + From a921372e4a671f402362af9eebc6fb4164f68125 Mon Sep 17 00:00:00 2001 From: Lalit Deore Date: Mon, 23 Mar 2026 23:51:18 +0530 Subject: [PATCH 21/61] remove folder column --- frontend/src/views/Workflows2.jsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/views/Workflows2.jsx b/frontend/src/views/Workflows2.jsx index f2b5e2f2..dea3a08d 100644 --- a/frontend/src/views/Workflows2.jsx +++ b/frontend/src/views/Workflows2.jsx @@ -6310,7 +6310,7 @@ const Workflows2 = (props) => { Name - Folder + {/* Folder */} Last Updated Action @@ -6319,7 +6319,7 @@ const Workflows2 = (props) => { {remoteWorkflows.map((wf) => ( {wf.name} - {wf.folder_name || "—"} + {/* {wf.folder_name || "—"} */} {wf.updated_at && wf.updated_at > 0 ? new Date(wf.updated_at * 1000).toLocaleDateString() From afb6ea84e4321ea2665b09e0a77b2927e8d27c25 Mon Sep 17 00:00:00 2001 From: Lalit Deore Date: Tue, 24 Mar 2026 00:23:13 +0530 Subject: [PATCH 22/61] fix - add app image when missing --- frontend/src/views/AngularWorkflow.jsx | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index f1a87e7f..22fa9947 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -10265,7 +10265,15 @@ const AngularWorkflow = (defaultprops) => { } }else if(!action.isStartNode) { // This is to round the corners of the image - const originalBase64 = action.large_image !== undefined && action.large_image !== null && action.large_image !== "" ? action.large_image : theme.palette.defaultImage + // If action has no large_image (e.g. imported/synced workflow where it was stripped), + // inject it from the available apps in the sidebar + let imageSource = (action.large_image !== undefined && action.large_image !== null && action.large_image !== "") ? action.large_image : "" + if (!imageSource) { + const foundApp = apps.find((a) => a.id === action.app_id) || + apps.find((a) => a.name === action.app_name) + imageSource = (foundApp && foundApp.large_image) ? foundApp.large_image : "" + } + const originalBase64 = imageSource !== "" ? imageSource : theme.palette.defaultImage const roundedImage = await roundBase64Image(originalBase64, 16); action = {...action, large_image: roundedImage} From ec84121ea2548d51409a88594a9d6d50d48e0081 Mon Sep 17 00:00:00 2001 From: Lalit Deore Date: Tue, 24 Mar 2026 23:27:53 +0530 Subject: [PATCH 23/61] fix - ui issues --- frontend/src/views/AngularWorkflow.jsx | 61 +++++++++++++++++++++----- 1 file changed, 49 insertions(+), 12 deletions(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 22fa9947..db69e886 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -692,6 +692,7 @@ const AngularWorkflow = (defaultprops) => { const [workflowExecutions, setWorkflowExecutions] = React.useState([]); const [executionTimeline, setExecutionTimeline] = React.useState([]); const [workflowExecutionCount, setWorkflowExecutionCount] = React.useState(0); + const [executionsLoading, setExecutionsLoading] = React.useState(false); const [defaultEnvironmentIndex, setDefaultEnvironmentIndex] = React.useState(0); const [workflowRecommendations, setWorkflowRecommendations] = React.useState(undefined); const [showErrors, setShowErrors] = React.useState(true); @@ -2343,6 +2344,8 @@ const AngularWorkflow = (defaultprops) => { return } + setExecutionsLoading(true); + var url = `${globalUrl}/api/v2/workflows/${id}/executions` var method = "GET" if (filter === undefined || filter === null || filter.toUpperCase() === "ALL") { @@ -2413,7 +2416,11 @@ const AngularWorkflow = (defaultprops) => { const newkeys = sortByKey(responseJson.executions, "-started_at"); setWorkflowExecutions(newkeys); - var tmpView = new URLSearchParams(cursearch).get("execution_id"); + // If an explicit orgId is passed (suborg context), don't read execution_id from the URL + // because that execution_id belongs to a different org's context. + var tmpView = (orgId !== undefined && orgId !== null && orgId.length > 0) + ? null + : new URLSearchParams(window.location.search).get("execution_id"); if (execution_id !== undefined && execution_id !== null && execution_id.length > 0 && (tmpView === undefined || tmpView === null || tmpView.length === 0)) { tmpView = execution_id; } @@ -2422,6 +2429,7 @@ const AngularWorkflow = (defaultprops) => { if (tmpView !== undefined && tmpView !== null && tmpView.length > 0) { // Don't clean up if it's already open if (executionModalOpen === true) { + setExecutionsLoading(false); return } @@ -2468,8 +2476,8 @@ const AngularWorkflow = (defaultprops) => { } } } else { - var tmpView = new URLSearchParams(cursearch).get("execution_id"); - if (tmpView === undefined || tmpView === null || tmpView.length === 0) { + var tmpView = new URLSearchParams(window.location.search).get("execution_id"); + if (tmpView !== undefined && tmpView !== null && tmpView.length > 0) { const execution_id = tmpView; setExecutionModalView(1); setExecutionRequest({ @@ -2479,10 +2487,12 @@ const AngularWorkflow = (defaultprops) => { start() } } + setExecutionsLoading(false); }) .catch((error) => { //toast(error.toString()); console.log("Get execution error: ", error.toString()); + setExecutionsLoading(false); }); }; @@ -21326,6 +21336,21 @@ const AngularWorkflow = (defaultprops) => { } if (inputworkflow !== undefined && inputworkflow !== null && inputworkflow.id !== undefined && inputworkflow.id !== null) { + const isOrgChange = currentWorkflow.org_id !== undefined && currentWorkflow.org_id !== null && currentWorkflow.org_id.length > 0 && currentWorkflow.org_id !== inputworkflow.org_id; + + if (isOrgChange) { + // Switching to a different suborg: clear stale execution state and URL param + const currentExecutionId = new URLSearchParams(window.location.search).get("execution_id"); + if (currentExecutionId) { + const newSearch = removeParam("execution_id", window.location.search); + navigate(curpath + newSearch); + } + setExecutionModalView(0); + setExecutionData({}); + setExecutionRunning(false); + stop(); + } + getRevisionHistory(inputworkflow.id, 50, 0, inputworkflow.org_id) getWorkflowExecution(inputworkflow.id, "", executionFilter, inputworkflow.org_id) loadTriggers(inputworkflow.org_id) @@ -23642,7 +23667,7 @@ const AngularWorkflow = (defaultprops) => { )}
) : ( -
+
{ onClick={() => { setExecutionRunning(false); stop(); - // getWorkflowExecution(currentWorkflow.id, ""); - getWorkflowExecution(props.match.params.key, ""); setExecutionModalView(0); setLastExecution(executionData.execution_id); + // getWorkflowExecution(currentWorkflow.id, ""); + setTimeout(() => { + getWorkflowExecution(props.match.params.key, ""); + }, 100); }} > { marginBottom: "auto", }} onClick={() => { + const newitem = removeParam("execution_id", cursearch); + navigate(curpath + newitem) setExecutionRunning(false); stop() }} @@ -24086,7 +24115,6 @@ const AngularWorkflow = (defaultprops) => { {environments.length > 0 && defaultEnvironmentIndex < environments.length && nonskippedResults.length === 0 && environments[defaultEnvironmentIndex].Name !== "Cloud" ? No results yet. Is Orborus running for the "{environments[defaultEnvironmentIndex].Name}" environment? Find out here. If the Workflow doesn't start within 30 seconds with Orborus running, contact support: {supportEmail} - No results yet. Is Orborus running for the "{environments[defaultEnvironmentIndex].Name}" environment? Find out here. If the Workflow doesn't start within 30 seconds with Orborus running, contact support: support@shuffler.io : null}
@@ -24168,6 +24196,14 @@ const AngularWorkflow = (defaultprops) => { } + // Fallback: if large_image is still missing, look up by app_id in the apps state + if ((imgSrc === undefined || imgSrc === null || imgSrc.length === 0) && data.action.label && apps.length > 0) { + const appById = apps.find((a) => a.name === data.action.label); + if (appById !== undefined && appById !== null && appById.large_image) { + imgSrc = appById.large_image; + } + } + var actionimg = curapp === null ? null : ( { if (data.action.app_name === "Shuffle Tools" && data.action.id !== undefined && cy !== undefined) { const nodedata = cy.getElementById(data.action.id).data(); //if (nodedata !== undefined && nodedata !== null && nodedata.fillstyle === "linear-gradient") { + const img = apps.find((a) => a.name === "Shuffle Tools")?.large_image if (nodedata !== undefined && nodedata !== null) { var imgStyle = { marginRight: 20, @@ -24240,7 +24277,7 @@ const AngularWorkflow = (defaultprops) => { actionimg = ( {nodedata.label} ); @@ -24249,7 +24286,7 @@ const AngularWorkflow = (defaultprops) => { actionimg = ( {data.action.app_name} { {data?.action?.name === "run_schemaless" || data?.action?.name === "run_singul" || data?.action?.name === "singul" && data?.action?.parameters?.length > 4 ?
param?.name === "x-debug-url")?.value || ""}`}> { }} edge="end" > - + : @@ -24763,7 +24800,7 @@ const AngularWorkflow = (defaultprops) => { variant="body1" style={{}} > - {data.name}: + {data.name}: {showVariable ? data.value : null} From 63f76e947bdf5c313ed8fc3a049b0b3ff3de2abb Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Wed, 25 Mar 2026 18:26:09 +0530 Subject: [PATCH 24/61] shuffle-shared bump --- backend/go-app/go.mod | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index fa62722c..58d10deb 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -26,7 +26,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v1.1.4-experimental + github.com/shuffle/shuffle-shared v1.2.8 github.com/shuffle/singul v0.0.29 golang.org/x/crypto v0.45.0 google.golang.org/api v0.236.0 @@ -76,7 +76,7 @@ require ( github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/frikky/schemaless v0.0.32 // indirect + github.com/frikky/schemaless v0.0.33 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-jose/go-jose/v4 v4.1.3 // indirect From d3bcc41ae5ac02440bfaf7badba8cf6e21c69d2d Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Wed, 25 Mar 2026 18:53:41 +0530 Subject: [PATCH 25/61] shuffle-shared bump worker --- functions/onprem/worker/go.mod | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 98432766..6be7413d 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -11,8 +11,8 @@ require ( github.com/docker/docker v28.3.3+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v1.1.4 - github.com/shuffle/singul v0.0.28 + github.com/shuffle/shuffle-shared v1.2.8 + github.com/shuffle/singul v0.0.30 k8s.io/api v0.34.2 k8s.io/apimachinery v0.34.2 k8s.io/client-go v0.34.2 @@ -61,7 +61,7 @@ require ( github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/frikky/kin-openapi v0.42.0 // indirect - github.com/frikky/schemaless v0.0.32 // indirect + github.com/frikky/schemaless v0.0.33 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect From d447e8a8419dc3ee45331e3a2ed6fa27a3a08470 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Wed, 25 Mar 2026 20:41:28 +0530 Subject: [PATCH 26/61] shuffle-shared one more time --- backend/go-app/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 58d10deb..f3ec5a8c 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -26,7 +26,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v1.2.8 + github.com/shuffle/shuffle-shared v1.2.9 github.com/shuffle/singul v0.0.29 golang.org/x/crypto v0.45.0 google.golang.org/api v0.236.0 From c07e2dd3914341132a8dbf57d96c8106aa88704b Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Tue, 31 Mar 2026 13:40:06 +0530 Subject: [PATCH 27/61] fix: auth issue after subflow execution (don't like the solution) --- functions/onprem/worker/worker.go | 90 +++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index af2ca8cd..28ca9dc4 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -115,6 +115,84 @@ var subflowPollBackoff sync.Map var window = shuffle.NewTimeWindow(10 * time.Second) +func restoreActionConfig(ctx context.Context, executionID string, action *shuffle.Action, workflowExecution *shuffle.WorkflowExecution) { + if len(executionID) == 0 || action == nil { + return + } + + cacheKey := fmt.Sprintf("workflowexecution_%s_action_snapshot", executionID) + cacheDataRaw, err := shuffle.GetCache(ctx, cacheKey) + if err != nil { + return + } + + originalActions := []shuffle.Action{} + cacheData := []byte(cacheDataRaw.([]uint8)) + if err := json.Unmarshal(cacheData, &originalActions); err != nil { + return + } + + original := shuffle.Action{} + for _, candidate := range originalActions { + if candidate.ID == action.ID { + original = candidate + break + } + } + + if len(original.ID) == 0 { + return + } + + for index, param := range action.Parameters { + if !param.Configuration || len(param.Value) > 0 { + continue + } + + for _, originalParam := range original.Parameters { + if !strings.EqualFold(strings.TrimSpace(param.Name), strings.TrimSpace(originalParam.Name)) { + continue + } + + if len(originalParam.Value) > 0 { + action.Parameters[index].Value = originalParam.Value + } + + break + } + } + + if workflowExecution == nil { + return + } + + for actionIndex, workflowAction := range workflowExecution.Workflow.Actions { + if workflowAction.ID != action.ID { + continue + } + + for parameterIndex, workflowParam := range workflowAction.Parameters { + if !workflowParam.Configuration || len(workflowParam.Value) > 0 { + continue + } + + for _, originalParam := range original.Parameters { + if !strings.EqualFold(strings.TrimSpace(workflowParam.Name), strings.TrimSpace(originalParam.Name)) { + continue + } + + if len(originalParam.Value) > 0 { + workflowExecution.Workflow.Actions[actionIndex].Parameters[parameterIndex].Value = originalParam.Value + } + + break + } + } + + break + } +} + // Images to be autodeployed in the latest version of Shuffle. var autoDeploy = map[string]string{ "http:1.4.0": "frikky/shuffle:http_1.4.0", @@ -142,6 +220,15 @@ func setWorkflowExecution(ctx context.Context, workflowExecution shuffle.Workflo return errors.New("ExecutionId can't be empty.") } + if len(workflowExecution.Workflow.Actions) > 0 { + snapshotKey := fmt.Sprintf("workflowexecution_%s_action_snapshot", workflowExecution.ExecutionId) + if _, err := shuffle.GetCache(ctx, snapshotKey); err != nil { + if payload, marshalErr := json.Marshal(workflowExecution.Workflow.Actions); marshalErr == nil { + _ = shuffle.SetCache(ctx, snapshotKey, payload, 3600) + } + } + } + //log.Printf("[DEBUG][%s] Setting with %d results (pre)", workflowExecution.ExecutionId, len(workflowExecution.Results)) workflowExecution, _ = shuffle.Fixexecution(ctx, workflowExecution) cacheKey := fmt.Sprintf("workflowexecution_%s", workflowExecution.ExecutionId) @@ -4169,6 +4256,9 @@ func sendAppRequest(ctx context.Context, incomingUrl, appName string, port int, parsedRequest.FullExecution = *exec } + restoreActionConfig(ctx, workflowExecution.ExecutionId, action, &parsedRequest.FullExecution) + parsedRequest.Action = *action + data, err := json.Marshal(parsedRequest) if err != nil { log.Printf("[ERROR] Failed marshalling worker request: %s", err) From 392e78208449f02c53a4d503e5949ac837509937 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Mon, 6 Apr 2026 13:46:56 +0530 Subject: [PATCH 28/61] fix: making disableRule and enableRule safer --- functions/onprem/orborus/orborus.go | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 99639e72..0ef9dc88 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -4009,7 +4009,8 @@ func removeFile(fileName string) error { } func removePath(containerName, path string) error { - rmCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("rm -rf %s", path)) + // rmCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("rm -rf %s", path)) + rmCmd := exec.Command("docker", "exec", "-u", "root", containerName, "rm", "-rf", path) output, err := rmCmd.CombinedOutput() if err != nil { return fmt.Errorf("error removing path: %v, output: %s", err, output) @@ -4068,7 +4069,8 @@ func disableRule(fileName string) error { destDir := "/var/lib/tenzir/disabled_rules" destPath := fmt.Sprintf("%s/%s", destDir, fileName) - checkSrcCmd := exec.Command("docker", "exec", containerName, "sh", "-c", fmt.Sprintf("test -f %s", srcPath)) + // checkSrcCmd := exec.Command("docker", "exec", containerName, "sh", "-c", fmt.Sprintf("test -f %s", srcPath)) + checkSrcCmd := exec.Command("docker", "exec", containerName, "test", "-f", srcPath) if err := checkSrcCmd.Run(); err != nil { if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 1 { fmt.Printf("File does not exist: %s\n", srcPath) @@ -4077,12 +4079,14 @@ func disableRule(fileName string) error { return fmt.Errorf("error checking source file: %v", err) } - checkDestDirCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mkdir -p %s", destDir)) + // checkDestDirCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mkdir -p %s", destDir)) + checkDestDirCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mkdir", "-p", destDir) if err := checkDestDirCmd.Run(); err != nil { return fmt.Errorf("error ensuring destination directory exists: %v", err) } - moveCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mv %s %s", srcPath, destPath)) + // moveCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mv %s %s", srcPath, destPath)) + moveCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mv", srcPath, destPath) if err := moveCmd.Run(); err != nil { return fmt.Errorf("error moving file: %v", err) } @@ -4097,7 +4101,8 @@ func enableRule(fileName string) error { destDir := "/var/lib/tenzir/sigma_rules" destPath := fmt.Sprintf("%s/%s", destDir, fileName) - checkSrcCmd := exec.Command("docker", "exec", containerName, "sh", "-c", fmt.Sprintf("test -f %s", srcPath)) + // checkSrcCmd := exec.Command("docker", "exec", containerName, "sh", "-c", fmt.Sprintf("test -f %s", srcPath)) + checkSrcCmd := exec.Command("docker", "exec", containerName, "test", "-f", srcPath) if err := checkSrcCmd.Run(); err != nil { if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 1 { fmt.Printf("File does not exist: %s\n", srcPath) @@ -4106,11 +4111,13 @@ func enableRule(fileName string) error { return fmt.Errorf("error checking source file: %v", err) } - checkDestDirCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mkdir -p %s", destDir)) + // checkDestDirCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mkdir -p %s", destDir)) + checkDestDirCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mkdir", "-p", destDir) if err := checkDestDirCmd.Run(); err != nil { return fmt.Errorf("error ensuring destination directory exists: %v", err) } - moveCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mv %s %s", srcPath, destPath)) + // moveCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mv %s %s", srcPath, destPath)) + moveCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mv", srcPath, destPath) if err := moveCmd.Run(); err != nil { return fmt.Errorf("error moving file: %v", err) } From 16a1d0ce33e2afc659dc7600b4808a7cc9c8b55a Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Mon, 6 Apr 2026 13:58:14 +0530 Subject: [PATCH 29/61] fix: making disableRule and enableRule safer --- functions/onprem/orborus/orborus.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 971d0d2d..2cf70720 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -4081,13 +4081,13 @@ func disableRule(fileName string) error { } // checkDestDirCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mkdir -p %s", destDir)) - checkDestDirCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mkdir", "-p", destDir) + checkDestDirCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mkdir", "-p", "--", destDir) if err := checkDestDirCmd.Run(); err != nil { return fmt.Errorf("error ensuring destination directory exists: %v", err) } // moveCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mv %s %s", srcPath, destPath)) - moveCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mv", srcPath, destPath) + moveCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mv", "--", srcPath, destPath) if err := moveCmd.Run(); err != nil { return fmt.Errorf("error moving file: %v", err) } @@ -4113,12 +4113,12 @@ func enableRule(fileName string) error { } // checkDestDirCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mkdir -p %s", destDir)) - checkDestDirCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mkdir", "-p", destDir) + checkDestDirCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mkdir", "-p", "--", destDir) if err := checkDestDirCmd.Run(); err != nil { return fmt.Errorf("error ensuring destination directory exists: %v", err) } // moveCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mv %s %s", srcPath, destPath)) - moveCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mv", srcPath, destPath) + moveCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mv", "--", srcPath, destPath) if err := moveCmd.Run(); err != nil { return fmt.Errorf("error moving file: %v", err) } From d07ef7376a7ca42c9cdee91cbdfc3451f51598f3 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Tue, 7 Apr 2026 13:00:49 +0200 Subject: [PATCH 30/61] automatically set GOMEMLIMIT env variable Signed-off-by: Pascal Sthamer --- .../charts/shuffle/templates/_helpers.tpl | 43 +++++++++++++++++++ .../shuffle/templates/backend/_helpers.tpl | 6 +++ .../shuffle/templates/orborus/_helpers.tpl | 6 +++ .../templates/shuffle-app/_helpers.tpl | 7 +-- .../templates/shuffle-app/shuffle-apps.yaml | 4 ++ .../templates/shuffle-worker/_helpers.tpl | 10 ++--- .../shuffle-worker/shuffle-worker-dpl.yaml | 17 ++++++++ .../kubernetes/charts/shuffle/values.yaml | 20 +++++++++ 8 files changed, 104 insertions(+), 9 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/templates/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/_helpers.tpl index b78602bb..c3fae7b2 100644 --- a/functions/kubernetes/charts/shuffle/templates/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/_helpers.tpl @@ -4,3 +4,46 @@ Return the proper image name (for the init container volume-permissions image) {{- define "shuffle.volumePermissions.image" -}} {{- include "common.images.image" ( dict "imageRoot" .Values.volumePermissions.image "global" .Values.global "chart" .Chart ) -}} {{- end -}} + +{{/* +Return a value for the GOMEMLIMIT env variable based on a given kubernetes resource memory limit. +Usage: +{{ include "shuffle.k8sMemoryLimitToGOMEMLIMIT" (dict "k8sMemoryLimit" $.Values.resources.limits.memory "context" $) }} +*/}} +{{- define "shuffle.k8sMemoryLimitToGOMEMLIMIT" -}} +{{- $memoryLimit := .k8sMemoryLimit | default "" -}} +{{- $result := "" -}} +{{- if and $memoryLimit (gt (len $memoryLimit) 0) -}} +{{- $bytes := 0 -}} +{{- if hasSuffix "Ki" $memoryLimit -}} +{{- $bytes = mul ($memoryLimit | trimSuffix "Ki" | int) 1024 -}} +{{- else if hasSuffix "Mi" $memoryLimit -}} +{{- $bytes = mul ($memoryLimit | trimSuffix "Mi" | int) 1048576 -}} +{{- else if hasSuffix "Gi" $memoryLimit -}} +{{- $bytes = mul ($memoryLimit | trimSuffix "Gi" | int) 1073741824 -}} +{{- else if hasSuffix "Ti" $memoryLimit -}} +{{- $bytes = mul ($memoryLimit | trimSuffix "Ti" | int) 1099511627776 -}} +{{- else if hasSuffix "Pi" $memoryLimit -}} +{{- $bytes = mul ($memoryLimit | trimSuffix "Pi" | int) 1125899906842624 -}} +{{- else if hasSuffix "Ei" $memoryLimit -}} +{{- $bytes = mul ($memoryLimit | trimSuffix "Ei" | int) 1152921504606846976 -}} +{{- else if hasSuffix "K" $memoryLimit -}} +{{- $bytes = mul ($memoryLimit | trimSuffix "K" | int) 1000 -}} +{{- else if hasSuffix "M" $memoryLimit -}} +{{- $bytes = mul ($memoryLimit | trimSuffix "M" | int) 1000000 -}} +{{- else if hasSuffix "G" $memoryLimit -}} +{{- $bytes = mul ($memoryLimit | trimSuffix "G" | int) 1000000000 -}} +{{- else if hasSuffix "T" $memoryLimit -}} +{{- $bytes = mul ($memoryLimit | trimSuffix "T" | int) 1000000000000 -}} +{{- else if hasSuffix "P" $memoryLimit -}} +{{- $bytes = mul ($memoryLimit | trimSuffix "P" | int) 1000000000000000 -}} +{{- else if hasSuffix "E" $memoryLimit -}} +{{- $bytes = mul ($memoryLimit | trimSuffix "E" | int) 1000000000000000000 -}} +{{- else -}} +{{- $bytes = $memoryLimit | int -}} +{{- end -}} +{{- $gomaxmem := div (mul $bytes 9) 10 -}} +{{- $result = printf "%d" $gomaxmem -}} +{{- end -}} +{{- $result -}} +{{- end -}} \ No newline at end of file diff --git a/functions/kubernetes/charts/shuffle/templates/backend/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/backend/_helpers.tpl index 1d61b12e..13cb0ce7 100644 --- a/functions/kubernetes/charts/shuffle/templates/backend/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/backend/_helpers.tpl @@ -116,4 +116,10 @@ REGISTRY_URL: "{{ .Values.shuffle.appRegistry }}" # Used by app builder {{- if .Values.backend.debug }} DEBUG: "true" {{- end }} +{{- if .Values.backend.autoGOMEMLIMIT }} +{{- $backendResources := (.Values.backend.resources | default (include "common.resources.preset" (dict "type" .Values.backend.resourcesPreset) | fromYaml)) -}} +{{- if and $backendResources $backendResources.limits $backendResources.limits.memory }} +GOMEMLIMIT: {{ include "shuffle.k8sMemoryLimitToGOMEMLIMIT" (dict "k8sMemoryLimit" $backendResources.limits.memory) | quote }} +{{- end }} +{{- end }} {{- end -}} \ No newline at end of file diff --git a/functions/kubernetes/charts/shuffle/templates/orborus/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/orborus/_helpers.tpl index c69c84f3..0af574a1 100644 --- a/functions/kubernetes/charts/shuffle/templates/orborus/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/orborus/_helpers.tpl @@ -92,6 +92,12 @@ SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY: {{ .Values.orborus.executionConcurrency | {{- if .Values.orborus.debug }} DEBUG: "true" {{- end }} +{{- if .Values.orborus.autoGOMEMLIMIT }} +{{- $orborusResources := (.Values.orborus.resources | default (include "common.resources.preset" (dict "type" .Values.orborus.resourcesPreset) | fromYaml)) -}} +{{- if and $orborusResources $orborusResources.limits $orborusResources.limits.memory }} +GOMEMLIMIT: {{ include "shuffle.k8sMemoryLimitToGOMEMLIMIT" (dict "k8sMemoryLimit" $orborusResources.limits.memory) | quote }} +{{- end }} +{{- end }} {{- if .Values.orborus.manageWorkerDeployments }} # Shuffle worker configuration diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl index bcce6348..fd6cb6c9 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl @@ -123,12 +123,13 @@ Return the environment variables of shuffle apps in the format KEY: VALUE Usage: {{- include "shuffle.appInstance.env" (dict "app" $appValues "context" $) -}} + +WARNING: Do NOT add environment variables here that would conflict with shuffle.worker.env or shuffle.orborus.env. +Worker also sets all env variables that are defined here, because they will be passed down to apps when worker.manageAppDeployments is set. +Instead, add them directly to the deployment template (shuffle-apps.yaml). */}} {{- define "shuffle.appInstance.env" -}} SHUFFLE_APP_SDK_TIMEOUT: {{ .app.sdkTimeout | quote }} SHUFFLE_APP_EXPOSED_PORT: {{ .app.exposedContainerPort | quote }} SHUFFLE_LOGS_DISABLED: {{ .app.disableLogs | quote }} -{{- if .app.debug }} -DEBUG: "true" -{{- end }} {{- end -}} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml index 6ee29a84..521e7fe8 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml @@ -168,6 +168,10 @@ spec: value: {{ include "shuffle.backend.baseUrl" $ | quote }} - name: SHUFFLE_SWARM_CONFIG value: run # Shuffle Worker requires this to be set even when using K8s instead of swarm + {{- if $appValues.debug }} + - name: DEBUG + value: "true" + {{- end }} {{- $env := include "shuffle.appInstance.env" (dict "app" $appValues "context" $) | fromYaml }} {{- range $key, $val := $env }} - name: {{ $key | quote }} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl index 2e57fe04..c0e75c72 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl @@ -131,17 +131,15 @@ http://shuffle-workers.{{ .Release.Namespace }}.svc.cluster.local {{/* Return the environment variables of shuffle-worker in the format KEY: VALUE + +WARNING: Do NOT add environment variables here that would conflict with shuffle.orborus.env. +Orborus also sets all env variables that are defined here, because they will be passed down to worker when orborus.manageWorkerDeployments is set. +Instead, add them directly to the deployment template (shuffle-worker-dpl.yaml). */}} {{- define "shuffle.workerInstance.env" -}} -IS_KUBERNETES: "true" -KUBERNETES_NAMESPACE: "{{ .Release.Namespace }}" SHUFFLE_SWARM_CONFIG: "run" # Shuffle Worker requires this to be set even when using K8s instead of swarm -BASE_URL: {{ include "shuffle.backend.baseUrl" . | quote }} SHUFFLE_APP_EXPOSED_PORT: {{ .Values.app.exposedContainerPort | quote }} WORKER_HOSTNAME: {{ include "shuffle.worker.hostname" . }} -{{- if .Values.worker.debug }} -DEBUG: "true" -{{- end }} {{- if .Values.worker.manageAppDeployments }} # Shuffle app images diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml index 1e724ab3..eb1c1682 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml @@ -85,6 +85,23 @@ spec: env: - name: CLEANUP value: "false" # Do not remove resources when restarting worker + - name: IS_KUBERNETES + value: "true" + - name: KUBERNETES_NAMESPACE + value: {{ .Release.Namespace | quote }} + - name: BASE_URL + value: {{ include "shuffle.backend.baseUrl" . | quote }} + {{- if .Values.worker.debug }} + - name: DEBUG + value: "true" + {{- end }} + {{- if .Values.worker.autoGOMEMLIMIT }} + {{- $workerResources := (.Values.worker.resources | default (include "common.resources.preset" (dict "type" .Values.worker.resourcesPreset) | fromYaml)) -}} + {{- if and $workerResources $workerResources.limits $workerResources.limits.memory }} + - name: GOMEMLIMIT + value: {{ include "shuffle.k8sMemoryLimitToGOMEMLIMIT" (dict "k8sMemoryLimit" $workerResources.limits.memory) | quote }} + {{- end }} + {{- end }} {{- $env := include "shuffle.workerInstance.env" . | fromYaml }} {{- range $key, $val := $env }} - name: {{ $key | quote }} diff --git a/functions/kubernetes/charts/shuffle/values.yaml b/functions/kubernetes/charts/shuffle/values.yaml index 27e04aa2..87fd3bb0 100644 --- a/functions/kubernetes/charts/shuffle/values.yaml +++ b/functions/kubernetes/charts/shuffle/values.yaml @@ -520,6 +520,12 @@ backend: ## debug: false + ## @param backend.autoGOMEMLIMIT Automatically set GOMEMLIMIT environment variable to 90% of the container memory limit. + ## This helps prevent Go runtime memory limits from exceeding container limits, which can cause OOM kills. + ## Only effective when backend.resources.limits.memory is set or a resourcesPreset is used that defines memory limits. + ## + autoGOMEMLIMIT: true + ## @param backend.cleanupSchedule The interval in seconds at which the cleanup job runs ## cleanupSchedule: 300 @@ -1337,6 +1343,13 @@ orborus: ## debug: false + ## @param orborus.autoGOMEMLIMIT Automatically set GOMEMLIMIT environment variable to 90% of the container memory limit. + ## This helps prevent Go runtime memory limits from exceeding container limits, which can cause OOM kills. + ## Only effective when orborus.resources.limits.memory is set or a resourcesPreset is used that defines memory limits. + ## Only effective when orborus is deployed via helm. + ## + autoGOMEMLIMIT: true + ## @param orborus.executionConcurrency The maximum amount of concurrent workflow executions per worker ## executionConcurrency: 25 @@ -1751,6 +1764,13 @@ worker: ## debug: false + ## @param worker.autoGOMEMLIMIT Automatically set GOMEMLIMIT environment variable to 90% of the container memory limit. + ## This helps prevent Go runtime memory limits from exceeding container limits, which can cause OOM kills. + ## Only effective when worker.resources.limits.memory is set or a resourcesPreset is used that defines memory limits. + ## Only effective when worker is deployed via helm (see worker.enableHelmDeployment). + ## + autoGOMEMLIMIT: true + ## @param worker.manageAppDeployments Whether apps are deployed and managed by worker. When disabled, every used app is expected to to be already deployed (see apps.enabled). ## This effectively removes required RBAC permissions from the shuffle-worker service account to create deployments and services. ## The worker might still attempt to create kubernetes objects, resulting in an error. There is currently no way to tell the worker, that it should not manage k8s resources. From 4c50916fd630660b784cfd70de80b77ccf8a5f37 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Tue, 7 Apr 2026 13:10:52 +0200 Subject: [PATCH 31/61] update readme Signed-off-by: Pascal Sthamer --- functions/kubernetes/charts/shuffle/README.md | 130 ++++++++++++++++-- .../charts/shuffle/values.schema.json | 15 ++ 2 files changed, 136 insertions(+), 9 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/README.md b/functions/kubernetes/charts/shuffle/README.md index fa20d558..ee8f8e06 100644 --- a/functions/kubernetes/charts/shuffle/README.md +++ b/functions/kubernetes/charts/shuffle/README.md @@ -215,7 +215,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia ## Parameters -### Global parameters +#### Global parameters | Name | Description | Value | | ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | @@ -225,7 +225,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `global.compatibility.openshift.adaptSecurityContext` | Adapt the securityContext sections of the deployment to make them compatible with Openshift restricted-v2 SCC: remove runAsUser, runAsGroup and fsGroup and let the platform use their allowed default IDs. Possible values: auto (apply if the detected running cluster is Openshift), force (perform the adaptation always), disabled (do not perform adaptation) | `auto` | | `global.compatibility.omitEmptySeLinuxOptions` | If set to true, removes the seLinuxOptions from the securityContexts when it is set to an empty object | `false` | -### Common parameters +#### Common parameters | Name | Description | Value | | ------------------------ | --------------------------------------------------------------------------------------- | --------------- | @@ -241,7 +241,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `diagnosticMode.command` | Command to override all containers in the chart release | `["sleep"]` | | `diagnosticMode.args` | Args to override all containers in the chart release | `["infinity"]` | -### Shared Shuffle Parameters +#### Shared Shuffle Parameters | Name | Description | Value | | -------------------------- | -------------------------------------------------------------------------------------------------------------------------- | --------------- | @@ -251,7 +251,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `shuffle.appBaseImageName` | The base image used for shuffle apps. The final image for an app is //: | `frikky` | | `shuffle.timezone` | The timezone used by Shuffle | `Europe/Berlin` | -### backend Parameters +#### backend Parameters | Name | Description | Value | | ----------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------- | @@ -357,6 +357,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `backend.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` | | `backend.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | | `backend.debug` | Enable debug mode for backend, which enables extra logging. orborus.debug, worker.debug, and app.debug should be set to identical values, if shuffle manages these deployments. | `false` | +| `backend.autoGOMEMLIMIT` | Automatically set GOMEMLIMIT environment variable to 90% of the container memory limit. | `true` | | `backend.cleanupSchedule` | The interval in seconds at which the cleanup job runs | `300` | | `backend.openSearch.url` | The URL at which OpenSearch is available | `http://{{ .Release.Name }}-opensearch:9200` | | `backend.openSearch.username` | The username that is used for authenticating with OpenSearch | `admin` | @@ -367,7 +368,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `backend.apps.downloadBranch` | The branch from which apps should be downloaded on startup. | `master` | | `backend.apps.forceUpdate` | Force an update of apps on startup. | `false` | -### frontend Parameters +#### frontend Parameters | Name | Description | Value | | ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------- | @@ -473,7 +474,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `frontend.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` | | `frontend.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | -### orborus Parameters +#### orborus Parameters | Name | Description | Value | | ----------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------- | @@ -577,10 +578,11 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `orborus.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` | | `orborus.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | | `orborus.debug` | Enable debug mode for orborus, which enables extra logging. orborus.debug, worker.debug, and app.debug should be set to identical values, if shuffle manages these deployments. | `false` | +| `orborus.autoGOMEMLIMIT` | Automatically set GOMEMLIMIT environment variable to 90% of the container memory limit. | `true` | | `orborus.executionConcurrency` | The maximum amount of concurrent workflow executions per worker | `25` | | `orborus.manageWorkerDeployments` | Whether workers are deployed and managed by orborus. When disabled, every worker is expected to be already deployed (see worker.enableHelmDeployment). | `true` | -### worker Parameters +#### worker Parameters | Name | Description | Value | | ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------ | @@ -687,9 +689,10 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `worker.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` | | `worker.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | | `worker.debug` | Enable debug mode for worker, which enables extra logging. orborus.debug, worker.debug, and app.debug should be set to identical values, if shuffle manages these deployments. | `false` | +| `worker.autoGOMEMLIMIT` | Automatically set GOMEMLIMIT environment variable to 90% of the container memory limit. | `true` | | `worker.manageAppDeployments` | Whether apps are deployed and managed by worker. When disabled, every used app is expected to to be already deployed (see apps.enabled). | `true` | -### app Parameters +#### app Parameters | Name | Description | Value | | ------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | @@ -798,7 +801,116 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `app.sdkTimeout` | The timeout in seconds for app actions. | `300` | | `app.disableLogs` | Do not capture app logs. By default, app logs are captured, so that they are visible in the frontend. | `false` | -### Parameters to deploy apps using helm +#### Parameters to deploy apps using helm + +| Name | Description | Value | +| ----------------------------- | -------------------------------------------------- | ------- | +| `apps.enabled` | Whether apps should be deployed using helm. | `false` | +| `apps.shuffleTools.enabled` | Whether the shuffle-tools app is enabled | `true` | +| `apps.shuffleTools.version` | The version of the shuffle-tools app to deploy. | `1.2.0` | +| `apps.shuffleSubflow.enabled` | Whether the shuffle-subflow app is enabled | `true` | +| `apps.shuffleSubflow.version` | The version of the shuffle-subflow app to deploy. | `1.1.0` | +| `apps.http.enabled` | Whether the http app is enabled | `true` | +| `apps.http.version` | The version of the http app to deploy. | `1.4.0` | +| `apps.MY_APP.app` | The name of the app (required, e.g. shuffle-tools) | | +| `apps.MY_APP.version` | The version of the app (required, e.g. 1.2.0) | | + +#### Traffic Exposure Parameters + +| Name | Description | Value | +| -------------------------- | ----------------------------------------------------------------------------------------------------- | --------------- | +| `ingress.enabled` | Enable ingress record generation for frontend and backend | `false` | +| `ingress.pathType` | Ingress path type for the frontend path | `Prefix` | +| `ingress.backendPathType` | Ingress path type for the backend path | `Prefix` | +| `ingress.apiVersion` | Force Ingress API version (automatically detected if not set) | `""` | +| `ingress.hostname` | Default host for the ingress record | `shuffle.local` | +| `ingress.ingressClassName` | IngressClass that will be be used to implement the Ingress (Kubernetes 1.18+) | `nginx` | +| `ingress.path` | Ingress path for Shuffle frontend | `"/"` | +| `ingress.backendPath` | Ingress path for Shuffle backend | `"/api/"` | +| `ingress.annotations` | Additional annotations for the Ingress resource. | `{}` | +| `ingress.tls` | Enable TLS configuration for the host defined at `ingress.hostname` parameter | `false` | +| `ingress.selfSigned` | Create a TLS secret for this ingress record using self-signed certificates generated by Helm | `false` | +| `ingress.extraHosts` | An array with additional hostname(s) to be covered with the ingress record | `[]` | +| `ingress.extraPaths` | An array with additional arbitrary paths that may need to be added to the ingress under the main host | `[]` | +| `ingress.extraTls` | TLS configuration for additional hostname(s) to be covered with this ingress record | `[]` | +| `ingress.secrets` | Custom TLS certificates as secrets | `[]` | +| `ingress.extraRules` | Additional rules to be covered with this ingress record | `[]` | + +#### Istio Parameters + +| Name | Description | Value | +| --------------------------------------- | ------------------------------------------------------------------------------- | ------------------------ | +| `istio.enabled` | Enable creation of an Istio Gateway and VirtualService for frontend and backend | `false` | +| `istio.apiVersion` | The istio apiVersion to use for Gateway and VirtualService resources | `networking.istio.io/v1` | +| `istio.hosts` | One or more hosts exposed by Istio | `[]` | +| `istio.gateway.annotations` | Additional annotations for the Gateway resource | `{}` | +| `istio.gateway.selector` | The selector matches the ingress gateway pod labels | `{ istio: ingress }` | +| `istio.gateway.http.enabled` | Enable HTTP server port 80 | `true` | +| `istio.gateway.http.httpsRedirect` | If set to true, a 301 redirect is send for all HTTP connections | `false` | +| `istio.gateway.https.enabled` | Enable HTTPS server on port 443 | `false` | +| `istio.gateway.https.tlsCredentialName` | The name of the secret that holds the TLS certs including the CA certificates. | `""` | +| `istio.gateway.https.tlsCipherSuites` | If specified, only support the specified cipher list. | `[]` | +| `istio.gateway.extraServers` | Additional servers for the Gateway resource | `[]` | +| `istio.virtualService.annotations` | Additional annotations for the VirtualService resource. | `{}` | +| `istio.virtualService.backendHeaders` | Header manipulation rules for backend traffic | `{}` | +| `istio.virtualService.frontendHeaders` | Header manipulation rules for frontend traffic | `{}` | + +#### Persistence Parameters + +| Name | Description | Value | +| ------------------------------------- | ------------------------------------------------- | ------------------- | +| `persistence.enabled` | Enable persistence using Persistent Volume Claims | `true` | +| `persistence.apps.existingClaim` | Name of an existing PVC to use | `""` | +| `persistence.apps.storageClass` | PVC Storage Class for shuffle-apps volume | `""` | +| `persistence.apps.subPath` | The sub path used in the volume | `""` | +| `persistence.apps.accessModes` | The access mode of the volume | `["ReadWriteOnce"]` | +| `persistence.apps.size` | The size of the volume | `5Gi` | +| `persistence.apps.annotations` | Annotations for the PVC | `{}` | +| `persistence.apps.selector` | Selector to match an existing Persistent Volume | `{}` | +| `persistence.appBuilder.storageClass` | PVC Storage Class for backend-apps-claim volume | `""` | +| `persistence.appBuilder.accessModes` | The access mode of the volume | `["ReadWriteOnce"]` | +| `persistence.appBuilder.size` | The size of the volume | `5Gi` | +| `persistence.appBuilder.annotations` | Annotations for the PVC | `{}` | +| `persistence.appBuilder.selector` | Selector to match an existing Persistent Volume | `{}` | +| `persistence.files.existingClaim` | Name of an existing PVC to use | `""` | +| `persistence.files.storageClass` | PVC Storage Class for shuffle-files volume | `""` | +| `persistence.files.subPath` | The sub path used in the volume | `""` | +| `persistence.files.accessModes` | The access mode of the volume | `["ReadWriteOnce"]` | +| `persistence.files.size` | The size of the volume | `5Gi` | +| `persistence.files.annotations` | Annotations for the PVC | `{}` | +| `persistence.files.selector` | Selector to match an existing Persistent Volume | `{}` | + +#### Init Container Parameters + +| Name | Description | Value | +| ----------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------ | +| `volumePermissions.enabled` | Enable init container that changes the owner/group of the PV mount point to `runAsUser:fsGroup` | `false` | +| `volumePermissions.image.registry` | OS Shell + Utility image registry | `docker.io` | +| `volumePermissions.image.repository` | OS Shell + Utility image repository | `bitnamilegacy/os-shell` | +| `volumePermissions.image.tag` | OS Shell + Utility image tag (immutable tags are recommended) | `12-debian-12-r30` | +| `volumePermissions.image.pullPolicy` | OS Shell + Utility image pull policy | `IfNotPresent` | +| `volumePermissions.image.pullSecrets` | OS Shell + Utility image pull secrets | `[]` | +| `volumePermissions.resourcesPreset` | Set init container resources according to one common preset (allowed values: none, nano, small, medium, large, xlarge, 2xlarge). This is ignored if volumePermissions.resources is set (volumePermissions.resources is recommended for production). | `nano` | +| `volumePermissions.resources` | Set init container requests and limits for different resources like CPU or memory (essential for production workloads) | `{}` | +| `volumePermissions.containerSecurityContext.enabled` | Enabled init container' Security Context | `true` | +| `volumePermissions.containerSecurityContext.seLinuxOptions` | Set SELinux options in init container | `{}` | +| `volumePermissions.containerSecurityContext.runAsUser` | Set init container's Security Context runAsUser | `0` | + +#### OpenSearch Parameters + +| Name | Description | Value | +| -------------------- | ----------------------------------------------------- | ------ | +| `opensearch.enabled` | Switch to enable or disable the opensearch helm chart | `true` | + +#### Vault Parameters + +| Name | Description | Value | +| --------------- | -------------------------------------------------------------------------- | ----- | +| `vault.role` | Specify the Vault role, which should be used to get the secret from Vault. | `""` | +| `vault.secrets` | A list of VaultSecrets to create | `[]` | + +#### Other Parameters + | Name | Description | Value | | ----------------------------- | -------------------------------------------------- | ------- | diff --git a/functions/kubernetes/charts/shuffle/values.schema.json b/functions/kubernetes/charts/shuffle/values.schema.json index 5ac3da8e..10ae8ca9 100644 --- a/functions/kubernetes/charts/shuffle/values.schema.json +++ b/functions/kubernetes/charts/shuffle/values.schema.json @@ -776,6 +776,11 @@ "description": "Enable debug mode for backend, which enables extra logging. orborus.debug, worker.debug, and app.debug should be set to identical values, if shuffle manages these deployments.", "default": false }, + "autoGOMEMLIMIT": { + "type": "boolean", + "description": "Automatically set GOMEMLIMIT environment variable to 90% of the container memory limit.", + "default": true + }, "cleanupSchedule": { "type": "number", "description": "The interval in seconds at which the cleanup job runs", @@ -2069,6 +2074,11 @@ "description": "Enable debug mode for orborus, which enables extra logging. orborus.debug, worker.debug, and app.debug should be set to identical values, if shuffle manages these deployments.", "default": false }, + "autoGOMEMLIMIT": { + "type": "boolean", + "description": "Automatically set GOMEMLIMIT environment variable to 90% of the container memory limit.", + "default": true + }, "executionConcurrency": { "type": "number", "description": "The maximum amount of concurrent workflow executions per worker", @@ -2718,6 +2728,11 @@ "description": "Enable debug mode for worker, which enables extra logging. orborus.debug, worker.debug, and app.debug should be set to identical values, if shuffle manages these deployments.", "default": false }, + "autoGOMEMLIMIT": { + "type": "boolean", + "description": "Automatically set GOMEMLIMIT environment variable to 90% of the container memory limit.", + "default": true + }, "manageAppDeployments": { "type": "boolean", "description": "Whether apps are deployed and managed by worker. When disabled, every used app is expected to to be already deployed (see apps.enabled).", From a1633756621e34de911a977cecf9c794e552b097 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Thu, 9 Apr 2026 19:14:58 +0530 Subject: [PATCH 32/61] shuffle-shared bump --- backend/go-app/go.mod | 2 +- backend/go-app/main.go | 2 +- functions/onprem/orborus/go.mod | 2 +- functions/onprem/worker/go.mod | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index f3ec5a8c..6f8bdfbf 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -26,7 +26,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v1.2.9 + github.com/shuffle/shuffle-shared v1.2.24 github.com/shuffle/singul v0.0.29 golang.org/x/crypto v0.45.0 google.golang.org/api v0.236.0 diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 48fe90e1..d3675a9f 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -893,7 +893,7 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { childOrgs := []shuffle.Org{} if len(org.CreatorOrg) > 0 { - childOrgs, err = shuffle.GetAllChildOrgs(ctx, org.CreatorOrg) + childOrgs, _, err = shuffle.GetAllChildOrgs(ctx, org.CreatorOrg) if err != nil { log.Printf("[ERROR] Failed to get child orgs during getinfo: %s", err) childOrgs = []shuffle.Org{} diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index b131c52d..ba7076c0 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -10,7 +10,7 @@ require ( github.com/docker/docker v28.3.3+incompatible github.com/docker/go-connections v0.5.0 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v1.0.6 + github.com/shuffle/shuffle-shared v1.2.24 k8s.io/api v0.34.2 k8s.io/apimachinery v0.34.2 ) diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 6be7413d..183552c1 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -11,7 +11,7 @@ require ( github.com/docker/docker v28.3.3+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v1.2.8 + github.com/shuffle/shuffle-shared v1.2.24 github.com/shuffle/singul v0.0.30 k8s.io/api v0.34.2 k8s.io/apimachinery v0.34.2 From 7a4416d9cef2128282de7d0e8257a019f306d22f Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Fri, 10 Apr 2026 18:32:52 +0530 Subject: [PATCH 33/61] hard code the Kaniko image --- backend/go-app/docker.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index fc3cbdd6..4c55cce8 100755 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -389,6 +389,10 @@ func buildImage(tags []string, dockerfileLocation string) error { backendNodeName := backendPodList.Items[0].Spec.NodeName log.Printf("[INFO] Backend running on: %s", backendNodeName) + kanikoImage := os.Getenv("SHUFFLE_BUILDER_IMAGE") + if len(kanikoImage) == 0 { + kanikoImage = "gcr.io/kaniko-project/executor:latest" + } job := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ @@ -400,7 +404,7 @@ func buildImage(tags []string, dockerfileLocation string) error { Containers: []corev1.Container{ { Name: "kaniko", - Image: "gcr.io/kaniko-project/executor:latest", + Image: kanikoImage, Args: []string{ "--verbosity=debug", "--log-format=text", From 4d1f3a0ba79b04e9d8d134dc834f6c0f46176e64 Mon Sep 17 00:00:00 2001 From: Frikky Date: Mon, 13 Apr 2026 23:22:03 +0200 Subject: [PATCH 34/61] Made Orborus capable of local compliance and response --- functions/onprem/orborus/go.mod | 4 +- functions/onprem/orborus/go.sum | 8 +- functions/onprem/orborus/orborus.go | 921 +++++++++++++++++----------- 3 files changed, 554 insertions(+), 379 deletions(-) diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index ba7076c0..08977c36 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -4,7 +4,7 @@ go 1.24.0 toolchain go1.24.4 -//replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared +replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared require ( github.com/docker/docker v28.3.3+incompatible @@ -58,7 +58,7 @@ require ( github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/frikky/kin-openapi v0.42.0 // indirect - github.com/frikky/schemaless v0.0.28 // indirect + github.com/frikky/schemaless v0.0.33 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index 85701632..38fe9d50 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -128,8 +128,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0= github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4= -github.com/frikky/schemaless v0.0.32 h1:tbLxdi3GIJZaQDbfiCGEOFDogeBYaHn+IF+Vp9Xzqv4= -github.com/frikky/schemaless v0.0.32/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= +github.com/frikky/schemaless v0.0.33 h1:5Soj6VQc+ozqLh4R6MatWOl/atAeNpdon+nV5EKwjOI= +github.com/frikky/schemaless v0.0.33/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -313,8 +313,8 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shuffle/opensearch-go/v4 v4.0.0 h1:Mh85CD1MwOgXiFFYlzS1llnvdqL3CztRdR1ZT/SLIjU= github.com/shuffle/opensearch-go/v4 v4.0.0/go.mod h1:gVLZKQE5khQWMb68XBtgKrhu78oLGL2zHwAGnFMDwC0= -github.com/shuffle/shuffle-shared v1.1.4 h1:TfeP9yslJfajdS2Sk/I81xU+R6ZxkyC49h9hc7oPBv4= -github.com/shuffle/shuffle-shared v1.1.4/go.mod h1:ycTfvlAIXv78Vxu+fHtPPIEP8mK9Pwtfk/yw2uMiAB0= +github.com/shuffle/shuffle-shared v1.2.24 h1:5jH7/QE4Lf+Yt55oPuszE93Ce34hn1lBVuyEdkX9nic= +github.com/shuffle/shuffle-shared v1.2.24/go.mod h1:RSKyexqkGDB+WbboGfWMj1Sfl4e49MI2CvW3pFmirg4= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 2cf70720..c1890a14 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -134,46 +134,122 @@ var window = shuffle.NewTimeWindow(1 * time.Minute) func init() { var err error + // Look for argc/argv and map environment variables + sensorMode := false + for _, arg := range os.Args { + if !strings.HasPrefix(arg, "--") { + continue + } - // dockercli, err = dockerclient.NewEnvClient() - dockercli, dockerApiVersion, err = shuffle.GetDockerClient() - if err != nil { - log.Printf("Unable to create docker client: %s", err) + // Split away = + value := "" + if strings.Contains(arg, "=") { + newArg := strings.Split(arg, "=")[0] + value = strings.Split(arg, "=")[1] + + arg = newArg + } else { + continue + } + + if len(value) == 0 { + continue + } + + parsedArg := strings.TrimPrefix(arg, "--") + parsedArg = strings.ReplaceAll(strings.ToUpper(parsedArg), " ", "_") + if !strings.HasPrefix(parsedArg, "SHUFFLE_") { + parsedArg = "SHUFFLE_" + parsedArg + } + + if parsedArg == "SHUFFLE_SENSOR_MODE" { + parsedArg = "SHUFFLE_AGENT_SENSOR_MODE" + } else if parsedArg == "SHUFFLE_AGENT_MODE" { + parsedArg = "SHUFFLE_AGENT_SENSOR_MODE" + } + + if parsedArg == "SHUFFLE_AGENT_SENSOR_MODE" && strings.ToLower(value) == "true" { + sensorMode = true + } + + os.Setenv(parsedArg, value) } - if os.Getenv("SHUFFLE_EC2_INSTANCE") == "true" { - log.Printf("[INFO] Detected AWS EC2 instance. Setting up Docker Swarm with AWS optimizations.") - containers, err := dockercli.ContainerList(context.Background(), container.ListOptions{}) - if err == nil { - for _, container := range containers { - if strings.Contains(container.Image, "shuffle-orborus") { - if len(container.Names) != 0 { - if strings.Contains(container.Names[0], "shuffle-orborus") { - containerName = container.Names[0] - containerName = strings.TrimPrefix(containerName, "/") - os.Setenv("ORBORUS_CONTAINER_NAME", containerName) - log.Printf("[DEBUG] Found orborus container name: %s", containerName) - break + if sensorMode { + log.Printf("[INFO] Enabling sensormode (init check)") + for _, arg := range os.Args { + if !strings.HasPrefix(arg, "--") { + continue + } + + // Split away = + value := "" + if strings.Contains(arg, "=") { + newArg := strings.Split(arg, "=")[0] + value = strings.Split(arg, "=")[1] + + arg = newArg + } else { + continue + } + + if len(value) == 0 { + continue + } + + arg = strings.TrimPrefix(arg, "--") + + if arg == "queue" { + os.Setenv("ENVIRONMENT_NAME", value) + environment = value + } else if arg == "auth" { + os.Setenv("AUTH", value) + auth = value + } else if arg == "org_id" { + os.Setenv("ORG", value) + org = value + } + } + } else { + // dockercli, err = dockerclient.NewEnvClient() + dockercli, dockerApiVersion, err = shuffle.GetDockerClient() + if err != nil { + log.Printf("Unable to create docker client: %s", err) + } + + if os.Getenv("SHUFFLE_EC2_INSTANCE") == "true" { + log.Printf("[INFO] Detected AWS EC2 instance. Setting up Docker Swarm with AWS optimizations.") + containers, err := dockercli.ContainerList(context.Background(), container.ListOptions{}) + if err == nil { + for _, container := range containers { + if strings.Contains(container.Image, "shuffle-orborus") { + if len(container.Names) != 0 { + if strings.Contains(container.Names[0], "shuffle-orborus") { + containerName = container.Names[0] + containerName = strings.TrimPrefix(containerName, "/") + os.Setenv("ORBORUS_CONTAINER_NAME", containerName) + log.Printf("[DEBUG] Found orborus container name: %s", containerName) + break + } } } } + } else { + log.Printf("[ERROR] Failed to find orborus container: %s", err) + } + } + + getThisContainerId() + + if len(pipelineApikey) == 0 { + if len(os.Getenv("SHUFFLE_AUTHORIZATION")) > 0 { + log.Printf("[DEBUG] No pipeline API key found. Overriding with api key from SHUFFLE_AUTHORIZATION") + + pipelineApikey = os.Getenv("SHUFFLE_AUTHORIZATION") + os.Setenv("SHUFFLE_PIPELINE_AUTH", pipelineApikey) } - } else { - log.Printf("[ERROR] Failed to find orborus container: %s", err) } } - - getThisContainerId() - - if len(pipelineApikey) == 0 { - if len(os.Getenv("SHUFFLE_AUTHORIZATION")) > 0 { - log.Printf("[DEBUG] No pipeline API key found. Overriding with api key from SHUFFLE_AUTHORIZATION") - - pipelineApikey = os.Getenv("SHUFFLE_AUTHORIZATION") - os.Setenv("SHUFFLE_PIPELINE_AUTH", pipelineApikey) - } - } - } // form id of current running container @@ -2014,7 +2090,15 @@ func parseResourceUsage(body io.Reader) (float64, float64, error) { } -func getOrborusStats(ctx context.Context) shuffle.OrborusStats { +func getHostname() (string, error) { + hostname, err := os.Hostname() + if err != nil { + return "", fmt.Errorf("failed to get hostname: %w", err) + } + return hostname, nil +} + +func getOrborusStats(ctx context.Context, sensorMode shuffle.SensorMode) shuffle.OrborusStats { newStats := shuffle.OrborusStats{ OrgId: org, Environment: environment, @@ -2037,15 +2121,53 @@ func getOrborusStats(ctx context.Context) shuffle.OrborusStats { return newStats } + // FIXME: Returning for now due to this causing network congestion + // and database fillup. The backend api also has it disabled. + if sensorMode.Enabled { + newStats.SensorDetails.SensorMode = true + hostname, err := getHostname() + if err == nil { + newStats.SensorDetails.Hostname = hostname + } + + newStats.SensorDetails.OS = runtime.GOOS + newStats.SensorDetails.Arch = runtime.GOARCH + newStats.SensorDetails.ElevatedAccess = shuffle.IsElevated() + newStats.SensorDetails.Serial = shuffle.GetProfiler() + + if sensorMode.SoftwareListEnabled { + // Check cache first before running the command + newStats.SensorDetails.InstalledSoftware = shuffle.ListInstalledSoftware() + } + + if sensorMode.HdEncryptedCheck { + newStats.SensorDetails.HdEncrypted = fmt.Sprintf("%t", shuffle.IsDiskEncrypted()) + } + + if sensorMode.ScreenlockCheck { + newStats.SensorDetails.AutomaticScreenlockEnabled = fmt.Sprintf("%t", shuffle.IsAutomaticScreenlockEnabled()) + } + + if len(sensorMode.LogForwarding) > 0 { + newStats.SensorDetails.LogForwarding = fmt.Sprintf("not implemented: %s", sensorMode.LogForwarding) + } + + if sensorMode.ResponseActionsEnabled { + newStats.SensorDetails.ResponseActionsEnabled = fmt.Sprintf("not implemented: %s", sensorMode.ResponseActionsEnabled) + } + + + return newStats + } else { + return newStats + } + + // FIXME: Should we reach here anymore? Can it be useful? Primarily used for stats. // Disable orborus stats if os.Getenv("SHUFFLE_STATS_DISABLED") == "true" { return newStats } - // FIXME: Returning for now due to this causing network congestion - // and database fillup. The backend api also has it disabled. - return newStats - // Use the docker API to get the CPU usage of the docker engine machine pers, err := dockercli.Info(ctx) if err != nil { @@ -2240,17 +2362,30 @@ func cleanup() { log.Printf("[INFO] Cleaning up during shutdown") ctx := context.Background() cleanupExistingNodes(ctx) - zombiecheck(ctx, 600) + zombiecheck(ctx, 600, shuffle.SensorMode{ + Enabled: os.Getenv("SHUFFLE_AGENT_MODE") == "true", + }) os.Exit(0) } -func StartAgent() { - log.Printf("[INFO] Starting Orborus agent mode") +func StartAgentSensor(sensorMode shuffle.SensorMode) error { + if sensorMode.Enabled == false { + return errors.New("Sensor mode is not enabled. Set SHUFFLE_AGENT_MODE to true to enable it.") + } - auditLogEnabled := os.Getenv("SHUFFLE_AUDIT_LOG_ENABLED") == "true" + log.Printf("[INFO] Starting Orborus - host monitoring mode (sensor/agent)") - if auditLogEnabled { - log.Printf("[INFO] Audit log monitoring is enabled") + // Check if inside Docker/Kubernetes. Use Docker/Kubernetes libraries + if isKubernetes == "true" || shuffle.IsRunningInCluster() { + log.Printf("[INFO] Detected Kubernetes environment. Not valid for Sensor Mode. Exiting.") + return errors.New("Kubernetes environment detected. Sensor mode is not valid in Kubernetes. Exiting.") + } else if swarmConfig == "run" || swarmConfig == "swarm" { + log.Printf("[INFO] Detected Docker Swarm environment. Not valid for Sensor Mode. Exiting.") + return errors.New("Docker Swarm environment detected. Sensor mode is not valid in Docker Swarm. Exiting.") + } + + if len(sensorMode.LogForwarding) > 0 { + log.Printf("[INFO] Audit log monitoring is enabled (SHUFFLE_LOG_FORWARDING=)") // Initialize telemetry configuration telemetryConfig := shuffle.TelemetryConfig{ @@ -2299,232 +2434,38 @@ func StartAgent() { } } } else { - log.Printf("[INFO] Audit log monitoring is disabled") + log.Printf("[INFO] Audit log monitoring is NOT enabled (SHUFFLE_LOG_FORWARDING=") } - select {} + + return nil } // Initial loop etc func main() { - // Get arch. amd64 or arm64 - //sigCh := make(chan os.Signal, 1) - //signal.Notify(sigCh, os.Interrupt, syscall.SIGTERM) - //defer cleanup() + // Checks for whether sensor mode is enabled for detection/response + sensorMode := shuffle.SensorMode{ + Enabled: os.Getenv("SHUFFLE_AGENT_SENSOR_MODE") == "true", - agentMode := os.Getenv("SHUFFLE_AGENT_MODE") - if agentMode == "true" { - log.Printf("[INFO] Running in agent mode. Starting the agent.") - StartAgent() - return - } + LogForwarding: os.Getenv("SHUFFLE_LOG_FORWARDING"), + SoftwareListEnabled: os.Getenv("SHUFFLE_SOFTWARE_LIST_ENABLED") == "true", + HdEncryptedCheck: os.Getenv("SHUFFLE_HD_ENCRYPTED_CHECK") == "true", + ScreenlockCheck: os.Getenv("SHUFFLE_SCREENLOCK_CHECK") == "true", - if os.Getenv("SHUFFLE_PIPELINE_STANDALONE") == "true" { - log.Printf("[INFO] Allowing use of standalone pipeline (tenzir). URL: %s", pipelineUrl) - - tenzirDisabled = false - os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") - os.Setenv("SHUFFLE_PIPELINE_ENABLED", "true") - } - - // Block until a signal is received - if shuffle.IsRunningInCluster() { - log.Printf("[INFO] Running inside k8s cluster") - } - - if isKubernetes == "true" { - fixk8sRoles() - } - - startupDelay := os.Getenv("SHUFFLE_ORBORUS_STARTUP_DELAY") - if len(startupDelay) > 0 { - log.Printf("[DEBUG] Setting startup delay to %#v", startupDelay) - - tmpInt, err := strconv.Atoi(startupDelay) - if err == nil { - time.Sleep(time.Duration(tmpInt) * time.Second) - } else { - log.Printf("[WARNING] Env SHUFFLE_ORBORUS_STARTUP_DELAY must be a number, not '%s'. Using default.", startupDelay) - } - } - - // Auto enables pipelines IF they are not mentioned - if len(os.Getenv("SHUFFLE_SKIP_PIPELINES")) == 0 { - tenzirDisabled = false - os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") - os.Setenv("SHUFFLE_PIPELINE_ENABLED", "true") - } - - if os.Getenv("SHUFFLE_SKIP_PIPELINES") != "true" && os.Getenv("SHUFFLE_PIPELINE_ENABLED") != "false" { - // Run in 15 seconds in a goroutine - go func() { - time.Sleep(15 * time.Second) - log.Printf("[INFO] Auto-downloading Sigma rules during startup") - ruleType := "sigma" - err := handleFileCategoryChange(ruleType) - if err != nil { - log.Printf("[WARNING] Failed downloading %s rules: %s", ruleType, err) - } - }() - } - - log.Println("[INFO] Setting up execution environment for env '%s'", environment) - // //FIXME - if baseUrl == "" { - baseUrl = "https://shuffler.io" - } - - if len(orborusUuid) == 0 { - orborusUuid = uuid.NewV4().String() - } - - //if orgId == "" { - // log.Printf("[ERROR] Org not defined. Set variable ORG_ID based on your org") - // os.Exit(3) - //} - if environment == "" { - log.Printf("[ERROR] Environment not defined. Set variable ENVIRONMENT_NAME to configure it.") - os.Exit(3) - } - - if timezone == "" { - timezone = "Europe/Amsterdam" - } - - log.Printf("[INFO] Using environment '%s' with timezone %s", environment, timezone) - - if len(os.Getenv("SHUFFLE_ORBORUS_PULL_TIME")) > 0 { - log.Printf("[INFO] Trying to set Orborus sleep time between polls to %s", os.Getenv("SHUFFLE_ORBORUS_PULL_TIME")) - - tmpInt, err := strconv.Atoi(os.Getenv("SHUFFLE_ORBORUS_PULL_TIME")) - if err == nil { - sleepTime = tmpInt - } - } - - // Handle Cleanup - made it cleanup by default - if strings.ToLower(os.Getenv("SHUFFLE_CONTAINER_AUTO_CLEANUP")) != "false" && os.Getenv("CLEANUP") == "" { - cleanupEnv = "true" - } - - if len(cleanupEnv) > 0 { - log.Printf("[DEBUG] Verbose mode. NOT cleaning up. Cleanup env: %s", cleanupEnv) - } - - // Default to 120 instead of default 30 - if len(os.Getenv("SHUFFLE_APP_SDK_TIMEOUT")) == 0 { - os.Setenv("SHUFFLE_APP_SDK_TIMEOUT", "120") - } - - workerTimeout := 600 - if workerTimeoutEnv != "" { - tmpInt, err := strconv.Atoi(workerTimeoutEnv) - if err == nil { - workerTimeout = tmpInt - } else { - log.Printf("[WARNING] Env SHUFFLE_ORBORUS_EXECUTION_TIMEOUT must be a number, not %s", workerTimeoutEnv) - } - - log.Printf("[INFO] Cleanup process running every %d seconds", workerTimeout) - } - - if concurrencyEnv != "" { - //var concurrencyEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY") - tmpInt, err := strconv.Atoi(concurrencyEnv) - if err == nil { - maxConcurrency = tmpInt - log.Printf("[INFO] Max workflow execution concurrency set to %d", maxConcurrency) - } else { - log.Printf("[WARNING] Env SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY must be a number, not %s. Defaulted to %d", workerTimeoutEnv, maxConcurrency) - } - } - - if len(os.Getenv("DOCKER_HOST")) > 0 { - log.Printf("[DEBUG] Running docker with socket proxy %s instead of default", os.Getenv("DOCKER_HOST")) - - } else { - log.Printf(`[DEBUG] Running docker with default socket /var/run/docker.sock or `) + ResponseActionsEnabled: os.Getenv("SHUFFLE_RESPONSE_ACTIONS_ENABLED") == "true", } ctx := context.Background() - // Run by default from now - //commenting for now as its stoppoing minikube - - log.Printf("[INFO] Running towards %s (BASE_URL) with environment name %s", baseUrl, environment) - - if environment == "" { - environment = "onprem" - log.Printf("[WARNING] Defaulting to environment name %s. Set environment variable ENVIRONMENT_NAME to change. This should be the same as in the frontend action.", environment) - } - - if pipelineUrl == "" { - pipelineUrl = "http://localhost:5160" - - // Find the IP in baseUrl. Base format is http://: - if baseUrl != "" && !strings.Contains(baseUrl, "shuffle") && !strings.Contains(baseUrl, "localhost") && !strings.Contains(baseUrl, "run.app") { - urlSplit := strings.Split(baseUrl, "://") - if len(urlSplit) > 1 { - // Find the IP - ipSplit := strings.Split(urlSplit[1], ":") - if len(ipSplit) > 0 { - pipelineUrl = fmt.Sprintf("http://%s:5160", ipSplit[0]) - } - } - } - - if len(containerId) > 0 { - pipelineUrl = "http://tenzir-node:5160" - } - - log.Printf("[WARNING] SHUFFLE_PIPELINE_URL not set, falling back to default URL: %s. If BASE_URL is set, we use the external IP for that", pipelineUrl) - os.Setenv("SHUFFLE_PIPELINE_URL", pipelineUrl) - } - - // FIXME - during init, BUILD and/or LOAD worker and app_sdk - // Build/load app_sdk so it can be loaded as 127.0.0.1:5000/walkoff_app_sdk - log.Printf("[INFO] Setting up Docker environment. Downloading worker and App SDK!") - - initializeImages() + workerTimeout := 600 workerImage := fmt.Sprintf("ghcr.io/shuffle/shuffle-worker:%s", workerVersion) if len(newWorkerImage) > 0 { workerImage = newWorkerImage } - if swarmConfig == "run" || swarmConfig == "swarm" || isKubernetes == "true" { - - if isKubernetes != "true" { - checkSwarmService(ctx) - } - - log.Printf("[DEBUG] Cleaning up containers from previous run") - cleanupExistingNodes(ctx) - time.Sleep(time.Duration(5) * time.Second) - - log.Printf("[DEBUG] Deploying worker image %s to swarm", workerImage) - - runString := "Run: \"docker service ls\" for more info" - - if isKubernetes != "true" { - deployServiceWorkers(workerImage) - - err := setBackendToSwarmNetwork(ctx) - if err != nil { - log.Printf("[WARNING] Failed setting backend to swarm network: %s", err) - } - - } else { - deployK8sWorker(workerImage, "shuffle-workers", []string{}) - runString = "Run: \"kubectl get pods\" for more info" - } - - log.Printf("[DEBUG] Waiting 45 seconds to ensure workers are deployed. %s", runString) - time.Sleep(time.Duration(45) * time.Second) - - //deployServiceWorkers(workerImage) + if len(orborusUuid) == 0 { + orborusUuid = uuid.NewV4().String() } - zombiecheck(ctx, workerTimeout) - client := shuffle.GetExternalClient(baseUrl) fullUrl := fmt.Sprintf("%s/api/v1/workflows/queue", baseUrl) @@ -2533,10 +2474,220 @@ func main() { fullUrl += "?amount=50" } - if isKubernetes == "true" { - log.Printf("[INFO] Finished configuring kubernetes environment. Connecting to %s", fullUrl) + if sensorMode.Enabled { + // Start high on purpose for now + if sleepTime < 30 { + sleepTime = 30 + } + + log.Printf("[INFO] Running in sensor/agent mode. Starting the agent.") + err := StartAgentSensor(sensorMode) + if err != nil { + log.Printf("[ERROR] Failed to start sensor/agent mode: %#v", err) + return + } } else { - log.Printf("[INFO] Finished configuring docker environment. Connecting to %s", fullUrl) + if os.Getenv("SHUFFLE_PIPELINE_STANDALONE") == "true" { + log.Printf("[INFO] Allowing use of standalone pipeline (tenzir). URL: %s", pipelineUrl) + + tenzirDisabled = false + os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") + os.Setenv("SHUFFLE_PIPELINE_ENABLED", "true") + } + + // Block until a signal is received + if shuffle.IsRunningInCluster() { + log.Printf("[INFO] Running inside k8s cluster") + } + + if isKubernetes == "true" { + fixk8sRoles() + } + + startupDelay := os.Getenv("SHUFFLE_ORBORUS_STARTUP_DELAY") + if len(startupDelay) > 0 { + log.Printf("[DEBUG] Setting startup delay to %#v", startupDelay) + + tmpInt, err := strconv.Atoi(startupDelay) + if err == nil { + time.Sleep(time.Duration(tmpInt) * time.Second) + } else { + log.Printf("[WARNING] Env SHUFFLE_ORBORUS_STARTUP_DELAY must be a number, not '%s'. Using default.", startupDelay) + } + } + + // Auto enables pipelines IF they are not mentioned + if len(os.Getenv("SHUFFLE_SKIP_PIPELINES")) == 0 { + tenzirDisabled = false + os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") + os.Setenv("SHUFFLE_PIPELINE_ENABLED", "true") + } + + if os.Getenv("SHUFFLE_SKIP_PIPELINES") != "true" && os.Getenv("SHUFFLE_PIPELINE_ENABLED") != "false" { + // Run in 15 seconds in a goroutine + go func() { + time.Sleep(15 * time.Second) + log.Printf("[INFO] Auto-downloading Sigma rules during startup") + ruleType := "sigma" + err := handleFileCategoryChange(ruleType) + if err != nil { + log.Printf("[WARNING] Failed downloading %s rules: %s", ruleType, err) + } + }() + } + + log.Println("[INFO] Setting up execution environment for env '%s'", environment) + // //FIXME + if baseUrl == "" { + baseUrl = "https://shuffler.io" + } + + //if orgId == "" { + // log.Printf("[ERROR] Org not defined. Set variable ORG_ID based on your org") + // os.Exit(3) + //} + if environment == "" { + log.Printf("[ERROR] Environment not defined. Set variable ENVIRONMENT_NAME to configure it.") + os.Exit(3) + } + + if timezone == "" { + timezone = "Europe/Amsterdam" + } + + log.Printf("[INFO] Using environment '%s' with timezone %s", environment, timezone) + + if len(os.Getenv("SHUFFLE_ORBORUS_PULL_TIME")) > 0 { + log.Printf("[INFO] Trying to set Orborus sleep time between polls to %s", os.Getenv("SHUFFLE_ORBORUS_PULL_TIME")) + + tmpInt, err := strconv.Atoi(os.Getenv("SHUFFLE_ORBORUS_PULL_TIME")) + if err == nil { + sleepTime = tmpInt + } + } + + // Handle Cleanup - made it cleanup by default + if strings.ToLower(os.Getenv("SHUFFLE_CONTAINER_AUTO_CLEANUP")) != "false" && os.Getenv("CLEANUP") == "" { + cleanupEnv = "true" + } + + if len(cleanupEnv) > 0 { + log.Printf("[DEBUG] Verbose mode. NOT cleaning up. Cleanup env: %s", cleanupEnv) + } + + // Default to 120 instead of default 30 + if len(os.Getenv("SHUFFLE_APP_SDK_TIMEOUT")) == 0 { + os.Setenv("SHUFFLE_APP_SDK_TIMEOUT", "120") + } + + if workerTimeoutEnv != "" { + tmpInt, err := strconv.Atoi(workerTimeoutEnv) + if err == nil { + workerTimeout = tmpInt + } else { + log.Printf("[WARNING] Env SHUFFLE_ORBORUS_EXECUTION_TIMEOUT must be a number, not %s", workerTimeoutEnv) + } + + log.Printf("[INFO] Cleanup process running every %d seconds", workerTimeout) + } + + if concurrencyEnv != "" { + //var concurrencyEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY") + tmpInt, err := strconv.Atoi(concurrencyEnv) + if err == nil { + maxConcurrency = tmpInt + log.Printf("[INFO] Max workflow execution concurrency set to %d", maxConcurrency) + } else { + log.Printf("[WARNING] Env SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY must be a number, not %s. Defaulted to %d", workerTimeoutEnv, maxConcurrency) + } + } + + if len(os.Getenv("DOCKER_HOST")) > 0 { + log.Printf("[DEBUG] Running docker with socket proxy %s instead of default", os.Getenv("DOCKER_HOST")) + + } else { + log.Printf(`[DEBUG] Running docker with default socket /var/run/docker.sock or `) + } + + // Run by default from now + //commenting for now as its stoppoing minikube + + log.Printf("[INFO] Running towards %s (BASE_URL) with environment name %s", baseUrl, environment) + + if environment == "" { + environment = "onprem" + log.Printf("[WARNING] Defaulting to environment name %s. Set environment variable ENVIRONMENT_NAME to change. This should be the same as in the frontend action.", environment) + } + + if pipelineUrl == "" { + pipelineUrl = "http://localhost:5160" + + // Find the IP in baseUrl. Base format is http://: + if baseUrl != "" && !strings.Contains(baseUrl, "shuffle") && !strings.Contains(baseUrl, "localhost") && !strings.Contains(baseUrl, "run.app") { + urlSplit := strings.Split(baseUrl, "://") + if len(urlSplit) > 1 { + // Find the IP + ipSplit := strings.Split(urlSplit[1], ":") + if len(ipSplit) > 0 { + pipelineUrl = fmt.Sprintf("http://%s:5160", ipSplit[0]) + } + } + } + + if len(containerId) > 0 { + pipelineUrl = "http://tenzir-node:5160" + } + + log.Printf("[WARNING] SHUFFLE_PIPELINE_URL not set, falling back to default URL: %s. If BASE_URL is set, we use the external IP for that", pipelineUrl) + os.Setenv("SHUFFLE_PIPELINE_URL", pipelineUrl) + } + + // FIXME - during init, BUILD and/or LOAD worker and app_sdk + // Build/load app_sdk so it can be loaded as 127.0.0.1:5000/walkoff_app_sdk + log.Printf("[INFO] Setting up Docker environment. Downloading worker and App SDK!") + + initializeImages() + + if swarmConfig == "run" || swarmConfig == "swarm" || isKubernetes == "true" { + + if isKubernetes != "true" { + checkSwarmService(ctx) + } + + log.Printf("[DEBUG] Cleaning up containers from previous run") + cleanupExistingNodes(ctx) + time.Sleep(time.Duration(5) * time.Second) + + log.Printf("[DEBUG] Deploying worker image %s to swarm", workerImage) + + runString := "Run: \"docker service ls\" for more info" + + if isKubernetes != "true" { + deployServiceWorkers(workerImage) + + err := setBackendToSwarmNetwork(ctx) + if err != nil { + log.Printf("[WARNING] Failed setting backend to swarm network: %s", err) + } + + } else { + deployK8sWorker(workerImage, "shuffle-workers", []string{}) + runString = "Run: \"kubectl get pods\" for more info" + } + + log.Printf("[DEBUG] Waiting 45 seconds to ensure workers are deployed. %s", runString) + time.Sleep(time.Duration(45) * time.Second) + + //deployServiceWorkers(workerImage) + } + + zombiecheck(ctx, workerTimeout, sensorMode) + + if isKubernetes == "true" { + log.Printf("[INFO] Finished configuring kubernetes environment. Connecting to %s", fullUrl) + } else { + log.Printf("[INFO] Finished configuring docker environment. Connecting to %s", fullUrl) + } } forwardData := bytes.NewBuffer([]byte{}) @@ -2591,21 +2742,21 @@ func main() { swarmControlMode = true } - log.Printf("[INFO] Waiting for executions at %s with Environment %#v", fullUrl, environment) + log.Printf("[INFO] Waiting for executions at %s with Environment %#v. Sensormode: %#v", fullUrl, environment, sensorMode.Enabled) hasStarted := false for { - if req.Method == "POST" { + if req.Method == "POST" && !sensorMode.Enabled { // Should find data to send (memory etc.) - // Create timeout of max 4 seconds just in case + // Create timeout of max a few seconds just in case ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() // Marshal and set body - orborusStats := getOrborusStats(ctx) + orborusStats := getOrborusStats(ctx, sensorMode) - pipelinePayload, pipelineerr := sendPipelineHealthStatus() + pipelinePayload, pipelineerr := sendPipelineHealthStatus(sensorMode) if pipelineerr != nil { // Too verbose to be enabled. @@ -2625,6 +2776,17 @@ func main() { time.Sleep(time.Duration(sleepTime) * time.Second) continue } + } else if sensorMode.Enabled { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + orborusStats := getOrborusStats(ctx, sensorMode) + + jsonData, err := json.Marshal(orborusStats) + if err == nil { + req.Body = ioutil.NopCloser(bytes.NewBuffer(jsonData)) + } else { + log.Printf("[ERROR] Failed marshalling. Maybe max 4 second timeout? %s", err) + } } newresp, err := client.Do(req) @@ -2633,7 +2795,7 @@ func main() { zombiecounter += 1 if zombiecounter*sleepTime > workerTimeout { - go zombiecheck(ctx, workerTimeout) + go zombiecheck(ctx, workerTimeout, sensorMode) zombiecounter = 0 } time.Sleep(time.Duration(sleepTime) * time.Second) @@ -2656,7 +2818,7 @@ func main() { log.Printf("[ERROR] Failed reading body from Shuffle: %s", err) zombiecounter += 1 if zombiecounter*sleepTime > workerTimeout { - go zombiecheck(ctx, workerTimeout) + go zombiecheck(ctx, workerTimeout, sensorMode) zombiecounter = 0 } time.Sleep(time.Duration(sleepTime) * time.Second) @@ -2672,7 +2834,7 @@ func main() { log.Printf("[ERROR] Backend connection failed for url '%s', or is missing (%d): %s", fullUrl, newresp.StatusCode, string(body)) } else { if !hasStarted { - log.Printf("[DEBUG] Starting iteration on environment %#v (default = Shuffle). Got statuscode %d from backend on first request", environment, newresp.StatusCode) + log.Printf("[DEBUG] Starting iteration on environment %#v (default: Shuffle). Got statuscode %d from backend on first request", environment, newresp.StatusCode) } if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" && os.Getenv("SHUFFLE_SCALE_REPLICAS") == "" { @@ -2688,7 +2850,7 @@ func main() { sleepTime = 10 zombiecounter += 1 if zombiecounter*sleepTime > workerTimeout { - go zombiecheck(ctx, workerTimeout) + go zombiecheck(ctx, workerTimeout, sensorMode) zombiecounter = 0 } time.Sleep(time.Duration(sleepTime) * time.Second) @@ -2700,7 +2862,6 @@ func main() { // Type string `json:"type"` } - // FIXME: Add features here for orborus & worker to // do things on behalf of backend var toBeRemoved shuffle.ExecutionRequestWrapper if len(executionRequests.Data) > 0 { @@ -2733,117 +2894,123 @@ func main() { executionRequests.Data = deduplicatedJobs for _, incRequest := range executionRequests.Data { - - // Looking for specific jobs - if incRequest.Type == "PIPELINE_CREATE" || incRequest.Type == "PIPELINE_START" || incRequest.Type == "PIPELINE_STOP" || incRequest.Type == "PIPELINE_DELETE" || incRequest.Type == "PIPELINE_UPDATE" { - log.Printf("[INFO] Handling pipeline request from backend: '%s' with argument '%s'", incRequest.Type, incRequest.ExecutionArgument) - - os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") - os.Setenv("SHUFFLE_PIPELINE_ENABLED", "true") - tenzirDisabled = false - - // Running NEW or editing pipelines - err := handlePipeline(incRequest) - if err != nil { - log.Printf("[ERROR] Failed handling pipeline ('%s' '%s'): %s. Deleting job anyway.", incRequest.Type, incRequest.ExecutionSource, err) - } + if sensorMode.Enabled { + log.Printf("[DEBUG] Sensor mode enabled. Removing job from queue without processing: %#v", incRequest) toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - } else if incRequest.Type == "DOCKER_IMAGE_DOWNLOAD" { - log.Printf("[INFO] Re-downloading new image(s) due to backend request: %#v", incRequest.ExecutionArgument) - - if len(incRequest.ExecutionArgument) > 0 { - go handleBackendImageDownload(ctx, incRequest.ExecutionArgument) - } else { - log.Printf("[ERROR] No image name provided for download. Removing job from queue.") - } - - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - - } else if incRequest.Type == "CATEGORY_UPDATE" { - os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") - - tenzirDisabled = false - err = handleFileCategoryChange("sigma") - if err != nil { - log.Printf("[ERROR] Failed to download the file category: %s", err) - } - - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - - } else if incRequest.Type == "DISABLE_SIGMA_FOLDER" { - log.Printf("[INFO] Got job to disable sigma rules") - - err = removeFileCategory("sigma") - if err != nil { - log.Printf("[ERROR] Failed to disable the sigma rules: %s", err) - } - - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - - } else if incRequest.Type == "DISABLE_SIGMA_FILE" { - fileName := incRequest.ExecutionArgument - log.Printf("[INFO] Got job to disable sigma file %s", fileName) - - err = disableRule(fileName) - if err != nil { - log.Printf("[ERROR] Failed to disable the sigma file %s, reason: %s", fileName, err) - } - - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - - } else if incRequest.Type == "ENABLE_SIGMA_FILE" { - fileName := incRequest.ExecutionArgument - log.Printf("[INFO] Got job to enable sigma file %s", fileName) - - err = enableRule(fileName) - if err != nil { - log.Printf("[ERROR] Failed to disable the sigma file %s, reason: %s", fileName, err) - } - - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - } else if incRequest.Type == "START_TENZIR" { - log.Printf("[INFO] Got job to start tenzir") - - // Manual command = overrides to allow starting of Tenzir from the frontend anyway. - //os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") - tenzirDisabled = false - - // Removed either way - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - - err := deployTenzirNode() - if err != nil { - if strings.Contains(fmt.Sprintf("%s", err), "node available") { - // Disabling until UI is updated - //os.Setenv("SHUFFLE_SKIP_PIPELINES", "true") - //tenzirDisabled = true - - log.Printf("[ERROR] Failed to start tenzir, reason: %s", err) - err = shuffle.CreateOrgNotification( - ctx, - fmt.Sprintf("Failed to start Tenzir: %s", err), - fmt.Sprintf("Tenzir failed to start due to: %s", err), - fmt.Sprintf("/detections/Sigma"), - org, - true, - "LOW", - "TENZIR_START", - ) - - if err != nil { - log.Printf("[ERROR] Failed to send notification: %s", err) - return - } - } - } } else { - if debug { - log.Printf("[DEBUG] Passing execution ID request to normal queue: %#v", incRequest.ExecutionId) - } + // Looking for specific jobs + if incRequest.Type == "PIPELINE_CREATE" || incRequest.Type == "PIPELINE_START" || incRequest.Type == "PIPELINE_STOP" || incRequest.Type == "PIPELINE_DELETE" || incRequest.Type == "PIPELINE_UPDATE" { + log.Printf("[INFO] Handling pipeline request from backend: '%s' with argument '%s'", incRequest.Type, incRequest.ExecutionArgument) - newrequests = append(newrequests, incRequest) + os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") + os.Setenv("SHUFFLE_PIPELINE_ENABLED", "true") + tenzirDisabled = false + + // Running NEW or editing pipelines + err := handlePipeline(incRequest) + if err != nil { + log.Printf("[ERROR] Failed handling pipeline ('%s' '%s'): %s. Deleting job anyway.", incRequest.Type, incRequest.ExecutionSource, err) + } + + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + } else if incRequest.Type == "DOCKER_IMAGE_DOWNLOAD" { + log.Printf("[INFO] Re-downloading new image(s) due to backend request: %#v", incRequest.ExecutionArgument) + + if len(incRequest.ExecutionArgument) > 0 { + go handleBackendImageDownload(ctx, incRequest.ExecutionArgument) + } else { + log.Printf("[ERROR] No image name provided for download. Removing job from queue.") + } + + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + + } else if incRequest.Type == "CATEGORY_UPDATE" { + os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") + + tenzirDisabled = false + err = handleFileCategoryChange("sigma") + if err != nil { + log.Printf("[ERROR] Failed to download the file category: %s", err) + } + + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + + } else if incRequest.Type == "DISABLE_SIGMA_FOLDER" { + log.Printf("[INFO] Got job to disable sigma rules") + + err = removeFileCategory("sigma") + if err != nil { + log.Printf("[ERROR] Failed to disable the sigma rules: %s", err) + } + + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + + } else if incRequest.Type == "DISABLE_SIGMA_FILE" { + fileName := incRequest.ExecutionArgument + log.Printf("[INFO] Got job to disable sigma file %s", fileName) + + err = disableRule(fileName) + if err != nil { + log.Printf("[ERROR] Failed to disable the sigma file %s, reason: %s", fileName, err) + } + + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + + } else if incRequest.Type == "ENABLE_SIGMA_FILE" { + fileName := incRequest.ExecutionArgument + log.Printf("[INFO] Got job to enable sigma file %s", fileName) + + err = enableRule(fileName) + if err != nil { + log.Printf("[ERROR] Failed to disable the sigma file %s, reason: %s", fileName, err) + } + + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + } else if incRequest.Type == "START_TENZIR" { + log.Printf("[INFO] Got job to start tenzir") + + // Manual command = overrides to allow starting of Tenzir from the frontend anyway. + //os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") + tenzirDisabled = false + + // Removed either way + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + + err := deployTenzirNode() + if err != nil { + if strings.Contains(fmt.Sprintf("%s", err), "node available") { + // Disabling until UI is updated + //os.Setenv("SHUFFLE_SKIP_PIPELINES", "true") + //tenzirDisabled = true + + log.Printf("[ERROR] Failed to start tenzir, reason: %s", err) + err = shuffle.CreateOrgNotification( + ctx, + fmt.Sprintf("Failed to start Tenzir: %s", err), + fmt.Sprintf("Tenzir failed to start due to: %s", err), + fmt.Sprintf("/detections/Sigma"), + org, + true, + "LOW", + "TENZIR_START", + ) + + if err != nil { + log.Printf("[ERROR] Failed to send notification: %s", err) + return + } + } + } + + } else { + if debug { + log.Printf("[DEBUG] Passing execution ID request to normal queue: %#v", incRequest.ExecutionId) + } + + newrequests = append(newrequests, incRequest) + } } } @@ -2865,7 +3032,7 @@ func main() { if len(executionRequests.Data) == 0 { zombiecounter += 1 if zombiecounter*sleepTime > workerTimeout { - go zombiecheck(ctx, workerTimeout) + go zombiecheck(ctx, workerTimeout, sensorMode) zombiecounter = 0 } time.Sleep(time.Duration(sleepTime) * time.Second) @@ -2876,7 +3043,7 @@ func main() { executionCount = getRunningWorkers(ctx, workerTimeout) if executionCount >= maxConcurrency { if zombiecounter*sleepTime > workerTimeout { - go zombiecheck(ctx, workerTimeout) + go zombiecheck(ctx, workerTimeout, sensorMode) zombiecounter = 0 } time.Sleep(time.Duration(sleepTime) * time.Second) @@ -4019,7 +4186,11 @@ func removePath(containerName, path string) error { return nil } -func sendPipelineHealthStatus() (shuffle.LakeConfig, error) { +func sendPipelineHealthStatus(sensorMode shuffle.SensorMode) (shuffle.LakeConfig, error) { + if sensorMode.Enabled { + return shuffle.LakeConfig{}, nil + } + pipelinePayload := shuffle.LakeConfig{ Enabled: false, Pipelines: []shuffle.PipelineInfo{}, @@ -4218,7 +4389,11 @@ func getRunningWorkers(ctx context.Context, workerTimeout int) int { // FIXME - add this to remove exited workers // Should it check what happened to the execution? idk -func zombiecheck(ctx context.Context, workerTimeout int) error { +func zombiecheck(ctx context.Context, workerTimeout int, sensorMode shuffle.SensorMode) error { + if sensorMode.Enabled { + return nil + } + isK8s := isKubernetes == "true" executionIds = []string{} From f846cd30d59d27cdcbe46092fc493abd04e8dec4 Mon Sep 17 00:00:00 2001 From: Frikky Date: Tue, 14 Apr 2026 10:47:34 +0200 Subject: [PATCH 35/61] Fixed base_url for orborus sensor --- functions/onprem/orborus/orborus.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index c1890a14..de7b9103 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -208,6 +208,9 @@ func init() { } else if arg == "org_id" { os.Setenv("ORG", value) org = value + } else if arg == "base_url" { + os.Setenv("BASE_URL", value) + baseUrl = value } } } else { From 1541e8bc4cf8665cd7cf2998a7f3489b39313c9a Mon Sep 17 00:00:00 2001 From: Frikky Date: Tue, 14 Apr 2026 16:40:19 +0200 Subject: [PATCH 36/61] Made Orborus controlled agent work --- backend/go-app/walkoff.go | 1 - functions/onprem/orborus/orborus.go | 167 ++++++++++++++++++++-------- 2 files changed, 122 insertions(+), 46 deletions(-) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index be8175ee..b8abdf8d 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -344,7 +344,6 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { if len(executionRequests.Data) > 50 { executionRequests.Data = executionRequests.Data[0:49] } - } newjson, err := json.Marshal(executionRequests) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index de7b9103..af98c137 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -2098,6 +2098,13 @@ func getHostname() (string, error) { if err != nil { return "", fmt.Errorf("failed to get hostname: %w", err) } + + // Split away TLD + parts := strings.Split(hostname, ".") + if len(parts) > 0 { + hostname = parts[0] + } + return hostname, nil } @@ -2124,9 +2131,30 @@ func getOrborusStats(ctx context.Context, sensorMode shuffle.SensorMode) shuffle return newStats } - // FIXME: Returning for now due to this causing network congestion - // and database fillup. The backend api also has it disabled. + // Handles Orborus in sensor mode. Sends minimal data per request, but + // once in a while (30 minutes) sends a lot of details like software etc if sensorMode.Enabled { + cacheKey := fmt.Sprintf("orborus_sensorDetails_cache") + cached, err := shuffle.GetCache(ctx, cacheKey) + if err == nil { + cacheData := []byte(cached.([]uint8)) + err := json.Unmarshal(cacheData, &newStats.SensorDetails) + if err == nil && len(newStats.SensorDetails.Hostname) > 0 { + newStats.SensorDetails.SensorMode = true + + // Not necessary to always send as it's big + // Backend optimises this anyway + if len(newStats.SensorDetails.Serial) > 100 { + newStats.SensorDetails.Serial = "" + } + + newStats.SensorDetails.InstalledSoftware = []shuffle.Software{} + return newStats + } + // If there's an error, we ignore the cache and continue to gather details + log.Printf("[WARNING] Failed to unmarshal cached sensor details: %s. Gathering new details.", err) + } + newStats.SensorDetails.SensorMode = true hostname, err := getHostname() if err == nil { @@ -2155,10 +2183,14 @@ func getOrborusStats(ctx context.Context, sensorMode shuffle.SensorMode) shuffle newStats.SensorDetails.LogForwarding = fmt.Sprintf("not implemented: %s", sensorMode.LogForwarding) } - if sensorMode.ResponseActionsEnabled { - newStats.SensorDetails.ResponseActionsEnabled = fmt.Sprintf("not implemented: %s", sensorMode.ResponseActionsEnabled) + if len(sensorMode.ResponseActions) > 0 { + newStats.SensorDetails.ResponseActions = sensorMode.ResponseActions } + marshalledStats, err := json.Marshal(newStats.SensorDetails) + if err == nil { + shuffle.SetCache(ctx, cacheKey, marshalledStats, 30) // Cache for 10 minutes + } return newStats } else { @@ -2450,12 +2482,12 @@ func main() { sensorMode := shuffle.SensorMode{ Enabled: os.Getenv("SHUFFLE_AGENT_SENSOR_MODE") == "true", - LogForwarding: os.Getenv("SHUFFLE_LOG_FORWARDING"), SoftwareListEnabled: os.Getenv("SHUFFLE_SOFTWARE_LIST_ENABLED") == "true", HdEncryptedCheck: os.Getenv("SHUFFLE_HD_ENCRYPTED_CHECK") == "true", ScreenlockCheck: os.Getenv("SHUFFLE_SCREENLOCK_CHECK") == "true", - ResponseActionsEnabled: os.Getenv("SHUFFLE_RESPONSE_ACTIONS_ENABLED") == "true", + LogForwarding: os.Getenv("SHUFFLE_LOG_FORWARDING"), + ResponseActions: os.Getenv("SHUFFLE_RESPONSE_ACTIONS"), } ctx := context.Background() @@ -2477,10 +2509,51 @@ func main() { fullUrl += "?amount=50" } + if len(os.Getenv("SHUFFLE_ORBORUS_PULL_TIME")) > 0 { + log.Printf("[INFO] Trying to set Orborus sleep time between polls to %s", os.Getenv("SHUFFLE_ORBORUS_PULL_TIME")) + + tmpInt, err := strconv.Atoi(os.Getenv("SHUFFLE_ORBORUS_PULL_TIME")) + if err == nil { + sleepTime = tmpInt + } + } + + log.Println("[INFO] Setting up execution environment for env '%s'", environment) + if baseUrl == "" { + baseUrl = "https://uk.shuffler.io" + } + + //if orgId == "" { + // log.Printf("[ERROR] Org not defined. Set variable ORG_ID based on your org") + // os.Exit(3) + //} + if environment == "" { + log.Printf("[ERROR] Environment not defined. Set variable ENVIRONMENT_NAME to configure it.") + os.Exit(3) + } + + if timezone == "" { + timezone = "Europe/Amsterdam" + } + + log.Printf("[INFO] Using environment '%s' with timezone %s", environment, timezone) + if sensorMode.Enabled { - // Start high on purpose for now - if sleepTime < 30 { - sleepTime = 30 + + // Start high on purpose for now (no overloads) + if sleepTime < 15 { + + // For prod + if strings.Contains(baseUrl, "shuffler.io") || strings.Contains(baseUrl, ".run.app") { + log.Printf("[INFO] Running in hosted environment. Setting default sleep time to 30 seconds to avoid hitting rate limits. You can adjust this with SHUFFLE_ORBORUS_PULL_TIME.", sleepTime) + sleepTime = 15 + } + } + + if sensorMode.ResponseActions != "full" && sensorMode.ResponseActions != "controlled" { + log.Printf("[WARNING] Invalid response actions mode '%s'. Disabling. Valid options are 'full', 'controlled', or empty.", sensorMode.ResponseActions) + sensorMode.ResponseActions = "" + } log.Printf("[INFO] Running in sensor/agent mode. Starting the agent.") @@ -2537,37 +2610,7 @@ func main() { log.Printf("[WARNING] Failed downloading %s rules: %s", ruleType, err) } }() - } - - log.Println("[INFO] Setting up execution environment for env '%s'", environment) - // //FIXME - if baseUrl == "" { - baseUrl = "https://shuffler.io" - } - - //if orgId == "" { - // log.Printf("[ERROR] Org not defined. Set variable ORG_ID based on your org") - // os.Exit(3) - //} - if environment == "" { - log.Printf("[ERROR] Environment not defined. Set variable ENVIRONMENT_NAME to configure it.") - os.Exit(3) - } - - if timezone == "" { - timezone = "Europe/Amsterdam" - } - - log.Printf("[INFO] Using environment '%s' with timezone %s", environment, timezone) - - if len(os.Getenv("SHUFFLE_ORBORUS_PULL_TIME")) > 0 { - log.Printf("[INFO] Trying to set Orborus sleep time between polls to %s", os.Getenv("SHUFFLE_ORBORUS_PULL_TIME")) - - tmpInt, err := strconv.Atoi(os.Getenv("SHUFFLE_ORBORUS_PULL_TIME")) - if err == nil { - sleepTime = tmpInt - } - } + } // Handle Cleanup - made it cleanup by default if strings.ToLower(os.Getenv("SHUFFLE_CONTAINER_AUTO_CLEANUP")) != "false" && os.Getenv("CLEANUP") == "" { @@ -2747,6 +2790,7 @@ func main() { log.Printf("[INFO] Waiting for executions at %s with Environment %#v. Sensormode: %#v", fullUrl, environment, sensorMode.Enabled) + hostname, err := getHostname() hasStarted := false for { if req.Method == "POST" && !sensorMode.Enabled { @@ -2850,7 +2894,10 @@ func main() { err = json.Unmarshal(body, &executionRequests) if err != nil { log.Printf("[WARNING] Failed executionrequest in queue unmarshaling: %s", err) - sleepTime = 10 + if !sensorMode.Enabled { + sleepTime = 10 + } + zombiecounter += 1 if zombiecounter*sleepTime > workerTimeout { go zombiecheck(ctx, workerTimeout, sensorMode) @@ -2897,10 +2944,38 @@ func main() { executionRequests.Data = deduplicatedJobs for _, incRequest := range executionRequests.Data { - if sensorMode.Enabled { - log.Printf("[DEBUG] Sensor mode enabled. Removing job from queue without processing: %#v", incRequest) - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + // Handles sensormode. ELSE handles Docker/K8s etc. + if sensorMode.Enabled { + + if len(incRequest.ExecutionSource) > 0 || len(incRequest.ExecutionArgument) == 0 { + parsedHostname := incRequest.ExecutionSource + if strings.Contains(parsedHostname, ".") { + parsedHostnameSplit := strings.Split(parsedHostname, ".") + parsedHostname = parsedHostnameSplit[0] + } + + if parsedHostname == hostname { + if debug { + log.Printf("[DEBUG]CORRECT HOSTNAME: %#v matches sensor hostname %#v. Removing from queue without processing.", parsedHostname, hostname) + } + + if sensorMode.ResponseActions != "" { + go shuffle.HandleSensorResponseAction(sensorMode, incRequest) + } + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + } else { + // Just ignore as other machines will handle it. + //log.Printf("[WARNING] Hostname '%s' from job does not match sensor hostname '%s'. Removing from queue without processing. Job: %#v", parsedHostname, hostname, incRequest) + } + } else { + // Invalid command + if debug { + log.Printf("[DEBUG] Removing invalid sensor command from queue: %#v", incRequest) + } + + toBeRemoved.Data = append(toBeRemoved.Data, incRequest) + } } else { // Looking for specific jobs @@ -3031,7 +3106,9 @@ func main() { } // Skipping throttling with swarm - if swarmConfig != "run" && swarmConfig != "swarm" { + if sensorMode.Enabled { + // Pass + } else if swarmConfig != "run" && swarmConfig != "swarm" { if len(executionRequests.Data) == 0 { zombiecounter += 1 if zombiecounter*sleepTime > workerTimeout { From 9ad57e0631a6ea4eb41e5b22b114d215be63dfdc Mon Sep 17 00:00:00 2001 From: Frikky Date: Tue, 14 Apr 2026 19:15:18 +0200 Subject: [PATCH 37/61] Last orborus changes --- functions/onprem/orborus/orborus.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index af98c137..a22768a5 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -2105,6 +2105,7 @@ func getHostname() (string, error) { hostname = parts[0] } + hostname = strings.ToUpper(hostname) return hostname, nil } @@ -2952,12 +2953,12 @@ func main() { parsedHostname := incRequest.ExecutionSource if strings.Contains(parsedHostname, ".") { parsedHostnameSplit := strings.Split(parsedHostname, ".") - parsedHostname = parsedHostnameSplit[0] + parsedHostname = strings.ToUpper(parsedHostnameSplit[0]) } if parsedHostname == hostname { if debug { - log.Printf("[DEBUG]CORRECT HOSTNAME: %#v matches sensor hostname %#v. Removing from queue without processing.", parsedHostname, hostname) + log.Printf("[DEBUG] CORRECT HOSTNAME: %#v matches sensor hostname %#v. Removing from queue without processing.", parsedHostname, hostname) } if sensorMode.ResponseActions != "" { From d906861bc823ac57c055b7e1381931dbedc7c99e Mon Sep 17 00:00:00 2001 From: Frikky Date: Tue, 14 Apr 2026 19:15:37 +0200 Subject: [PATCH 38/61] Removed Orborus and moved it to github.com/shuffle/orborus --- functions/onprem/orborus/Dockerfile | 27 - functions/onprem/orborus/build.sh | 16 - functions/onprem/orborus/docker-compose.yml | 25 - functions/onprem/orborus/go.mod | 160 - functions/onprem/orborus/go.sum | 651 --- functions/onprem/orborus/orborus.go | 5055 ------------------- functions/onprem/orborus/orborus.yaml | 82 - functions/onprem/orborus/proxy_server.py | 54 - functions/onprem/orborus/run.sh | 19 - 9 files changed, 6089 deletions(-) delete mode 100644 functions/onprem/orborus/Dockerfile delete mode 100755 functions/onprem/orborus/build.sh delete mode 100644 functions/onprem/orborus/docker-compose.yml delete mode 100644 functions/onprem/orborus/go.mod delete mode 100644 functions/onprem/orborus/go.sum delete mode 100755 functions/onprem/orborus/orborus.go delete mode 100644 functions/onprem/orborus/orborus.yaml delete mode 100644 functions/onprem/orborus/proxy_server.py delete mode 100755 functions/onprem/orborus/run.sh diff --git a/functions/onprem/orborus/Dockerfile b/functions/onprem/orborus/Dockerfile deleted file mode 100644 index d59aea8d..00000000 --- a/functions/onprem/orborus/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -FROM golang:1.24 as builder - -WORKDIR /app - -COPY orborus.go /app/orborus.go -COPY go.mod /app/go.mod - -#RUN go get -RUN go mod download -RUN go mod tidy -#RUN go build -v - -# Enabled CGO January 2025 (?) -#RUN CGO_ENABLED=1 GOOS=linux go build -a -installsuffix cgo -o orborus. -RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o /app/orborus . - -FROM alpine:3.22.1 - -RUN apk add --no-cache bash tzdata -#COPY --from=builder /app/orborus orborus -COPY --from=builder /app/ / -ENV ENVIRONMENT_NAME=Shuffle \ - BASE_URL=http://shuffle-backend:5001 \ - DOCKER_API_VERSION=1.40 \ - SHUFFLE_OPENSEARCH_URL=https://opensearch:9200 - -CMD ["./orborus"] diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh deleted file mode 100755 index bbec5067..00000000 --- a/functions/onprem/orborus/build.sh +++ /dev/null @@ -1,16 +0,0 @@ -NAME=shuffle-orborus -VERSION=1.3.0 - -echo "Running docker build with $NAME:$VERSION" -#docker rmi frikky/shuffle:$NAME --force -docker build . -t frikky/shuffle:$NAME -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly -t ghcr.io/shuffle/$NAME:$VERSION -t ghcr.io/shuffle/$NAME:nightly - -#docker push frikky/$NAME:$VERSION -# docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -docker push frikky/shuffle:$NAME -docker push ghcr.io/frikky/$NAME:$VERSION -docker push ghcr.io/frikky/$NAME:nightly - -docker push shuffle/shuffle:$NAME -docker push ghcr.io/shuffle/$NAME:$VERSION -docker push ghcr.io/shuffle/$NAME:nightly diff --git a/functions/onprem/orborus/docker-compose.yml b/functions/onprem/orborus/docker-compose.yml deleted file mode 100644 index ef6132e2..00000000 --- a/functions/onprem/orborus/docker-compose.yml +++ /dev/null @@ -1,25 +0,0 @@ -version: '3' -services: - orborus: - image: ghcr.io/frikky/shuffle-orborus:nightly - container_name: shuffle-orborus - hostname: shuffle-orborus - volumes: - - /var/run/docker.sock:/var/run/docker.sock - environment: - - SHUFFLE_APP_SDK_VERSION=nightly - - SHUFFLE_WORKER_VERSION=nightly - - ORG_ID=Shuffle - - ENVIRONMENT_NAME=Shuffle - - BASE_URL=http://192.168.86.39:5001 - - DOCKER_API_VERSION=1.40 - - SHUFFLE_SCALE_REPLICAS=5 - - SHUFFLE_SWARM_CONFIG=run - restart: unless-stopped - networks: - - shuffle-executions -networks: - shuffle-executions: - driver: overlay - external: true - diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod deleted file mode 100644 index 08977c36..00000000 --- a/functions/onprem/orborus/go.mod +++ /dev/null @@ -1,160 +0,0 @@ -module orborus - -go 1.24.0 - -toolchain go1.24.4 - -replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared - -require ( - github.com/docker/docker v28.3.3+incompatible - github.com/docker/go-connections v0.5.0 - github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v1.2.24 - k8s.io/api v0.34.2 - k8s.io/apimachinery v0.34.2 -) - -require ( - cloud.google.com/go/auth v0.16.1 // indirect - cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect -) - -require ( - cel.dev/expr v0.20.0 // indirect - cloud.google.com/go v0.121.1 // indirect - cloud.google.com/go/compute/metadata v0.7.0 // indirect - cloud.google.com/go/datastore v1.20.0 // indirect - cloud.google.com/go/iam v1.5.2 // indirect - cloud.google.com/go/monitoring v1.24.2 // indirect - cloud.google.com/go/scheduler v1.11.7 // indirect - cloud.google.com/go/storage v1.55.0 // indirect - dario.cat/mergo v1.0.0 // indirect - github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect - github.com/Masterminds/semver v1.5.0 // indirect - github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/ProtonMail/go-crypto v1.1.6 // indirect - github.com/adrg/strutil v0.3.1 // indirect - github.com/algolia/algoliasearch-client-go/v3 v3.31.4 // indirect - github.com/bradfitz/gomemcache v0.0.0-20250403215159-8d39553ac7cf // indirect - github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 // indirect - github.com/cenkalti/backoff/v5 v5.0.2 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cloudflare/circl v1.6.1 // indirect - github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 // indirect - github.com/containerd/errdefs v1.0.0 // indirect - github.com/containerd/errdefs/pkg v0.3.0 // indirect - github.com/cyphar/filepath-securejoin v0.4.1 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/distribution/reference v0.6.0 // indirect - github.com/docker/go-units v0.5.0 // indirect - github.com/emicklei/go-restful/v3 v3.12.2 // indirect - github.com/emirpasic/gods v1.18.1 // indirect - github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect - github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/frikky/kin-openapi v0.42.0 // indirect - github.com/frikky/schemaless v0.0.33 // indirect - github.com/fxamacker/cbor/v2 v2.9.0 // indirect - github.com/ghodss/yaml v1.0.0 // indirect - github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect - github.com/go-git/go-billy/v5 v5.6.2 // indirect - github.com/go-git/go-git/v5 v5.16.5 // indirect - github.com/go-jose/go-jose/v4 v4.1.3 // indirect - github.com/go-logr/logr v1.4.2 // indirect - github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-openapi/jsonpointer v0.21.0 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.23.0 // indirect - github.com/goccy/go-json v0.10.5 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect - github.com/golang/protobuf v1.5.4 // indirect - github.com/google/gnostic-models v0.7.0 // indirect - github.com/google/go-github/v28 v28.1.1 // indirect - github.com/google/go-querystring v1.1.0 // indirect - github.com/google/s2a-go v0.1.9 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect - github.com/googleapis/gax-go/v2 v2.14.2 // indirect - github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/mailru/easyjson v0.7.7 // indirect - github.com/moby/docker-image-spec v1.3.1 // indirect - github.com/moby/sys/sequential v0.6.0 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/openai/openai-go/v3 v3.8.1 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/osteele/liquid v1.7.0 // indirect - github.com/osteele/tuesday v1.0.3 // indirect - github.com/patrickmn/go-cache v2.1.0+incompatible // indirect - github.com/pjbgf/sha1cd v0.3.2 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect - github.com/sashabaranov/go-openai v1.40.5 // indirect - github.com/sendgrid/rest v2.6.9+incompatible // indirect - github.com/sendgrid/sendgrid-go v3.16.1+incompatible // indirect - github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect - github.com/shuffle/opensearch-go/v4 v4.0.0 // indirect - github.com/skeema/knownhosts v1.3.1 // indirect - github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect - github.com/spf13/pflag v1.0.6 // indirect - github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect - github.com/tidwall/gjson v1.18.0 // indirect - github.com/tidwall/match v1.1.1 // indirect - github.com/tidwall/pretty v1.2.1 // indirect - github.com/tidwall/sjson v1.2.5 // indirect - github.com/x448/float16 v0.8.4 // indirect - github.com/xanzy/ssh-agent v0.3.3 // indirect - github.com/zeebo/errs v1.4.0 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect - go.opentelemetry.io/otel v1.36.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 // indirect - go.opentelemetry.io/otel/metric v1.36.0 // indirect - go.opentelemetry.io/otel/sdk v1.36.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect - go.opentelemetry.io/otel/trace v1.36.0 // indirect - go.opentelemetry.io/proto/otlp v1.5.0 // indirect - go.yaml.in/yaml/v2 v2.4.2 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect - go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/crypto v0.45.0 // indirect - golang.org/x/net v0.47.0 // indirect - golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sync v0.18.0 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/term v0.37.0 // indirect - golang.org/x/text v0.31.0 // indirect - golang.org/x/time v0.11.0 // indirect - google.golang.org/api v0.236.0 // indirect - google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20250519155744-55703ea1f237 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a // indirect - google.golang.org/grpc v1.72.2 // indirect - google.golang.org/protobuf v1.36.6 // indirect - gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect - gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/warnings.v0 v0.1.2 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/client-go v0.34.2 // indirect - k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect - k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect - sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect - sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect - sigs.k8s.io/yaml v1.6.0 // indirect -) diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum deleted file mode 100644 index 38fe9d50..00000000 --- a/functions/onprem/orborus/go.sum +++ /dev/null @@ -1,651 +0,0 @@ -cel.dev/expr v0.20.0 h1:OunBvVCfvpWlt4dN7zg3FM6TDkzOePe1+foGJ9AXeeI= -cel.dev/expr v0.20.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.121.1 h1:S3kTQSydxmu1JfLRLpKtxRPA7rSrYPRPEUmL/PavVUw= -cloud.google.com/go v0.121.1/go.mod h1:nRFlrHq39MNVWu+zESP2PosMWA0ryJw8KUBZ2iZpxbw= -cloud.google.com/go/auth v0.16.1 h1:XrXauHMd30LhQYVRHLGvJiYeczweKQXZxsTbV9TiguU= -cloud.google.com/go/auth v0.16.1/go.mod h1:1howDHJ5IETh/LwYs3ZxvlkXF48aSqqJUM+5o02dNOI= -cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= -cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= -cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.20.0 h1:NNpXoyEqIJmZFc0ACcwBEaXnmscUpcG4NkKnbCePmiM= -cloud.google.com/go/datastore v1.20.0/go.mod h1:uFo3e+aEpRfHgtp5pp0+6M0o147KoPaYNaPAKpfh8Ew= -cloud.google.com/go/iam v1.5.2 h1:qgFRAGEmd8z6dJ/qyEchAuL9jpswyODjA2lS+w234g8= -cloud.google.com/go/iam v1.5.2/go.mod h1:SE1vg0N81zQqLzQEwxL2WI6yhetBdbNQuTvIKCSkUHE= -cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc= -cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA= -cloud.google.com/go/longrunning v0.6.7 h1:IGtfDWHhQCgCjwQjV9iiLnUta9LBCo8R9QmAFsS/PrE= -cloud.google.com/go/longrunning v0.6.7/go.mod h1:EAFV3IZAKmM56TyiE6VAP3VoTzhZzySwI/YI1s/nRsY= -cloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM= -cloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/scheduler v1.11.7 h1:zkMEJ0UbEJ3O7NwEUlKLIp6eXYv1L7wHjbxyxznajKM= -cloud.google.com/go/scheduler v1.11.7/go.mod h1:gqYs8ndLx2M5D0oMJh48aGS630YYvC432tHCnVWN13s= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.55.0 h1:NESjdAToN9u1tmhVqhXCaCwYBuvEhZLLv0gBr+2znf0= -cloud.google.com/go/storage v1.55.0/go.mod h1:ztSmTTwzsdXe5syLVS0YsbFxXuvEmEyZj7v7zChEmuY= -cloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4= -cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI= -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= -github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 h1:ErKg/3iS1AKcTkf3yixlZ54f9U1rljCkQyEXWUnIUxc= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0/go.mod h1:yAZHSGnqScoU556rBOVkwLze6WP5N+U11RHuWaGVxwY= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 h1:fYE9p3esPxA/C0rQ0AHhP0drtPXDRhaWiwg1DPqO7IU= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0/go.mod h1:BnBReJLvVYx2CS/UHOgVz2BXKXD9wsQPxZug20nZhd0= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0 h1:OqVGm6Ei3x5+yZmSJG1Mh2NwHvpVmZ08CB5qJhT9Nuk= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0/go.mod h1:SZiPHWGOOk3bl8tkevxkoiwPgsIl6CwrWcbwjfHZpdM= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 h1:6/0iUd0xrnX7qt+mLNRwg5c0PGv8wpE8K90ryANQwMI= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0/go.mod h1:otE2jQekW/PqXk1Awf5lmfokJx4uwuqcj1ab5SpGeW0= -github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= -github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= -github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= -github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw= -github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= -github.com/adrg/strutil v0.3.1 h1:OLvSS7CSJO8lBii4YmBt8jiK9QOtB9CzCzwl4Ic/Fz4= -github.com/adrg/strutil v0.3.1/go.mod h1:8h90y18QLrs11IBffcGX3NW/GFBXCMcNg4M7H6MspPA= -github.com/algolia/algoliasearch-client-go/v3 v3.31.4 h1:UJhx6AhZCYf0qZygDz2c1x1+1q2q2sfzsRaQM6yswWk= -github.com/algolia/algoliasearch-client-go/v3 v3.31.4/go.mod h1:i7tLoP7TYDmHX3Q7vkIOL4syVse/k5VJ+k0i8WqFiJk= -github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= -github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/bradfitz/gomemcache v0.0.0-20250403215159-8d39553ac7cf h1:TqhNAT4zKbTdLa62d2HDBFdvgSbIGB3eJE8HqhgiL9I= -github.com/bradfitz/gomemcache v0.0.0-20250403215159-8d39553ac7cf/go.mod h1:r5xuitiExdLAJ09PR7vBVENGvp4ZuTBeWTGtxuX3K+c= -github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 h1:/P9/RL0xgWE+ehnCUUN5h3RpG3dmoMCOONO1CCvq23Y= -github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013/go.mod h1:pccXHIvs3TV/TUqSNyEvF99sxjX2r4FFRIyw6TZY9+w= -github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8= -github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= -github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= -github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 h1:Om6kYQYDUk5wWbT0t0q6pvyM49i9XZAv9dDrkDA7gjk= -github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= -github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= -github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= -github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= -github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= -github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= -github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s= -github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= -github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= -github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= -github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= -github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= -github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= -github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= -github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= -github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA= -github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= -github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw= -github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= -github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= -github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0= -github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4= -github.com/frikky/schemaless v0.0.33 h1:5Soj6VQc+ozqLh4R6MatWOl/atAeNpdon+nV5EKwjOI= -github.com/frikky/schemaless v0.0.33/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= -github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= -github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= -github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= -github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= -github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= -github.com/go-git/go-billy/v5 v5.6.2 h1:6Q86EsPXMa7c3YZ3aLAQsMA0VlWmy43r6FHqa/UNbRM= -github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH+MJAU+jaUU= -github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= -github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= -github.com/go-git/go-git/v5 v5.16.5 h1:mdkuqblwr57kVfXri5TTH+nMFLNUxIj9Z7F5ykFbw5s= -github.com/go-git/go-git/v5 v5.16.5/go.mod h1:QOMLpNf1qxuSY4StA/ArOdfFR2TrKEjJiye2kel2m+M= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= -github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= -github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= -github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= -github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= -github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= -github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= -github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= -github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= -github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-github/v28 v28.1.1 h1:kORf5ekX5qwXO2mGzXXOjMe/g6ap8ahVe0sBEulhSxo= -github.com/google/go-github/v28 v28.1.1/go.mod h1:bsqJWQX05omyWVmc00nEUql9mhQyv38lDZ8kPZcQVoM= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= -github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= -github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= -github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= -github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= -github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.14.2 h1:eBLnkZ9635krYIPD+ag1USrOAI0Nr0QYF3+/3GqO0k0= -github.com/googleapis/gax-go/v2 v2.14.2/go.mod h1:ON64QhlJkhVtSqp4v1uaK92VyZ2gmvDQsweuyLV+8+w= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1 h1:VNqngBF40hVlDloBruUehVYC3ArSgIyScOAyMRqBxRg= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1/go.mod h1:RBRO7fro65R6tjKzYgLAFo0t1QEXY1Dp+i/bvpRiqiQ= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= -github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= -github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= -github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= -github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= -github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= -github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= -github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= -github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= -github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= -github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= -github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= -github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= -github.com/openai/openai-go/v3 v3.8.1 h1:b+YWsmwqXnbpSHWQEntZAkKciBZ5CJXwL68j+l59UDg= -github.com/openai/openai-go/v3 v3.8.1/go.mod h1:UOpNxkqC9OdNXNUfpNByKOtB4jAL0EssQXq5p8gO0Xs= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= -github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= -github.com/osteele/liquid v1.7.0 h1:VsbPSchE5D5S5scylAIvERET4dnCxsO6IDri2oSJ5Dk= -github.com/osteele/liquid v1.7.0/go.mod h1:xU0Z2dn2hOQIEFEWNmeltOmCtfhtoW/2fCyiNQeNG+U= -github.com/osteele/tuesday v1.0.3 h1:SrCmo6sWwSgnvs1bivmXLvD7Ko9+aJvvkmDjB5G4FTU= -github.com/osteele/tuesday v1.0.3/go.mod h1:pREKpE+L03UFuR+hiznj3q7j3qB1rUZ4XfKejwWFF2M= -github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= -github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= -github.com/pjbgf/sha1cd v0.3.2/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= -github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= -github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= -github.com/sashabaranov/go-openai v1.40.5 h1:SwIlNdWflzR1Rxd1gv3pUg6pwPc6cQ2uMoHs8ai+/NY= -github.com/sashabaranov/go-openai v1.40.5/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= -github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/sendgrid/rest v2.6.9+incompatible h1:1EyIcsNdn9KIisLW50MKwmSRSK+ekueiEMJ7NEoxJo0= -github.com/sendgrid/rest v2.6.9+incompatible/go.mod h1:kXX7q3jZtJXK5c5qK83bSGMdV6tsOE70KbHoqJls4lE= -github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1GjJohAA0p6hVEaDtHWWs= -github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= -github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= -github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/opensearch-go/v4 v4.0.0 h1:Mh85CD1MwOgXiFFYlzS1llnvdqL3CztRdR1ZT/SLIjU= -github.com/shuffle/opensearch-go/v4 v4.0.0/go.mod h1:gVLZKQE5khQWMb68XBtgKrhu78oLGL2zHwAGnFMDwC0= -github.com/shuffle/shuffle-shared v1.2.24 h1:5jH7/QE4Lf+Yt55oPuszE93Ce34hn1lBVuyEdkX9nic= -github.com/shuffle/shuffle-shared v1.2.24/go.mod h1:RSKyexqkGDB+WbboGfWMj1Sfl4e49MI2CvW3pFmirg4= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= -github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= -github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= -github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= -github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= -github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= -github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= -github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= -github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= -github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= -github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= -github.com/wI2L/jsondiff v0.7.0 h1:1lH1G37GhBPqCfp/lrs91rf/2j3DktX6qYAKZkLuCQQ= -github.com/wI2L/jsondiff v0.7.0/go.mod h1:KAEIojdQq66oJiHhDyQez2x+sRit0vIzC9KeK0yizxM= -github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= -github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= -github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= -github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/detectors/gcp v1.36.0 h1:F7q2tNlCaHY9nMKHR6XH9/qkp8FktLnIcy6jJNyOCQw= -go.opentelemetry.io/contrib/detectors/gcp v1.36.0/go.mod h1:IbBN8uAIIx734PTonTPxAxnjc2pQTxWNkwfstZ+6H2k= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 h1:x7wzEgXfnzJcHDwStJT+mxOz4etr2EcexjqhBvmoakw= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0/go.mod h1:rg+RlpR5dKwaS95IyyZqj5Wd4E13lk/msnTS0Xl9lJM= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= -go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= -go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 h1:1fTNlAIJZGWLP5FVu0fikVry1IsiUnXjf7QFvoNN3Xw= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0/go.mod h1:zjPK58DtkqQFn+YUMbx0M2XV3QgKU0gS9LeGohREyK4= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.36.0 h1:nRVXXvf78e00EwY6Wp0YII8ww2JVWshZ20HfTlE11AM= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.36.0/go.mod h1:r49hO7CgrxY9Voaj3Xe8pANWtr0Oq916d0XAmOoCZAQ= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 h1:rixTyDGXFxRy1xzhKrotaHy3/KXdPhlWARrCgK+eqUY= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw= -go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= -go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= -go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= -go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= -go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= -go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= -go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= -go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= -go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= -go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= -go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= -go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= -go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= -go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= -go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= -golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= -golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= -golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= -golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= -golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.236.0 h1:CAiEiDVtO4D/Qja2IA9VzlFrgPnK3XVMmRoJZlSWbc0= -google.golang.org/api v0.236.0/go.mod h1:X1WF9CU2oTc+Jml1tiIxGmWFK/UZezdqEu09gcxZAj4= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 h1:1tXaIXCracvtsRxSBsYDiSBN0cuJvM7QYW+MrpIRY78= -google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2/go.mod h1:49MsLSx0oWMOZqcpB3uL8ZOkAh1+TndpJ8ONoCBWiZk= -google.golang.org/genproto/googleapis/api v0.0.0-20250519155744-55703ea1f237 h1:Kog3KlB4xevJlAcbbbzPfRG0+X9fdoGM+UBRKVz6Wr0= -google.golang.org/genproto/googleapis/api v0.0.0-20250519155744-55703ea1f237/go.mod h1:ezi0AVyMKDWy5xAncvjLWH7UcLBB5n7y2fQ8MzjJcto= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a h1:v2PbRU4K3llS09c7zodFpNePeamkAwG3mPrAery9VeE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.72.2 h1:TdbGzwb82ty4OusHWepvFWGLgIbNo1/SUynEN0ssqv8= -google.golang.org/grpc v1.72.2/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= -gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= -gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= -gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -k8s.io/api v0.34.2 h1:fsSUNZhV+bnL6Aqrp6O7lMTy6o5x2C4XLjnh//8SLYY= -k8s.io/api v0.34.2/go.mod h1:MMBPaWlED2a8w4RSeanD76f7opUoypY8TFYkSM+3XHw= -k8s.io/apimachinery v0.34.2 h1:zQ12Uk3eMHPxrsbUJgNF8bTauTVR2WgqJsTmwTE/NW4= -k8s.io/apimachinery v0.34.2/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= -k8s.io/client-go v0.34.2 h1:Co6XiknN+uUZqiddlfAjT68184/37PS4QAzYvQvDR8M= -k8s.io/client-go v0.34.2/go.mod h1:2VYDl1XXJsdcAxw7BenFslRQX28Dxz91U9MWKjX97fE= -k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= -k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= -k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= -k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= -sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= -sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= -sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= -sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= -sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go deleted file mode 100755 index a22768a5..00000000 --- a/functions/onprem/orborus/orborus.go +++ /dev/null @@ -1,5055 +0,0 @@ -package main - -/* - Orborus exists to listen for new jobs from Shuffle. This is to run workflows, pipelines, and other tasks. -*/ -import ( - "archive/zip" - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "io/ioutil" - "log" - "math" - "net" - "net/http" - "os" - "os/exec" - "os/signal" - "path/filepath" - "runtime" - "strconv" - "strings" - "sync" - "syscall" - "time" - - "github.com/shuffle/shuffle-shared" - - "math/rand" - //"os/signal" - //"syscall" - - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/filters" - "github.com/docker/docker/api/types/image" - "github.com/docker/docker/api/types/mount" - "github.com/docker/docker/api/types/network" - "github.com/docker/docker/api/types/swarm" - "github.com/docker/go-connections/nat" - - //"github.com/docker/docker/api/types/filters" - dockerclient "github.com/docker/docker/client" - uuid "github.com/satori/go.uuid" - - //"github.com/mackerelio/go-osstat/disk" - //"github.com/mackerelio/go-osstat/memory" - //"github.com/shirou/gopsutil/cpu" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" -) - -// Starts jobs in bulk, so this could be increased or decreased based on who the user is -var sleepTime = 2 - -// Making it work on low-end machines even during busy times :) -// May cause some things to run slowly -var maxConcurrency = 25 - -// Timeout if something rashes -var workerTimeoutEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_TIMEOUT") -var concurrencyEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY") -var appSdkVersion = os.Getenv("SHUFFLE_APP_SDK_VERSION") -var workerVersion = os.Getenv("SHUFFLE_WORKER_VERSION") -var newWorkerImage = os.Getenv("SHUFFLE_WORKER_IMAGE") -var dockerSwarmBridgeMTU = os.Getenv("SHUFFLE_SWARM_BRIDGE_DEFAULT_MTU") -var dockerSwarmBridgeInterface = os.Getenv("SHUFFLE_SWARM_BRIDGE_DEFAULT_INTERFACE") -var maxCPUPercent = 90 - -// Kubernetes settings -var isKubernetes = os.Getenv("IS_KUBERNETES") -var kubernetesNamespace = os.Getenv("KUBERNETES_NAMESPACE") -var workerServiceAccountName = os.Getenv("SHUFFLE_WORKER_SERVICE_ACCOUNT_NAME") -var workerPodSecurityContext = os.Getenv("SHUFFLE_WORKER_POD_SECURITY_CONTEXT") -var workerContainerSecurityContext = os.Getenv("SHUFFLE_WORKER_CONTAINER_SECURITY_CONTEXT") -var appServiceAccountName = os.Getenv("SHUFFLE_APP_SERVICE_ACCOUNT_NAME") -var appPodSecurityContext = os.Getenv("SHUFFLE_APP_POD_SECURITY_CONTEXT") -var appContainerSecurityContext = os.Getenv("SHUFFLE_APP_CONTAINER_SECURITY_CONTEXT") -var debug = os.Getenv("DEBUG") == "true" - -// var baseimagename = "docker.pkg.github.com/shuffle/shuffle" -// var baseimagename = "ghcr.io/frikky" -// var baseimagename = "shuffle/shuffle" -var baseimageregistry = os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY") -var baseimagename = os.Getenv("SHUFFLE_BASE_IMAGE_NAME") - -//var baseimagetagsuffix = os.Getenv("SHUFFLE_BASE_IMAGE_TAG_SUFFIX") - -// Used for cloud with auth. Onprem in certain cases too. -var auth = os.Getenv("AUTH") -var org = os.Getenv("ORG") - -// var orgId = os.Getenv("ORG_ID") -var baseUrl = os.Getenv("BASE_URL") -var workerServerUrl = os.Getenv("SHUFFLE_WORKER_SERVER_URL") -var environment = os.Getenv("ENVIRONMENT_NAME") -var dockerApiVersion = os.Getenv("DOCKER_API_VERSION") -var runningMode = strings.ToLower(os.Getenv("RUNNING_MODE")) -var cleanupEnv = strings.ToLower(os.Getenv("CLEANUP")) -var timezone = os.Getenv("TZ") -var containerName = os.Getenv("ORBORUS_CONTAINER_NAME") -var swarmConfig = os.Getenv("SHUFFLE_SWARM_CONFIG") -var swarmNetworkName = os.Getenv("SHUFFLE_SWARM_NETWORK_NAME") -var orborusLabel = os.Getenv("SHUFFLE_ORBORUS_LABEL") -var memcached = os.Getenv("SHUFFLE_MEMCACHED") -var queuePerMinute = os.Getenv("SHUFFLE_EXECUTION_PER_MINIUTE") -var queuePerMinuteInt int - -// Used to download file categories. Not required since 2.1.1 -var pipelineApikey = "" -var pipelineUrl = os.Getenv("SHUFFLE_PIPELINE_URL") - -var executionIds = []string{} -var pipelines = []shuffle.PipelineInfo{} -var namespacemade = false // For K8s -var skipPipelineMount = false -var tenzirDisabled = false - -var dockercli *dockerclient.Client -var containerId string -var executionCount = 0 -var orborusUuid = os.Getenv("SHUFFLE_ORBORUS_UUID") - -var imagedownloadTimeout = time.Second * 300 -var window = shuffle.NewTimeWindow(1 * time.Minute) - -func init() { - var err error - // Look for argc/argv and map environment variables - sensorMode := false - for _, arg := range os.Args { - if !strings.HasPrefix(arg, "--") { - continue - } - - // Split away = - value := "" - if strings.Contains(arg, "=") { - newArg := strings.Split(arg, "=")[0] - value = strings.Split(arg, "=")[1] - - arg = newArg - } else { - continue - } - - if len(value) == 0 { - continue - } - - parsedArg := strings.TrimPrefix(arg, "--") - parsedArg = strings.ReplaceAll(strings.ToUpper(parsedArg), " ", "_") - if !strings.HasPrefix(parsedArg, "SHUFFLE_") { - parsedArg = "SHUFFLE_" + parsedArg - } - - if parsedArg == "SHUFFLE_SENSOR_MODE" { - parsedArg = "SHUFFLE_AGENT_SENSOR_MODE" - } else if parsedArg == "SHUFFLE_AGENT_MODE" { - parsedArg = "SHUFFLE_AGENT_SENSOR_MODE" - } - - if parsedArg == "SHUFFLE_AGENT_SENSOR_MODE" && strings.ToLower(value) == "true" { - sensorMode = true - } - - os.Setenv(parsedArg, value) - } - - if sensorMode { - log.Printf("[INFO] Enabling sensormode (init check)") - for _, arg := range os.Args { - if !strings.HasPrefix(arg, "--") { - continue - } - - // Split away = - value := "" - if strings.Contains(arg, "=") { - newArg := strings.Split(arg, "=")[0] - value = strings.Split(arg, "=")[1] - - arg = newArg - } else { - continue - } - - if len(value) == 0 { - continue - } - - arg = strings.TrimPrefix(arg, "--") - - if arg == "queue" { - os.Setenv("ENVIRONMENT_NAME", value) - environment = value - } else if arg == "auth" { - os.Setenv("AUTH", value) - auth = value - } else if arg == "org_id" { - os.Setenv("ORG", value) - org = value - } else if arg == "base_url" { - os.Setenv("BASE_URL", value) - baseUrl = value - } - } - } else { - // dockercli, err = dockerclient.NewEnvClient() - dockercli, dockerApiVersion, err = shuffle.GetDockerClient() - if err != nil { - log.Printf("Unable to create docker client: %s", err) - } - - if os.Getenv("SHUFFLE_EC2_INSTANCE") == "true" { - log.Printf("[INFO] Detected AWS EC2 instance. Setting up Docker Swarm with AWS optimizations.") - containers, err := dockercli.ContainerList(context.Background(), container.ListOptions{}) - if err == nil { - for _, container := range containers { - if strings.Contains(container.Image, "shuffle-orborus") { - if len(container.Names) != 0 { - if strings.Contains(container.Names[0], "shuffle-orborus") { - containerName = container.Names[0] - containerName = strings.TrimPrefix(containerName, "/") - os.Setenv("ORBORUS_CONTAINER_NAME", containerName) - log.Printf("[DEBUG] Found orborus container name: %s", containerName) - break - } - } - } - } - } else { - log.Printf("[ERROR] Failed to find orborus container: %s", err) - } - } - - getThisContainerId() - - if len(pipelineApikey) == 0 { - if len(os.Getenv("SHUFFLE_AUTHORIZATION")) > 0 { - log.Printf("[DEBUG] No pipeline API key found. Overriding with api key from SHUFFLE_AUTHORIZATION") - - pipelineApikey = os.Getenv("SHUFFLE_AUTHORIZATION") - os.Setenv("SHUFFLE_PIPELINE_AUTH", pipelineApikey) - } - } - } -} - -// form id of current running container -func getThisContainerId() { - fCol := "" - - // some adjusting based on current running mode - switch runningMode { - case "kubernetes": - // cgroup will be like: - // 11:net_cls,net_prio:/kubepods/besteffort/podf132b44d-cfcf-43f7-9906-79f58e268333/851466f8b5ed5aa0f265b1c95c6d2bafbc51a38dd5c5a1621b6e586572150009 - fCol = "5" - log.Printf("[INFO] Running containerized in Kubernetes!") - - case "docker": - // cgroup will be like: - // 12:perf_event:/docker/0f06810364f52a2cd6e80bfba27419cb8a29758a204cd676388f4913bb366f2b - fCol = "3" - log.Printf("[INFO] Running containerized in Docker!") - - default: - fCol = "3" // for backward-compatibility with production - log.Printf("[WARNING] RUNNING_MODE not set - defaulting to Docker (NOT Kubernetes).") - } - - if fCol != "" { - cmd := fmt.Sprintf("cat /proc/self/cgroup | grep memory | tail -1 | cut -d/ -f%s | grep -o -E '[0-9A-z]{64}'", fCol) - out, err := exec.Command("bash", "-c", cmd).Output() - if err == nil { - containerId = strings.TrimSpace(string(out)) - log.Printf("[DEBUG] Set containerId network to %s", containerId) - - // cgroup error. Use fallback strategy below. - // https://github.com/moby/moby/issues/7015 - //log.Printf("Checking if %s is in %s", ".scope", string(out)) - if strings.Contains(string(out), ".scope") { - log.Printf("[DEBUG] ContainerId contains scope. setting to empty.") - containerId = "" - //docker-76c537e9a4b7c7233011f5d70e6b7f2d600b6413ac58a96519b8dca7a3f7117a.scope - } - } else { - log.Printf("[WARNING] Failed getting container ID: %s", err) - } - } - - if containerId == "" { - if containerName != "" { - containerId = containerName - log.Printf("[INFO] Falling back to ORBORUS_CONTAINER_NAME as container ID") - } else { - containerId = "shuffle-orborus" - log.Printf(`[WARNING] ORBORUS_CONTAINER_NAME env is not set. Falling back to default name "%s" as container ID. This may cause issues on the same server`, containerId) - } - } - - log.Printf(`[INFO] Started with containerId "%s"`, containerId) -} - -func skipCheckInCleanup(name string) bool { - return strings.HasPrefix(name, "backend") || - strings.HasPrefix(name, "shuffle-backend") || - strings.HasPrefix(name, "frontend") || - strings.HasPrefix(name, "shuffle-frontend") || - strings.HasPrefix(name, "orborus") || - strings.HasPrefix(name, "shuffle-orborus") || - strings.HasPrefix(name, "opensearch") || - strings.HasPrefix(name, "shuffle-opensearch") || - strings.HasPrefix(name, "memcached") || - strings.HasPrefix(name, "shuffle-memcached") -} - -func cleanupExistingNodes(ctx context.Context) error { - if cleanupEnv != "true" { - log.Printf("[INFO] Skipping cleanup of existing workers as CLEANUP is NOT set to true. Swarm actions are being auto-discovered during executions then instead.") - return nil - } - - if isKubernetes == "true" { - // Cleanup all workers created by orborus and all apps created by workers. - - if kubernetesNamespace == "" { - kubernetesNamespace = "default" - } - - clientset, _, err := shuffle.GetKubernetesClient() - if err != nil { - log.Printf("[ERROR] Error getting kubernetes client:", err) - return err - } - - // Delete all services - services, err := clientset.CoreV1().Services(kubernetesNamespace).List(context.Background(), metav1.ListOptions{ - LabelSelector: "app.kubernetes.io/name in (shuffle-worker, shuffle-app),app.kubernetes.io/managed-by in (shuffle-orborus, shuffle-worker)", - }) - if err != nil { - log.Printf("[ERROR] Failed listing services: %s", err) - return err - } - - for _, service := range services.Items { - err := clientset.CoreV1().Services(kubernetesNamespace).Delete(context.Background(), service.Name, metav1.DeleteOptions{}) - if err != nil { - log.Printf("[ERROR] Failed deleting service %s: %s", service.Name, err) - } - } - - deployments, err := clientset.AppsV1().Deployments(kubernetesNamespace).List(context.Background(), metav1.ListOptions{ - LabelSelector: "app.kubernetes.io/name in (shuffle-worker, shuffle-app),app.kubernetes.io/managed-by in (shuffle-orborus, shuffle-worker)", - }) - if err != nil { - log.Printf("[ERROR] Failed listing deployments: %s", err) - return err - } - - for _, deployment := range deployments.Items { - err := clientset.AppsV1().Deployments(kubernetesNamespace).Delete(context.Background(), deployment.Name, metav1.DeleteOptions{}) - if err != nil { - log.Printf("[ERROR] Failed deleting deployment %s: %s", deployment.Name, err) - } - } - - log.Printf("[INFO] Cleaned up all services and deployments in namespace %s. Waiting 10 seconds for cleanup to reflect", kubernetesNamespace) - - time.Sleep(10 * time.Second) - - return nil - } - - serviceListOptions := types.ServiceListOptions{} - services, err := dockercli.ServiceList( - context.Background(), - serviceListOptions, - ) - - if err != nil { - log.Printf("[DEBUG] Failed finding containers: %s", err) - return err - } - - //log.Printf("\n\nFound %d contaienrs", len(services)) - - for _, service := range services { - - //portFound := false - //for _, endpoint := range service.Spec.EndpointSpec.Ports { - // if strings.Contains(endpoint.Name, "port") { - // //portFound = true - // } - //} - - if strings.Contains(service.Spec.Annotations.Name, "opensearch") { - continue - } - - if strings.Contains(service.Spec.TaskTemplate.ContainerSpec.Image, "shuffle") { - - if !strings.Contains(service.Spec.TaskTemplate.ContainerSpec.Image, "shuffle-frontend") && - !strings.Contains(service.Spec.TaskTemplate.ContainerSpec.Image, "shuffle-backend") && - !strings.Contains(service.Spec.TaskTemplate.ContainerSpec.Image, "shuffle-orborus") { - - err = dockercli.ServiceRemove(ctx, service.ID) - if err != nil { - log.Printf("[DEBUG] Failed to remove service %s", service.Spec.Annotations.Name) - } else { - log.Printf("[DEBUG] Removed service %#v", service.Spec.TaskTemplate.ContainerSpec.Image) - } - } - } - } - - return nil -} - -func deployServiceWorkers(image string) { - log.Printf("[DEBUG] Validating deployment of workers as services IF swarmConfig = run (value: %#v)", swarmConfig) - if swarmConfig != "run" && swarmConfig != "swarm" { - log.Printf("[DEBUG] Skipping deployment of workers as services as swarmConfig is not set to run or swarm. Value: %#v", swarmConfig) - return - } - ctx := context.Background() - - // Looks for and cleans up all existing items in swarm we can't re-use (Shuffle only) - - // frikky@debian:~/git/shuffle/functions/onprem/worker$ docker service create --replicas 5 --name shuffle-workers --env SHUFFLE_SWARM_CONFIG=run --publish published=33333,target=33333 ghcr.io/shuffle/shuffle-worker:nightly - - // Get a list of network interfaces - interfaces, err := net.Interfaces() - if err != nil { - log.Printf("[ERROR] Failed to get network interfaces: %s", err) - } - - mtu := 1500 - if len(dockerSwarmBridgeMTU) == 0 { - mtu, err = strconv.Atoi(dockerSwarmBridgeMTU) // by default - if err != nil { - if debug { - log.Printf("[DEBUG] Failed to convert the default MTU to int: %s. Using 1500 instead. Input: %s", err, dockerSwarmBridgeMTU) - } - - mtu = 1500 - } - } - - bridgeName := dockerSwarmBridgeInterface - if bridgeName == "" { - bridgeName = "eth0" - } - - // Check if there is at least one interface - if len(interfaces) < 2 { - // this assumes that the machine should have at least 2 network - // interfaces. If not, we will use the default MTU. - // interface 1 is the loopback interface - // interface 2 is eth0, The eth0 interface inside a - // Docker container corresponds to the virtual Ethernet - // interface that connects the container to the docker0 - log.Printf("[ERROR] Failed to get enough network interfaces") - } else { - // Get the preferred interface - for _, iface := range interfaces { - if strings.Contains(iface.Name, bridgeName) { - targetInterface := iface - mtu = targetInterface.MTU - log.Printf("[INFO] Using MTU %d from interface %s", mtu, targetInterface.Name) - break - } - } - } - - // Create the network options with the specified MTU - options := make(map[string]string) - options["com.docker.network.driver.mtu"] = fmt.Sprintf("%d", mtu) - - ingressOptions := network.CreateOptions{ - Driver: "overlay", - Attachable: false, - Ingress: true, - IPAM: &network.IPAM{ - Driver: "default", - Config: []network.IPAMConfig{ - network.IPAMConfig{ - Subnet: "10.225.225.0/24", - Gateway: "10.225.225.1", - }, - }, - }, - } - - _, err = dockercli.NetworkCreate( - ctx, - "ingress", - ingressOptions, - ) - - if err != nil { - log.Printf("[WARNING] Ingress network may already exist: %s", err) - } - - //docker network create --driver=overlay workers - // Specific subnet? - networkName := "shuffle_swarm_executions" - if len(swarmNetworkName) > 0 { - networkName = swarmNetworkName - } - - networkCreateOptions := network.CreateOptions{ - Driver: "overlay", - Options: options, - Attachable: true, - Ingress: false, - IPAM: &network.IPAM{ - Driver: "default", - Config: []network.IPAMConfig{ - network.IPAMConfig{ - Subnet: "10.224.224.0/24", - Gateway: "10.224.224.1", - }, - }, - }, - } - _, err = dockercli.NetworkCreate( - ctx, - networkName, - networkCreateOptions, - ) - - if err != nil { - if strings.Contains(fmt.Sprintf("%s", err), "already exists") { - // Try patching for attachable - if debug { - log.Printf("[DEBUG] Network %s already exists", networkName) - } - } else { - log.Printf("[DEBUG] Failed to create network %s for workers: %s. This is not critical, and containers will still be added", networkName, err) - } - } - - networkID := "" - - // find network ID - networks, err := dockercli.NetworkList(ctx, network.ListOptions{}) - if err == nil { - for _, net := range networks { - if net.Name == networkName { - if net.Scope == "swarm" { - log.Printf("[DEBUG] Found swarm-scoped network: %s (%s)", networkName, net.ID) - networkID = net.ID - } else { - log.Printf("[WARNING] Network %s exists but is not swarm scoped (scope=%s)", networkName, net.Scope) - } - break - } - } - } - - /* - isMemcachedRunning, err := checkMemcached(ctx, dockercli) - if err != nil { - log.Printf("[ERROR] Failed checking memcached: %s", err) - } - if isMemcachedRunning == false { - log.Printf("[ERROR] Memcached is not running. Will try to deploy it.") - deployMemcached(dockercli) - } - - ip := "shuffle-cache" - if len(os.Getenv("SHUFFLE_MEMCACHED")) == 0 { - os.Setenv("SHUFFLE_MEMCACHED", fmt.Sprintf("%s:11211", ip)) - } - */ - - if networkID == "" { - log.Printf("[ERROR] Network %s does not exist", networkName) - networkID = networkName - } - - defaultNetworkAttach := false - if containerId != "" { - log.Printf("[DEBUG] Should connect orborus container to worker network as it's running in Docker with name %#v!", containerId) - // https://pkg.go.dev/github.com/docker/docker@v20.10.12+incompatible/api/types/network#EndpointSettings - networkConfig := &network.EndpointSettings{} - err := dockercli.NetworkConnect(ctx, networkID, containerId, networkConfig) - if err != nil { - log.Printf("[ERROR] Failed connecting Orborus to docker network %s: %s", networkName, err) - } - - if len(containerId) == 64 && baseUrl == "http://shuffle-backend:5001" { - log.Printf("[WARNING] Network MAY not work due to backend being %s and container length 64. Will try to attach shuffle_shuffle network", baseUrl) - defaultNetworkAttach = true - } - } - - if len(os.Getenv("DOCKER_HOST")) > 0 { - log.Printf("[DEBUG] Deploying docker socket proxy to the network %s as the DOCKER_HOST variable is set", networkName) - - listOptions := container.ListOptions{ - All: true, - } - containers, err := dockercli.ContainerList(ctx, listOptions) - - if err == nil { - for _, container := range containers { - if strings.Contains(strings.ToLower(container.Image), "docker-socket-proxy") { - networkConfig := &network.EndpointSettings{} - err := dockercli.NetworkConnect(ctx, networkID, container.ID, networkConfig) - if err != nil { - log.Printf("[ERROR] Failed connecting Docker socket proxy to docker network %s: %s", networkName, err) - } else { - log.Printf("[INFO] Attached the docker socket proxy to the execution network") - } - - break - } - } - } else { - log.Printf("[ERROR] Failed listing containers when deploying socket proxy on swarm: %s", err) - } - //} else { - // log.Printf("[ERROR] Failed listing and finding the right image for docker socket proxy: %s", err) - //} - } - - // Running 2 by default instead of 1. Higher scale mechanisms - es - replicas := uint64(1) - scaleReplicas := os.Getenv("SHUFFLE_SCALE_REPLICAS") - if len(scaleReplicas) > 0 { - tmpInt, err := strconv.Atoi(scaleReplicas) - if err != nil { - log.Printf("[ERROR] %s is not a valid number for replication", scaleReplicas) - } else { - replicas = uint64(tmpInt) - } - - log.Printf("[DEBUG] SHUFFLE_SCALE_REPLICAS set to value %#v. Trying to overwrite default (%d/node)", scaleReplicas, replicas) - } - - innerContainerName := fmt.Sprintf("shuffle-workers") - cnt, err := findActiveSwarmNodes() - if err != nil { - log.Printf("[ERROR] Failed to find active swarm nodes: %s. Defaulting to 1", err) - } - - nodeCount := uint64(1) - if cnt > 0 { - nodeCount = uint64(cnt) - } - - appReplicas := os.Getenv("SHUFFLE_APP_REPLICAS") - appReplicaCnt := 2 - if len(appReplicas) > 0 { - newCnt, err := strconv.Atoi(appReplicas) - if err != nil { - log.Printf("[ERROR] %s is not a valid number for SHUFFLE_APP_REPLICAS", appReplicas) - } else { - appReplicaCnt = newCnt - } - } - - log.Printf("[DEBUG] Found %d node(s) to replicate over. Defaulting to 1 IF we can't auto-discover them.", cnt) - - // FIXME: From September 2025 - This is set back to 1, as this doesn't really reflect how scale works at all. It is just confusing, and makes number larger/smaller "arbitrarily" instead of using default docker scale - nodeCount = 1 - replicatedJobs := uint64(replicas * nodeCount) - - log.Printf("[DEBUG] Deploying %d container(s) for worker with swarm to each node. Service name: %s. Image: %s", replicas, innerContainerName, image) - - if timezone == "" { - timezone = "Europe/Amsterdam" - } - - // FIXME: May not need ingress ports. Could use internal services and DNS of swarm itself - // https://github.com/moby/moby/blob/e2f740de442bac52b280bc485a3ca5b31567d938/api/types/swarm/service.go#L46 - serviceSpec := swarm.ServiceSpec{ - Annotations: swarm.Annotations{ - Name: innerContainerName, - Labels: map[string]string{}, - }, - Mode: swarm.ServiceMode{ - Replicated: &swarm.ReplicatedService{ - Replicas: &replicatedJobs, - }, - }, - Networks: []swarm.NetworkAttachmentConfig{ - swarm.NetworkAttachmentConfig{ - Target: networkID, - }, - swarm.NetworkAttachmentConfig{ - Target: "ingress", - }, - }, - EndpointSpec: &swarm.EndpointSpec{ - Mode: "vip", - Ports: []swarm.PortConfig{ - swarm.PortConfig{ - Protocol: swarm.PortConfigProtocolTCP, - PublishMode: swarm.PortConfigPublishModeIngress, - Name: "worker-port", - PublishedPort: 33333, - TargetPort: 33333, - }, - }, - }, - TaskTemplate: swarm.TaskSpec{ - Resources: &swarm.ResourceRequirements{ - Reservations: &swarm.Resources{}, - }, - LogDriver: &swarm.Driver{ - Name: "json-file", - Options: map[string]string{ - "max-size": "10m", - }, - }, - ContainerSpec: &swarm.ContainerSpec{ - Image: image, - Env: []string{ - fmt.Sprintf("SHUFFLE_SWARM_CONFIG=%s", os.Getenv("SHUFFLE_SWARM_CONFIG")), - fmt.Sprintf("SHUFFLE_SWARM_NETWORK_NAME=%s", networkName), - fmt.Sprintf("SHUFFLE_APP_REPLICAS=%d", appReplicaCnt), - fmt.Sprintf("SHUFFLE_LOGS_DISABLED=%s", os.Getenv("SHUFFLE_LOGS_DISABLED")), - fmt.Sprintf("DEBUG_MEMORY=%s", os.Getenv("DEBUG_MEMORY")), - fmt.Sprintf("SHUFFLE_APP_SDK_TIMEOUT=%s", os.Getenv("SHUFFLE_APP_SDK_TIMEOUT")), - fmt.Sprintf("SHUFFLE_MAX_SWARM_NODES=%s", os.Getenv("SHUFFLE_MAX_SWARM_NODES")), - fmt.Sprintf("SHUFFLE_BASE_IMAGE_NAME=%s", os.Getenv("SHUFFLE_BASE_IMAGE_NAME")), - fmt.Sprintf("SHUFFLE_APP_REQUEST_TIMEOUT=%s", os.Getenv("SHUFFLE_APP_REQUEST_TIMEOUT")), - }, - //Hosts: []string{ - // innerContainerName, - //}, - }, - RestartPolicy: &swarm.RestartPolicy{ - Condition: swarm.RestartPolicyConditionOnFailure, - }, - Placement: &swarm.Placement{ - Constraints: []string{}, - }, - }, - } - - if defaultNetworkAttach == true || strings.ToLower(os.Getenv("SHUFFLE_DEFAULT_NETWORK_ATTACH")) == "true" { - targetName := "shuffle_shuffle" - isAttachable := false - networks, err := dockercli.NetworkList(ctx, network.ListOptions{}) - if err == nil { - for _, net := range networks { - if net.Name == targetName { - if net.Scope == "swarm" { - log.Printf("[DEBUG] Found swarm-scoped network: %s", targetName) - isAttachable = true - } else { - log.Printf("[WARNING] Network %s exist but is not swarm scoped (scope=%s)", targetName, net.Scope) - } - break - } - } - } - - if isAttachable { - log.Printf("[DEBUG] Adding network attach for network %s to worker in swarm", targetName) - serviceSpec.Networks = append(serviceSpec.Networks, swarm.NetworkAttachmentConfig{ - Target: targetName, - }) - - // FIXM: Remove this if deployment fails? - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_SWARM_OTHER_NETWORK=%s", targetName)) - } - } - - if dockerApiVersion != "" { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("DOCKER_API_VERSION=%s", dockerApiVersion)) - } - - if len(os.Getenv("SHUFFLE_SCALE_REPLICAS")) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_SCALE_REPLICAS=%s", os.Getenv("SHUFFLE_SCALE_REPLICAS"))) - } - - if len(os.Getenv("SHUFFLE_MEMCACHED")) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_MEMCACHED=%s", os.Getenv("SHUFFLE_MEMCACHED"))) - } - - if strings.ToLower(os.Getenv("SHUFFLE_PASS_WORKER_PROXY")) == "true" { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY"))) - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY"))) - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("NO_PROXY=%s", os.Getenv("NO_PROXY"))) - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("no_proxy=%s", os.Getenv("no_proxy"))) - } - - if len(workerServerUrl) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_WORKER_SERVER_URL=%s", os.Getenv("SHUFFLE_WORKER_SERVER_URL"))) - } - - // Handles backend - if len(os.Getenv("BASE_URL")) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("BASE_URL=%s", os.Getenv("BASE_URL"))) - } - - if len(os.Getenv("SHUFFLE_CLOUDRUN_URL")) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_CLOUDRUN_URL=%s", os.Getenv("SHUFFLE_CLOUDRUN_URL"))) - } - - if len(os.Getenv("SHUFFLE_AUTO_IMAGE_DOWNLOAD")) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_AUTO_IMAGE_DOWNLOAD=%s", os.Getenv("SHUFFLE_AUTO_IMAGE_DOWNLOAD"))) - } - - if len(os.Getenv("DOCKER_HOST")) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("DOCKER_HOST=%s", os.Getenv("DOCKER_HOST"))) - } else { - if runtime.GOOS == "windows" { - serviceSpec.TaskTemplate.ContainerSpec.Mounts = []mount.Mount{ - mount.Mount{ - Source: `\\.\pipe\docker_engine`, - Target: `\\.\pipe\docker_engine`, - Type: mount.TypeBind, - }, - } - } else { - serviceSpec.TaskTemplate.ContainerSpec.Mounts = []mount.Mount{ - mount.Mount{ - Source: "/var/run/docker.sock", - Target: "/var/run/docker.sock", - Type: mount.TypeBind, - }, - } - - } - } - - // Look for SHUFFLE_VOLUME_BINDS - if len(os.Getenv("SHUFFLE_VOLUME_BINDS")) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_VOLUME_BINDS=%s", os.Getenv("SHUFFLE_VOLUME_BINDS"))) - } - - overrideHttpProxy := os.Getenv("SHUFFLE_INTERNAL_HTTP_PROXY") - overrideHttpsProxy := os.Getenv("SHUFFLE_INTERNAL_HTTPS_PROXY") - if len(overrideHttpProxy) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_INTERNAL_HTTP_PROXY=%s", overrideHttpProxy)) - } - - if len(overrideHttpsProxy) > 0 { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_INTERNAL_HTTPS_PROXY=%s", overrideHttpsProxy)) - } - - serviceOptions := types.ServiceCreateOptions{} - _, err = dockercli.ServiceCreate( - ctx, - serviceSpec, - serviceOptions, - ) - - // Force deploy if it's not disabled - deployTenzirNode() - - if err == nil { - log.Printf("[DEBUG] Successfully deployed workers with %d replica(s) on %d node(s)", replicas, cnt) - // wait for service to be ready - time.Sleep(time.Duration(rand.Intn(4)+1) * time.Second) - - //log.Printf("[DEBUG] Servicecreate request: %#v %#v", service, err) - // patch service network - // this is an edgecase that we noticed on docker version 29 - // and API version 1.44 - services, serr := dockercli.ServiceList(ctx, types.ServiceListOptions{}) - if serr == nil { - for _, svc := range services { - if svc.Spec.Annotations.Name == innerContainerName { - log.Printf("[DEBUG] Found service %s (%s) — patching network attach", innerContainerName, svc.ID) - - spec := svc.Spec - spec.TaskTemplate.Networks = append(spec.TaskTemplate.Networks, swarm.NetworkAttachmentConfig{ - Target: networkID, - }) - - _, uerr := dockercli.ServiceUpdate(ctx, svc.ID, svc.Version, spec, types.ServiceUpdateOptions{}) - if uerr != nil { - log.Printf("[WARNING] Failed to patch service %s with network %s: %v", innerContainerName, networkID, uerr) - } else { - log.Printf("[INFO] Successfully attached network %s to service %s", networkID, innerContainerName) - } - break - } - } - } else { - log.Printf("[WARNING] Failed to list services for patching network attach: %v", serr) - } - } else { - if !strings.Contains(fmt.Sprintf("%s", err), "Already Exists") && !strings.Contains(fmt.Sprintf("%s", err), "is already in use by service") { - log.Printf("[ERROR] Failed making service: %s", err) - if strings.Contains(fmt.Sprintf("%s", err), "networks scoped to the swarm can be used") { - log.Printf("[WARNING] Swarm network attachment failed, retrying without shuffle_shuffle") - - var updatedNetworks []swarm.NetworkAttachmentConfig - for _, net := range serviceSpec.Networks { - if net.Target != "shuffle_shuffle" { - updatedNetworks = append(updatedNetworks, net) - } - } - serviceSpec.Networks = updatedNetworks - - var updatedEnv []string - for _, env := range serviceSpec.TaskTemplate.ContainerSpec.Env { - if !strings.HasPrefix(env, "SHUFFLE_SWARM_OTHER_NETWORK=") { - updatedEnv = append(updatedEnv, env) - } - } - serviceSpec.TaskTemplate.ContainerSpec.Env = updatedEnv - serviceOptions := types.ServiceCreateOptions{} - _, err = dockercli.ServiceCreate( - ctx, - serviceSpec, - serviceOptions, - ) - if err != nil { - log.Printf("[ERROR] Failed to deploy service even without shuffle_shuffle network: %s", err) - } - } - } else { - log.Printf("[WARNING] Failed deploying workers: %s", err) - if len(serviceSpec.Networks) > 1 { - serviceSpec.Networks = []swarm.NetworkAttachmentConfig{ - swarm.NetworkAttachmentConfig{ - Target: "shuffle_shuffle", - }, - } - - _, _ = dockercli.ServiceCreate( - ctx, - serviceSpec, - serviceOptions, - ) - } - } - } -} - -// Deploys the worker with the current available environments -// https://docs.docker.com/engine/api/sdk/examples/ -func buildEnvVars(envMap map[string]string) []corev1.EnvVar { - var envVars []corev1.EnvVar - for key, value := range envMap { - envVars = append(envVars, corev1.EnvVar{Name: key, Value: value}) - } - - return envVars -} - -func buildResourcesFromEnv() corev1.ResourceRequirements { - requests := corev1.ResourceList{} - limits := corev1.ResourceList{} - - type item struct { - env string - resourceName corev1.ResourceName - resourceList corev1.ResourceList - } - - items := []item{ - // kubernetes requests - {env: "SHUFFLE_WORKER_CPU_REQUEST", resourceName: corev1.ResourceCPU, resourceList: requests}, - {env: "SHUFFLE_WORKER_MEMORY_REQUEST", resourceName: corev1.ResourceMemory, resourceList: requests}, - {env: "SHUFFLE_WORKER_EPHEMERAL_STORAGE_REQUEST", resourceName: corev1.ResourceEphemeralStorage, resourceList: requests}, - // kubernetes limits - {env: "SHUFFLE_WORKER_CPU_LIMIT", resourceName: corev1.ResourceCPU, resourceList: limits}, - {env: "SHUFFLE_WORKER_MEMORY_LIMIT", resourceName: corev1.ResourceMemory, resourceList: limits}, - {env: "SHUFFLE_WORKER_EPHEMERAL_STORAGE_LIMIT", resourceName: corev1.ResourceEphemeralStorage, resourceList: limits}, - } - - for _, it := range items { - if value := strings.TrimSpace(os.Getenv(it.env)); value != "" { - if quantity, err := resource.ParseQuantity(value); err == nil { - it.resourceList[it.resourceName] = quantity - } else { - log.Printf("[WARNING] Cannot parse %s=%q as resource quantity: %v", it.env, value, err) - } - } - } - - rr := corev1.ResourceRequirements{} - if len(requests) > 0 { - rr.Requests = requests - } - if len(limits) > 0 { - rr.Limits = limits - } - - return rr -} - -func handleBackendImageDownload(ctx context.Context, images string) error { - - // Replicate images with lowercase, as the name may be wrong - // Most of the time lowercase is correct. Swapping to have that first - originalImages := images - images = strings.ToLower(images) + "," + originalImages - - // Remove the image - handled := []string{} - //log.Printf("[DEBUG] Removing existing image (s): %s", images) - newImages := []string{} - - successful := []string{} - for _, curimage := range strings.Split(images, ",") { - curimage = strings.TrimSpace(curimage) - if shuffle.ArrayContains(handled, curimage) { - continue - } - - handled = append(handled, curimage) - if !strings.Contains(curimage, "/") { - curimage = fmt.Sprintf("frikky/shuffle:%s", curimage) - } - - newImages = append(newImages, curimage) - - // Force remove the current image to avoid cached layers - // if swarmConfig == "run" || swarmConfig == "swarm" { - // _, err := dockercli.ImageRemove(ctx, curimage, image.RemoveOptions{ - // Force: true, - // PruneChildren: true, - // }) - - // if err != nil { - // log.Printf("[ERROR] Failed removing image for re-download: %s", err) - // } else { - // log.Printf("[DEBUG] Removed image: %s", curimage) - // } - // } else { - // //log.Printf("[DEBUG] Skipping image removal for %s as swarmConfig is not set to run or swarm. Value: %#v", curimage, swarmConfig) - // } - - err := shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, curimage) - if err != nil { - //log.Printf("[ERROR] Failed downloading image: %s", err) - } else { - //log.Printf("[DEBUG] Downloaded image: %s", curimage) - successful = append(successful, curimage) - } - } - - if len(successful) == 0 { - log.Printf("[ERROR] Failed downloading image copies: %s. This means the app may not have been updated.", strings.Join(handled, ", ")) - } else { - log.Printf("[DEBUG] Successfully downloaded image copies: %s", strings.Join(successful, ", ")) - } - - if swarmConfig == "run" || swarmConfig == "swarm" { - log.Printf("[DEBUG] Should update service with new image after updating(s): %s. \n\nBETA REPLACEMENT IMPLEMENTATION: Contact support@shuffler.io for support.", strings.Join(newImages, "\n")) - - // 1. Download the image - // 2. Find the existing service using the image - // 3. Update the service with the new image in a rolling restart - - // Find the existing service - serviceListOptions := types.ServiceListOptions{} - services, err := dockercli.ServiceList( - ctx, - serviceListOptions, - ) - - if err != nil { - log.Printf("[ERROR] Failed finding services: %s", err) - } else { - found := false - for _, service := range services { - //log.Printf("Service image: %s", service.Spec.TaskTemplate.ContainerSpec.Image) - - for _, image := range newImages { - if !strings.Contains(service.Spec.TaskTemplate.ContainerSpec.Image, image) { - continue - } - - log.Printf("[DEBUG] Found service for image: %#v", service.Spec.Annotations.Name) - - // Update the service to run with the new image - //docker service update --image username/imagename:latest servicename --force - serviceUpdateOptions := types.ServiceUpdateOptions{} - service.Spec.TaskTemplate.ForceUpdate++ - resp, err := dockercli.ServiceUpdate( - ctx, - service.ID, - service.Version, - service.Spec, - serviceUpdateOptions, - ) - - if err != nil { - log.Printf("[ERROR] Failed updating service %s with the new image %s: %s. Resp: %#v", service.Spec.Annotations.Name, image, err, resp) - } else { - log.Printf("[DEBUG] Updated service %s with the new image %s. Resp: %#v", service.Spec.Annotations.Name, image, resp) - - found = true - - if !strings.Contains(fmt.Sprintf("%s", resp), "error") { - break - } else { - log.Printf("[ERROR] Failed updating service %s with the new image %s: %s. Resp: %#v", service.Spec.Annotations.Name, image, err, resp) - } - } - } - - if found { - break - } - } - - if !found { - log.Printf("[DEBUG] Failed to find service to update for service %s", newImages) - } - - } - - } - - return nil -} - -func fixk8sRoles() { - clientset, _, err := shuffle.GetKubernetesClient() - if err != nil { - log.Printf("[ERROR] Error getting kubernetes client: %s", err) - os.Exit(1) - } - - kubernetesNamespace := "default" - - // Check if namespace exist as variable. If so, make it - if len(os.Getenv("KUBERNETES_NAMESPACE")) > 0 { - kubernetesNamespace = os.Getenv("KUBERNETES_NAMESPACE") - } - - // fix roles - // check if "service-creator" role is assigned to the service account "default" - // roleBindingNames := []string{"service-creator-binding", "pod-creator-binding", "deployment-creator-binding"} - serviceAccountName := "default" - roleBindingName := "creator-all" - - resourceTypes := []string{"services", "pods", "deployments"} - - // Check if the RoleBinding exists - roleBinding, err := clientset.RbacV1().RoleBindings(kubernetesNamespace).Get(context.TODO(), roleBindingName, metav1.GetOptions{}) - if err != nil { - log.Printf("[WARNING] Failed to get RoleBinding %s: %s", roleBindingName, err) - - // create role and rolebinding - role := &rbacv1.Role{ - ObjectMeta: metav1.ObjectMeta{ - Name: roleBindingName, - }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"", "apps"}, - Resources: resourceTypes, - Verbs: []string{"create", "list"}, - }, - }, - } - - ctx := context.TODO() - - _, err := clientset.RbacV1().Roles(kubernetesNamespace).Create(ctx, role, metav1.CreateOptions{}) - if err != nil { - log.Printf("[ERROR] Failed to create Role %s: %s", roleBindingName, err) - if !strings.Contains(fmt.Sprintf("%s", err), "already exists") { - log.Printf("[INFO] role %s already exists", roleBindingName) - } - } - - roleBinding := &rbacv1.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: roleBindingName, - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: serviceAccountName, - Namespace: kubernetesNamespace, - }, - }, - RoleRef: rbacv1.RoleRef{ - Kind: "Role", - Name: roleBindingName, - }, - } - - _, err = clientset.RbacV1().RoleBindings(kubernetesNamespace).Create(ctx, roleBinding, metav1.CreateOptions{}) - if err != nil { - log.Printf("[ERROR] Failed to create RoleBinding %s: %s", roleBindingName, err) - if strings.Contains(fmt.Sprintf("%s", err), "already exists") { - log.Printf("[INFO] rolebinding %s already exists", roleBindingName) - } - } - - log.Printf("[INFO] Created Role %s and RoleBinding %s", roleBindingName, roleBindingName) - } else { - log.Printf("[INFO] RoleBinding %s exists", roleBindingName) - } - - // Check if the RoleBinding is assigned to the service account - var found bool - for _, subject := range roleBinding.Subjects { - if subject.Kind == "ServiceAccount" && subject.Name == serviceAccountName { - found = true - break - } - } - - if !found { - log.Printf("[WARNING] Service account %s is not assigned to RoleBinding %s\n", serviceAccountName, roleBindingName) - // assign the service account to the rolebinding - roleBinding.Subjects = append(roleBinding.Subjects, rbacv1.Subject{ - Kind: "ServiceAccount", - Name: serviceAccountName, - Namespace: kubernetesNamespace, - }) - - ctx := context.TODO() - - _, err := clientset.RbacV1().RoleBindings(kubernetesNamespace).Update(ctx, roleBinding, metav1.UpdateOptions{}) - if err != nil { - log.Printf("[ERROR](ns - %s) Failed to update RoleBinding %s: %s", kubernetesNamespace, roleBindingName, err) - if !strings.Contains(fmt.Sprintf("%s", err), "already exists") { - log.Printf("[INFO] rolebinding %s already exists", roleBindingName) - } - } - } -} - -// TODO: Check if deployment or service already exist by labels and only create if not already exists -func deployK8sWorker(image string, identifier string, env []string) error { - env = append(env, fmt.Sprintf("IS_KUBERNETES=true")) - env = append(env, fmt.Sprintf("KUBERNETES_NAMESPACE=%s", os.Getenv("KUBERNETES_NAMESPACE"))) - - // app resource env - for _, k := range []string{ - "SHUFFLE_APP_CPU_REQUEST", - "SHUFFLE_APP_MEMORY_REQUEST", - "SHUFFLE_APP_EPHEMERAL_STORAGE_REQUEST", - "SHUFFLE_APP_CPU_LIMIT", - "SHUFFLE_APP_MEMORY_LIMIT", - "SHUFFLE_APP_EPHEMERAL_STORAGE_LIMIT", - } { - if v := os.Getenv(k); v != "" { - env = append(env, fmt.Sprintf("%s=%s", k, v)) - } - } - - if len(os.Getenv("KUBERNETES_SERVICE_HOST")) > 0 { - env = append(env, fmt.Sprintf("KUBERNETES_SERVICE_HOST=%s", os.Getenv("KUBERNETES_SERVICE_HOST"))) - } - - if len(os.Getenv("SHUFFLE_MEMCACHED")) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_MEMCACHED=%s", os.Getenv("SHUFFLE_MEMCACHED"))) - } - - if len(os.Getenv("KUBERNETES_SERVICE_PORT")) > 0 { - env = append(env, fmt.Sprintf("KUBERNETES_SERVICE_PORT=%s", os.Getenv("KUBERNETES_SERVICE_PORT"))) - } - - if len(os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_BASE_IMAGE_REGISTRY=%s", os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY"))) - } - - if len(os.Getenv("SHUFFLE_BASE_IMAGE_NAME")) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_BASE_IMAGE_NAME=%s", os.Getenv("SHUFFLE_BASE_IMAGE_NAME"))) - } else { - log.Printf("[INFO] SHUFFLE_BASE_IMAGE_NAME is not set. Defaulting to %s", baseimagename) - env = append(env, fmt.Sprintf("SHUFFLE_BASE_IMAGE_NAME=%s", baseimagename)) - } - - if len(os.Getenv("REGISTRY_URL")) > 0 { - env = append(env, fmt.Sprintf("REGISTRY_URL=%s", os.Getenv("REGISTRY_URL"))) - } - - if len(os.Getenv("SHUFFLE_USE_GHCR_OVERRIDE_FOR_AUTODEPLOY")) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_USE_GHCR_OVERRIDE_FOR_AUTODEPLOY=%s", os.Getenv("SHUFFLE_USE_GHCR_OVERRIDE_FOR_AUTODEPLOY"))) - } - - if len(os.Getenv("SHUFFLE_APP_EXPOSED_PORT")) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_APP_EXPOSED_PORT=%s", os.Getenv("SHUFFLE_APP_EXPOSED_PORT"))) - } - - if len(appServiceAccountName) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_APP_SERVICE_ACCOUNT_NAME=%s", appServiceAccountName)) - } - - if len(appPodSecurityContext) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_APP_POD_SECURITY_CONTEXT=%s", appPodSecurityContext)) - } - - if len(appContainerSecurityContext) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_APP_CONTAINER_SECURITY_CONTEXT=%s", appContainerSecurityContext)) - } - - if len(os.Getenv("SHUFFLE_APP_MOUNT_TMP_VOLUME")) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_APP_MOUNT_TMP_VOLUME=%s", os.Getenv("SHUFFLE_APP_MOUNT_TMP_VOLUME"))) - } - - if len(os.Getenv("SHUFFLE_LOGS_DISABLED")) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_LOGS_DISABLED=%s", os.Getenv("SHUFFLE_LOGS_DISABLED"))) - } - - if len(os.Getenv("SHUFFLE_APP_REPLICAS")) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_APP_REPLICAS=%s", os.Getenv("SHUFFLE_APP_REPLICAS"))) - } - - clientset, _, err := shuffle.GetKubernetesClient() - if err != nil { - log.Printf("[ERROR] Error getting kubernetes client:", err) - return err - } - - //env = append(env, fmt.Sprintf("KUBERNETES_CONFIG=%s", config.String())) - - // Check if namespace exist as variable. If so, make it - if len(os.Getenv("KUBERNETES_NAMESPACE")) > 0 && !namespacemade { - kubernetesNamespace = os.Getenv("KUBERNETES_NAMESPACE") - - // Make the namespace - namespace := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: os.Getenv("KUBERNETES_NAMESPACE"), - }, - } - - _, err := clientset.CoreV1().Namespaces().Create(context.Background(), namespace, metav1.CreateOptions{}) - if err != nil { - if !strings.Contains(strings.ToLower(fmt.Sprintf("%s", err)), "already exists") { - log.Printf("[ERROR] Failed creating Kubernetes namespace: %s", err) - } else { - namespacemade = true - } - } else { - namespacemade = true - } - } - - // Required format: - // url/org/repo/appname:tag - // url/org/repo/appname:tag - - //env = append(env, fmt.Sprintf("SHUFFLE_SWARM_CONFIG=%s", swarmConfig)) - env = append(env, fmt.Sprintf("BASE_URL=%s", baseUrl)) - env = append(env, fmt.Sprintf("SHUFFLE_SWARM_CONFIG=run")) - env = append(env, fmt.Sprintf("WORKER_HOSTNAME=%s", "shuffle-workers")) - - if len(kubernetesNamespace) == 0 { - foundNamespace, err := shuffle.GetKubernetesNamespace() - if err != nil { - //log.Printf("[ERROR] Failed getting Kubernetes namespace: %s", err) - } - - if len(foundNamespace) > 0 { - kubernetesNamespace = foundNamespace - os.Setenv("KUBERNETES_NAMESPACE", kubernetesNamespace) - } - } - - if len(kubernetesNamespace) == 0 { - kubernetesNamespace = "default" - } - - kubernetesImage := os.Getenv("SHUFFLE_KUBERNETES_WORKER") - if len(kubernetesImage) == 0 { - kubernetesImage = image - } - log.Printf("[DEBUG] Using Kubernetes worker image '%s'", kubernetesImage) - // image = "shuffle-worker:v1" //hard coded image name to test locally - - envMap := make(map[string]string) - for _, envStr := range env { - parts := strings.SplitN(envStr, "=", 2) - if len(parts) == 2 { - envMap[parts[0]] = parts[1] - } - } - - labels := map[string]string{ - // Well-known Kubernetes labels - "app.kubernetes.io/name": "shuffle-worker", - "app.kubernetes.io/instance": identifier, - "app.kubernetes.io/part-of": "shuffle", - "app.kubernetes.io/managed-by": "shuffle-orborus", - // Keep legacy labels for backward compatibility - "container": "shuffle-worker", - } - - matchLabels := map[string]string{ - "app.kubernetes.io/name": "shuffle-worker", - "app.kubernetes.io/instance": identifier, - } - - // Parse security contexts from env - var podSecurityContext *corev1.PodSecurityContext - var containerSecurityContext *corev1.SecurityContext - - if len(workerPodSecurityContext) > 0 { - podSecurityContext = &corev1.PodSecurityContext{} - err = json.Unmarshal([]byte(workerPodSecurityContext), podSecurityContext) - if err != nil { - log.Printf("[ERROR] Failed to unmarshal worker pod security context: %v", err) - return fmt.Errorf("failed to unmarshal worker pod security context: %v", err) - } - } - - if len(workerContainerSecurityContext) > 0 { - containerSecurityContext = &corev1.SecurityContext{} - err = json.Unmarshal([]byte(workerContainerSecurityContext), containerSecurityContext) - if err != nil { - log.Printf("[ERROR] Failed to unmarshal worker container security context: %v", err) - return fmt.Errorf("failed to unmarshal worker container security context: %v", err) - } - } - - containerAttachment := corev1.Container{ - Name: identifier, - Image: kubernetesImage, - Env: buildEnvVars(envMap), - SecurityContext: containerSecurityContext, - Resources: buildResourcesFromEnv(), - - //ImagePullPolicy: "Never", - ImagePullPolicy: corev1.PullIfNotPresent, - } - - if len(os.Getenv("REGISTRY_URL")) > 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_NAME")) > 0 { - log.Printf("[INFO] Setting image pull policy to Always as private registry is used.") - containerAttachment.ImagePullPolicy = corev1.PullAlways - } - - podname := shuffle.GetPodName() - ctx := context.Background() - if len(podname) > 0 { - _, err := shuffle.GetCurrentPodNetworkConfig(ctx, clientset, kubernetesNamespace, podname) - if err != nil { - log.Printf("[ERROR] Failed getting current pod network: %s", err) - } else { - log.Printf("[DEBUG] Current pod found!") - // currentPodStatus = k8s.io/api/core/v1.PodStatus - } - } - - // While testing: - // kubectl delete pods --all --all-namespaces; kubectl delete services --all --all-namespaces - // pod := &corev1.Pod{ - // ObjectMeta: metav1.ObjectMeta{ - // Name: identifier, - // Labels: containerLabels, - // }, - // Spec: corev1.PodSpec{ - // RestartPolicy: "Never", - // // DNSPolicy: "Default", - // DNSPolicy: corev1.DNSClusterFirst, - // // NodeSelector: map[string]string{ - // // "node": "master", - // // }, - // Containers: []corev1.Container{ - // containerAttachment, - // }, - // }, - // } - - // // Check if running on ARM or x86 to download the correct image - - // // Get current pod's network so we can make the pod in it - - // _, err = clientset.CoreV1().Pods(kubernetesNamespace).List(context.Background(), metav1.ListOptions{}) - // if err != nil { - // log.Printf("[ERROR] Failed listing pods: %s", err) - // } - - // createdPod, err := clientset.CoreV1().Pods(kubernetesNamespace).Create(context.Background(), pod, metav1.CreateOptions{}) - // if err != nil { - // //log.Printf("[ERROR] Failed creating pod: %v", err) - // return err - // } - - // log.Printf("[INFO] Created pod %q in namespace %q\n", createdPod.Name, createdPod.Namespace) - - // // kubectl expose pod shuffle-workers --type=LoadBalancer --port=33333 - // service := &corev1.Service{ - // ObjectMeta: metav1.ObjectMeta{ - // Name: identifier, - // }, - // Spec: corev1.ServiceSpec{ - // Selector: map[string]string{ - // "container": "shuffle-workers", - // }, - // Ports: []corev1.ServicePort{ - // { - // Protocol: "TCP", - // Port: 33333, - // TargetPort: intstr.FromInt(33333), - // }, - // }, - // Type: corev1.ServiceTypeLoadBalancer, - // }, - // } - - // _, err = clientset.CoreV1().Services(kubernetesNamespace).Create(context.TODO(), service, metav1.CreateOptions{}) - // if err != nil { - // log.Printf("[ERROR] Failed creating service: %v", err) - // return err - // } - - replicaNumberStr := os.Getenv("SHUFFLE_SCALE_REPLICAS") - replicaNumber := 1 - if len(replicaNumberStr) > 0 { - tmpInt, err := strconv.Atoi(replicaNumberStr) - if err != nil { - log.Printf("[ERROR] %s is not a valid number for replication", replicaNumberStr) - } else { - replicaNumber = tmpInt - - } - } - - existing, err := clientset.AppsV1().Deployments(kubernetesNamespace).List(ctx, metav1.ListOptions{ - LabelSelector: "app.kubernetes.io/name=shuffle-worker", - }) - if err != nil { - log.Printf("[ERROR] Failed listing existing deployments: %v", err) - } - - if len(existing.Items) > 0 { - log.Printf("[INFO] Found existing deployments, skipping creation") - return nil - } - - replicaNumberInt32 := int32(replicaNumber) - // worker makes authenticated requests to the k8s api to create app deployments. - // Therefore, it needs to have access to the service account token. - automountServiceAccountToken := true - - deployment := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: identifier, - Labels: labels, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: &replicaNumberInt32, - Selector: &metav1.LabelSelector{ - MatchLabels: matchLabels, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - containerAttachment, - }, - DNSPolicy: corev1.DNSClusterFirst, - ServiceAccountName: workerServiceAccountName, - AutomountServiceAccountToken: &automountServiceAccountToken, - SecurityContext: podSecurityContext, - }, - }, - }, - } - - _, err = clientset.AppsV1().Deployments(kubernetesNamespace).Create(context.Background(), deployment, metav1.CreateOptions{}) - if err != nil { - log.Printf("[ERROR] Failed creating deployment: %v", err) - return err - } - - svcAppProtocol := "http" - service := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: identifier, - Labels: labels, - }, - Spec: corev1.ServiceSpec{ - Selector: matchLabels, - Ports: []corev1.ServicePort{ - { - Protocol: "TCP", - AppProtocol: &svcAppProtocol, - Port: 33333, - TargetPort: intstr.FromInt(33333), - }, - }, - Type: corev1.ServiceTypeClusterIP, - }, - } - - _, err = clientset.CoreV1().Services(kubernetesNamespace).Create(context.Background(), service, metav1.CreateOptions{}) - if err != nil { - log.Printf("[ERROR] Failed creating service: %v", err) - return err - } - - return nil -} - -func deployWorker(image string, identifier string, env []string, executionRequest shuffle.ExecutionRequest) error { - - if len(os.Getenv("REGISTRY_URL")) > 0 && os.Getenv("REGISTRY_URL") != "" { - env = append(env, fmt.Sprintf("REGISTRY_URL=%s", os.Getenv("REGISTRY_URL"))) - } - - if swarmConfig == "run" || swarmConfig == "swarm" || isKubernetes == "true" { - // FIXME: Should we handle replies properly? - // In certain cases, a workflow may e.g. be aborted already. If it's aborted, that returns - // a 401 from the worker, which returns an error here - go sendWorkerRequest(executionRequest, image, env) - - return nil - } - - // Binds is the actual "-v" volume. - // Max 20% CPU every second - - //CPUQuota: 25000, - //CPUPeriod: 100000, - //CPUShares: 256, - hostConfig := &container.HostConfig{ - LogConfig: container.LogConfig{ - Type: "json-file", - Config: map[string]string{ - "max-size": "10m", - }, - }, - Resources: container.Resources{}, - } - - // This is just to test the mounting locally so - // I can control from what source I'm mounting - // the certs to. Default behaviour is: - // /certs:/certs. - certPath := "/certs" - if os.Getenv("SHUFFLE_CERT_PATH") != "" { - certPath = os.Getenv("SHUFFLE_CERT_PATH") - } - - _, err := os.ReadDir(certPath) - if certPath != "" && err == nil { - certVol := mount.Mount{ - Type: mount.TypeBind, - Source: certPath, - Target: "/certs", - } - - hostConfig.Mounts = append(hostConfig.Mounts, certVol) - } - - if len(os.Getenv("DOCKER_HOST")) == 0 { - if runtime.GOOS == "windows" { - hostConfig.Binds = []string{`\\.\pipe\docker_engine:\\.\pipe\docker_engine`} - } else { - hostConfig.Binds = []string{"/var/run/docker.sock:/var/run/docker.sock:rw"} - } - } - - //var swarmConfig = os.Getenv("SHUFFLE_SWARM_CONFIG") - parsedUuid := uuid.NewV4() - - config := &container.Config{ - Image: image, - Env: env, - } - - if isKubernetes != "true" { - hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) - - if strings.ToLower(cleanupEnv) == "true" { - hostConfig.AutoRemove = true - } - } - - //log.Printf("[INFO] Identifier: %s", identifier) - cont, err := dockercli.ContainerCreate( - context.Background(), - config, - hostConfig, - nil, - nil, - identifier, - ) - - if err != nil { - if strings.Contains(fmt.Sprintf("%s", err), "Conflict. The container name ") { - identifier = fmt.Sprintf("%s-%s", identifier, parsedUuid) - //log.Printf("[INFO] 2 - Identifier: %s", identifier) - cont, err = dockercli.ContainerCreate( - context.Background(), - config, - hostConfig, - nil, - nil, - identifier, - ) - - if err != nil { - log.Printf("[ERROR][%s] Container create error(2): %s", executionRequest.ExecutionId, err) - return err - } - } else { - log.Printf("[ERROR][%s] Container create error: %s", executionRequest.ExecutionId, err) - return err - } - } - - // FIXME: Verbosity for testing - //log.Printf("WORKER STARTING WITH ENV: %#v", env) - - ctx := context.Background() - containerStartOptions := container.StartOptions{} - err = dockercli.ContainerStart(ctx, cont.ID, containerStartOptions) - if err != nil { - // Trying to recreate and start WITHOUT network if it's possible. No extended checks. Old execution system (<0.9.30) - if strings.Contains(fmt.Sprintf("%s", err), "cannot join network") || strings.Contains(fmt.Sprintf("%s", err), "No such container") { - hostConfig.NetworkMode = "" - //container.NetworkMode(fmt.Sprintf("container:%s", containerId)) - cont, err = dockercli.ContainerCreate( - context.Background(), - config, - hostConfig, - nil, - nil, - identifier+"-2", - ) - if err != nil { - log.Printf("[ERROR][%s] Failed to CREATE container (2): %s", executionRequest.ExecutionId, err) - } - - err = dockercli.ContainerStart(context.Background(), cont.ID, containerStartOptions) - if err != nil { - log.Printf("[ERROR][%s] Failed to start container (2): %s", executionRequest.ExecutionId, err) - } - } else { - log.Printf("[ERROR][%s] Failed initial container start. Quitting as this is NOT a simple network issue. Err: %s", executionRequest.ExecutionId, err) - } - - if err != nil { - log.Printf("[ERROR][%s] Failed to start worker container in environment '%s': %s", executionRequest.ExecutionId, environment, err) - return err - } else { - log.Printf("[INFO][%s] Worker Container created (2). Runtime Location '%s': docker logs -f %s", executionRequest.ExecutionId, environment, cont.ID) - } - - stats, err := dockercli.ContainerInspect(ctx, cont.ID) - if err != nil { - log.Printf("[WARNING][%s] Failed checking worker '%s': %s", executionRequest.ExecutionId, cont.ID, err) - return nil - } - - containerStatus := stats.ContainerJSONBase.State.Status - if containerStatus != "running" { - log.Printf("[ERROR][%s] Status of %s is %s. Should be running. Contact support@shuffler.io if this persists.", executionRequest.ExecutionId, cont.ID, containerStatus) - } - /* - err = stopWorker(containerName) - if err != nil { - log.Printf("Failed stopping worker %s", execution.ExecutionId) - return nil - } - - err = deployWorker(dockercli, workerImage, containerName, env) - if err != nil { - log.Printf("Failed executing worker %s in state %s", execution.ExecutionId, containerStatus) - return nil - } - } - */ - } else { - log.Printf("[INFO][%s] New Worker created. Environment %s: docker logs %s", executionRequest.ExecutionId, environment, cont.ID) - } - - return nil -} - -func stopWorker(containername string) error { - ctx := context.Background() - - // containers, err := cli.ContainerList(ctx, types.ContainerListOptions{ - // All: true, - // }) - - //if err := dockercli.ContainerStop(ctx, containername, nil); err != nil { - var options container.StopOptions - if err := dockercli.ContainerStop(ctx, containername, options); err != nil { - log.Printf("[ERROR] Unable to stop container %s - running removal anyway, just in case: %s", containername, err) - } - - removeOptions := container.RemoveOptions{ - RemoveVolumes: true, - Force: true, - } - - if err := dockercli.ContainerRemove(ctx, containername, removeOptions); err != nil { - log.Printf("[ERROR] Unable to remove container: %s", err) - } - - return nil -} - -func initializeImages() { - ctx := context.Background() - - if appSdkVersion == "" { - appSdkVersion = "latest" - log.Printf("[INFO] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %#v", appSdkVersion) - } - - if workerVersion == "" { - workerVersion = "latest" - log.Printf("[INFO] SHUFFLE_WORKER_VERSION not defined. Defaulting to %#v", workerVersion) - } - - if baseimageregistry == "" { - //baseimageregistry = "ghcr.io" // Github - baseimageregistry = "docker.io" // Dockerhub - - if len(os.Getenv("REGISTRY_URL")) > 0 { - baseimageregistry = os.Getenv("REGISTRY_URL") - } else { - // os.Setenv("REGISTRY_URL", baseimageregistry) - } - - os.Setenv("SHUFFLE_BASE_IMAGE_REGISTRY", baseimageregistry) - - log.Printf("[INFO] Setting baseimageregistry to %#v", baseimageregistry) - } - - if baseimagename == "" { - // FIXME: This is probably the problem for image names tbh - //baseimagename = "shuffle" // Github (ghcr.io) - baseimagename = "frikky/shuffle" // Dockerhub - - os.Setenv("SHUFFLE_BASE_IMAGE_NAME", baseimagename) - log.Printf("[INFO] Setting baseimagename to %#v", baseimagename) - } - - // Old sane default overrides: - if baseimageregistry == "ghcr.io" && baseimagename == "shuffle" { - baseimageregistry = "docker.io" - baseimagename = "frikky/shuffle" - - os.Setenv("REGISTRY_URL", baseimageregistry) - os.Setenv("SHUFFLE_BASE_IMAGE_REGISTRY", baseimageregistry) - os.Setenv("SHUFFLE_BASE_IMAGE_NAME", baseimagename) - - log.Printf("[WARNING] Overriding bad defaults of ghcr.io/shuffle") - } - - log.Printf("[DEBUG] Setting swarm config to %#v. Default is empty.", swarmConfig) - - // This is now always static - newWorker := fmt.Sprintf("ghcr.io/shuffle/shuffle-worker:%s", workerVersion) - if len(newWorkerImage) > 0 { - newWorker = newWorkerImage - } - - // Check whether they are the same first - if os.Getenv("SHUFFLE_AUTO_IMAGE_DOWNLOAD") == "false" { - log.Printf("[DEBUG] Skipping image download as SHUFFLE_AUTO_IMAGE_DOWNLOAD is set to false") - } else { - images := []string{ - fmt.Sprintf("frikky/shuffle:app_sdk"), - newWorker, - } - - pullOptions := image.PullOptions{} - for _, image := range images { - if isKubernetes == "true" { - log.Printf("[DEBUG] Skipping image pull of '%s' because Kubernetes does it in realtime instead", image) - } else { - log.Printf("[DEBUG] Pulling image %s", image) - reader, err := dockercli.ImagePull(ctx, image, pullOptions) - if err != nil { - log.Printf("[ERROR] Failed getting image %s: %s", image, err) - - continue - } - - io.Copy(os.Stdout, reader) - log.Printf("[DEBUG] Successfully downloaded and built %s", image) - } - } - } -} - -func findActiveSwarmNodes() (int64, error) { - ctx := context.Background() - nodes, err := dockercli.NodeList(ctx, types.NodeListOptions{}) - if err != nil { - return 1, err - } - - nodeCount := int64(0) - for _, node := range nodes { - //log.Printf("ID: %s - %#v", node.ID, node.Status.State) - if node.Status.State == "ready" { - nodeCount += 1 - } - } - - // Check for SHUFFLE_MAX_NODES - // Make it into a number and check if it's lower than nodeCount - maxNodesString := os.Getenv("SHUFFLE_MAX_SWARM_NODES") - if len(maxNodesString) > 0 { - maxNodes, err := strconv.ParseInt(maxNodesString, 10, 64) - if err != nil { - return nodeCount, err - } - - if nodeCount > maxNodes { - nodeCount = maxNodes - } - } - - return nodeCount, nil -} - -// Get IP -func getLocalIP() string { - addrs, err := net.InterfaceAddrs() - if err != nil { - return "" - } - - for _, address := range addrs { - // check the address type and if it is not a loopback the display it - if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { - if ipnet.IP.To4() != nil { - return ipnet.IP.String() - } - } - } - - return "" -} - -// Get all local IPs in the system -func getLocalIPs() ([]string, error) { - var ipv4s []string - var ipv6s []string - - ifaces, err := net.Interfaces() - if err != nil { - return nil, err - } - - for _, iface := range ifaces { - if iface.Flags&net.FlagUp == 0 { - continue - } - if iface.Flags&net.FlagLoopback != 0 { - continue - } - - addrs, err := iface.Addrs() - if err != nil { - continue - } - - for _, address := range addrs { - ipnet, ok := address.(*net.IPNet) - if !ok || ipnet.IP == nil { - continue - } - - ip := ipnet.IP - if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() { - continue - } - - if ip4 := ip.To4(); ip4 != nil { - ipv4s = append(ipv4s, ip4.String()) - continue - } - - if ip.To16() != nil { - ipv6s = append(ipv6s, ip.String()) - } - } - } - - return append(ipv4s, ipv6s...), nil -} - -func checkSwarmService(ctx context.Context) { - // https://docs.docker.com/engine/reference/commandline/swarm_init/ - ip := getLocalIP() - log.Printf("[DEBUG] Attempting swarm setup on %s", ip) - - info, err := dockercli.Info(ctx) - if err != nil { - log.Printf("[WARNING] Failed to get Docker Info: %s", err) - } - - if info.Swarm.ControlAvailable { - log.Printf("[INFO] Already part of swarm as a manager") - return - } - - listenAddr := "0.0.0.0" - req := swarm.InitRequest{ - ListenAddr: fmt.Sprintf("%s:2377", listenAddr), - AdvertiseAddr: fmt.Sprintf("%s:2377", ip), - } - - id, err := dockercli.SwarmInit(ctx, req) - if err != nil { - log.Printf("[ERROR] Swarm init issue: %s. Retrying with a failover IP address from interface.", err) - - // Dummy message used for testing - //err = errors.New("Error response from daemon: could not choose an IP address to advertise since this system has multiple addresses on different interfaces (10.52.208.221 on eno1 and 192.168.122.1 on virbr0) - specify one with --advertise-addr") - // Update 28 Jan 2026: The error message updated and not as clear - - candidates, err := getLocalIPs() - if len(candidates) > 0 && err == nil { - for cnt, candidate := range candidates { - if cnt > 5 { - break - } - - req.AdvertiseAddr = fmt.Sprintf("%s:2377", candidate) - id, err = dockercli.SwarmInit(context.Background(), req) - if err != nil { - continue - } - - log.Printf("[INFO] Swarm init ID: '%s'.", id) - return - } - } - - log.Printf("[ERROR] Swarm init failed after advertise-addr retries: %s, try running swarm init manually: docker swarm init", err) - return - } -} - -func getContainerResourceUsage(ctx context.Context, cli *dockerclient.Client, containerID string) (float64, float64, error) { - // Get container stats - stats, err := cli.ContainerStats(ctx, containerID, false) - if err != nil { - return 0, 0, err - } - - defer stats.Body.Close() - // Parse and return CPU and memory utilization - cpuUsage, memoryUsage, err := parseResourceUsage(stats.Body) - if err != nil { - return 0, 0, err - } - - return cpuUsage, memoryUsage, nil -} - -func parseResourceUsage(body io.Reader) (float64, float64, error) { - //var stats types.StatsJSON - var stats container.Stats - - // Decode the stream of stats as JSON - decoder := json.NewDecoder(body) - if err := decoder.Decode(&stats); err != nil { - return 0, 0, err - } - - //log.Printf("[DEBUG] CPU : %d", stats.CPUStats.CPUUsage.TotalUsage) - //log.Printf("[DEBUG] CPU2: %d", stats.PreCPUStats.CPUUsage.TotalUsage) - if stats.CPUStats.CPUUsage.TotalUsage == 0 || stats.PreCPUStats.CPUUsage.TotalUsage == 0 { - //log.Printf("[DEBUG] BODY: %#v", stats) - return 0, 0, nil - } - - // Calculate time difference between current and previous stats in nanoseconds - timeDelta := float64(stats.Read.Sub(stats.PreRead).Nanoseconds()) - - // Calculate CPU usage percentage - cpuDelta := float64(stats.CPUStats.CPUUsage.TotalUsage - stats.PreCPUStats.CPUUsage.TotalUsage) - cpuUsage := (cpuDelta / timeDelta) * 100.0 - - // Calculate memory usage percentage - memoryUsage := float64(stats.MemoryStats.Usage) / float64(stats.MemoryStats.Limit) * 100.0 - - return cpuUsage, memoryUsage, nil - -} - -func getHostname() (string, error) { - hostname, err := os.Hostname() - if err != nil { - return "", fmt.Errorf("failed to get hostname: %w", err) - } - - // Split away TLD - parts := strings.Split(hostname, ".") - if len(parts) > 0 { - hostname = parts[0] - } - - hostname = strings.ToUpper(hostname) - return hostname, nil -} - -func getOrborusStats(ctx context.Context, sensorMode shuffle.SensorMode) shuffle.OrborusStats { - newStats := shuffle.OrborusStats{ - OrgId: org, - Environment: environment, - OrborusLabel: orborusLabel, - Timestamp: time.Now().Unix(), - - Uuid: orborusUuid, - } - - if (swarmConfig == "run" || swarmConfig == "swarm") && strings.Contains(newWorkerImage, "scale") { - newStats.Swarm = true - } - - newStats.PollTime = sleepTime - newStats.MaxQueue = maxConcurrency - newStats.Queue = executionCount - - if isKubernetes == "true" || runningMode == "kubernetes" || runningMode == "k8s" { - newStats.Kubernetes = true - return newStats - } - - // Handles Orborus in sensor mode. Sends minimal data per request, but - // once in a while (30 minutes) sends a lot of details like software etc - if sensorMode.Enabled { - cacheKey := fmt.Sprintf("orborus_sensorDetails_cache") - cached, err := shuffle.GetCache(ctx, cacheKey) - if err == nil { - cacheData := []byte(cached.([]uint8)) - err := json.Unmarshal(cacheData, &newStats.SensorDetails) - if err == nil && len(newStats.SensorDetails.Hostname) > 0 { - newStats.SensorDetails.SensorMode = true - - // Not necessary to always send as it's big - // Backend optimises this anyway - if len(newStats.SensorDetails.Serial) > 100 { - newStats.SensorDetails.Serial = "" - } - - newStats.SensorDetails.InstalledSoftware = []shuffle.Software{} - return newStats - } - // If there's an error, we ignore the cache and continue to gather details - log.Printf("[WARNING] Failed to unmarshal cached sensor details: %s. Gathering new details.", err) - } - - newStats.SensorDetails.SensorMode = true - hostname, err := getHostname() - if err == nil { - newStats.SensorDetails.Hostname = hostname - } - - newStats.SensorDetails.OS = runtime.GOOS - newStats.SensorDetails.Arch = runtime.GOARCH - newStats.SensorDetails.ElevatedAccess = shuffle.IsElevated() - newStats.SensorDetails.Serial = shuffle.GetProfiler() - - if sensorMode.SoftwareListEnabled { - // Check cache first before running the command - newStats.SensorDetails.InstalledSoftware = shuffle.ListInstalledSoftware() - } - - if sensorMode.HdEncryptedCheck { - newStats.SensorDetails.HdEncrypted = fmt.Sprintf("%t", shuffle.IsDiskEncrypted()) - } - - if sensorMode.ScreenlockCheck { - newStats.SensorDetails.AutomaticScreenlockEnabled = fmt.Sprintf("%t", shuffle.IsAutomaticScreenlockEnabled()) - } - - if len(sensorMode.LogForwarding) > 0 { - newStats.SensorDetails.LogForwarding = fmt.Sprintf("not implemented: %s", sensorMode.LogForwarding) - } - - if len(sensorMode.ResponseActions) > 0 { - newStats.SensorDetails.ResponseActions = sensorMode.ResponseActions - } - - marshalledStats, err := json.Marshal(newStats.SensorDetails) - if err == nil { - shuffle.SetCache(ctx, cacheKey, marshalledStats, 30) // Cache for 10 minutes - } - - return newStats - } else { - return newStats - } - - // FIXME: Should we reach here anymore? Can it be useful? Primarily used for stats. - // Disable orborus stats - if os.Getenv("SHUFFLE_STATS_DISABLED") == "true" { - return newStats - } - - // Use the docker API to get the CPU usage of the docker engine machine - pers, err := dockercli.Info(ctx) - if err != nil { - log.Printf("[ERROR] Failed getting docker info: %s. This is normal IF there are many containers running.", err) - return newStats - } else { - newStats.TotalContainers = pers.Containers - newStats.StoppedContainers = pers.ContainersStopped - - // Calculate the amount of CPU utilization on the host - newStats.CPU = int(pers.NCPU) - newStats.MaxCPU = int(pers.NCPU) - newStats.Memory = int(pers.MemTotal) - newStats.MaxMemory = int(pers.MemTotal) - } - - // Get list of all running containers - containers, err := dockercli.ContainerList(ctx, container.ListOptions{}) - - if err != nil { - log.Printf("[ERROR] Failed getting container list: %s", err) - return newStats - } - - // Use a WaitGroup to wait for all goroutines to finish - var wg sync.WaitGroup - - // Channel to collect results - resultCh := make(chan struct { - containerID string - cpuUsage float64 - memoryUsage float64 - }) - - // Iterate through containers and start a goroutine for each container - for _, container := range containers { - // Check if container is running - if container.State != "running" { - continue - } - - wg.Add(1) - go func(container types.Container) { - defer wg.Done() - - // Get CPU and memory usage for the container - cpuUsage, memoryUsage, err := getContainerResourceUsage(ctx, dockercli, container.ID) - if err != nil { - //log.Printf("[DEBUG] Error getting resource usage for container %s: %v\n", container.ID, err) - } - - // Send the result to the channel - resultCh <- struct { - containerID string - cpuUsage float64 - memoryUsage float64 - }{container.ID, cpuUsage, memoryUsage} - }(container) - } - - // Close the result channel after all goroutines are done - go func() { - wg.Wait() - close(resultCh) - }() - - // Collect results from the channel - - // Iterate through containers and get CPU usage - totalCPU := float64(0.0) - memUsage := float64(0.0) - for result := range resultCh { - //log.Printf("[DEBUG] Container %s CPU utilization: %.2f%%, Memory utilization: %.2f%%\n", result.containerID, result.cpuUsage, result.memoryUsage) - - // check if it's NaN or Inf - if !math.IsNaN(result.cpuUsage) { - totalCPU += float64(result.cpuUsage) - } - - if !math.IsNaN(result.memoryUsage) { - memUsage += float64(result.memoryUsage) - } - } - - newStats.CPUPercent = totalCPU / float64(newStats.CPU) - newStats.MemoryPercent = memUsage - - //log.Printf("[DEBUG] CPU: %.2f, Memory: %.2f", newStats.CPUPercent, newStats.MemoryPercent) - - /* - cpuPercent, err := cpu.Percent(250*time.Millisecond, false) - if err == nil && len(cpuPercent) > 0 { - newStats.CPUPercent = cpuPercent[0] - } - //Percent(interval time.Duration, percpu bool) ([]float64, error) - - // Get memory usage - memory, err := memory.Get() - if err != nil { - log.Printf("[ERROR] Failed getting memory stats: %s", err) - } else { - newStats.Memory = int(memory.Used) - newStats.MaxMemory = int(memory.Total) - } - */ - - // Get disk usage - /* - disk, err := disk.Get() - if err != nil { - log.Printf("[ERROR] Failed getting disk stats: %s", err) - } else { - newStats.Disk = int(disk.Used) - newStats.MaxDisk = int(disk.Total) - } - */ - - /* - // General - Disk int `json:"disk"` - - // Docker - AppContainers int `json:"app_containers"` - WorkerContainers int `json:"worker_containers"` - TotalContainers int `json:"total_containers"` - } - */ - return newStats -} - -func sendRemoveRequest(client *http.Client, toBeRemoved shuffle.ExecutionRequestWrapper, baseUrl, environment, auth, org string, sleepTime int) error { - confirmUrl := fmt.Sprintf("%s/api/v1/workflows/queue/confirm", baseUrl) - data, err := json.Marshal(toBeRemoved) - if err != nil { - log.Printf("[WARNING] Failed removal marshalling: %s", err) - time.Sleep(time.Duration(sleepTime) * time.Second) - return err - } - - result, err := http.NewRequest( - "POST", - confirmUrl, - bytes.NewBuffer([]byte(data)), - ) - - if err != nil { - log.Printf("[ERROR] Failed building confirm request: %s", err) - time.Sleep(time.Duration(sleepTime) * time.Second) - return err - } - - result.Header.Add("Content-Type", "application/json") - result.Header.Add("Org-Id", environment) - - if len(auth) > 0 { - result.Header.Add("Authorization", auth) - } - - if len(org) > 0 { - result.Header.Add("Org", org) - } - - if len(orborusLabel) > 0 { - result.Header.Add("X-Orborus-Label", orborusLabel) - } - - resultResp, err := client.Do(result) - if err != nil { - if !strings.Contains(fmt.Sprintf("%s", err), "timeout") { - log.Printf("[ERROR] Failed making confirm request: %s", err) - } - - time.Sleep(time.Duration(sleepTime) * time.Second) - return err - } - - defer resultResp.Body.Close() - body, err := ioutil.ReadAll(resultResp.Body) - if err != nil { - log.Printf("[ERROR] Failed reading confirm body: %s", err) - time.Sleep(time.Duration(sleepTime) * time.Second) - return err - } - - _ = body - //log.Printf("[DEBUG] Confirm response: %s", string(body)) - - return nil -} - -func cleanup() { - log.Printf("[INFO] Cleaning up during shutdown") - ctx := context.Background() - cleanupExistingNodes(ctx) - zombiecheck(ctx, 600, shuffle.SensorMode{ - Enabled: os.Getenv("SHUFFLE_AGENT_MODE") == "true", - }) - os.Exit(0) -} - -func StartAgentSensor(sensorMode shuffle.SensorMode) error { - if sensorMode.Enabled == false { - return errors.New("Sensor mode is not enabled. Set SHUFFLE_AGENT_MODE to true to enable it.") - } - - log.Printf("[INFO] Starting Orborus - host monitoring mode (sensor/agent)") - - // Check if inside Docker/Kubernetes. Use Docker/Kubernetes libraries - if isKubernetes == "true" || shuffle.IsRunningInCluster() { - log.Printf("[INFO] Detected Kubernetes environment. Not valid for Sensor Mode. Exiting.") - return errors.New("Kubernetes environment detected. Sensor mode is not valid in Kubernetes. Exiting.") - } else if swarmConfig == "run" || swarmConfig == "swarm" { - log.Printf("[INFO] Detected Docker Swarm environment. Not valid for Sensor Mode. Exiting.") - return errors.New("Docker Swarm environment detected. Sensor mode is not valid in Docker Swarm. Exiting.") - } - - if len(sensorMode.LogForwarding) > 0 { - log.Printf("[INFO] Audit log monitoring is enabled (SHUFFLE_LOG_FORWARDING=)") - - // Initialize telemetry configuration - telemetryConfig := shuffle.TelemetryConfig{ - Enabled: true, - Modes: []string{"audit_log"}, - BufferSize: 1000, - FlushInterval: 10 * time.Second, - } - - if excludePatterns := os.Getenv("SHUFFLE_AUDIT_LOG_EXCLUDE"); excludePatterns != "" { - patterns := strings.Split(excludePatterns, ",") - telemetryConfig.Filters = append(telemetryConfig.Filters, shuffle.TelemetryFilter{ - Type: "message", - Exclude: patterns, - }) - } - - if includePatterns := os.Getenv("SHUFFLE_AUDIT_LOG_INCLUDE"); includePatterns != "" { - patterns := strings.Split(includePatterns, ",") - telemetryConfig.Filters = append(telemetryConfig.Filters, shuffle.TelemetryFilter{ - Type: "message", - Include: patterns, - }) - } - - collector, err := shuffle.NewAuditLogCollector(telemetryConfig) - if err != nil { - log.Printf("[ERROR] Failed to create audit log collector: %v", err) - } else { - ctx := context.Background() - if err := collector.LogCollectorStart(ctx); err != nil { - log.Printf("[ERROR] Failed to start audit log collector: %v", err) - } else { - log.Printf("[INFO] Audit log collector started successfully") - - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) - - go func() { - <-sigChan - log.Printf("[INFO] Received shutdown signal, stopping audit log collector...") - collector.Stop() - - os.Exit(0) - }() - } - } - } else { - log.Printf("[INFO] Audit log monitoring is NOT enabled (SHUFFLE_LOG_FORWARDING=") - } - - return nil -} - -// Initial loop etc -func main() { - - // Checks for whether sensor mode is enabled for detection/response - sensorMode := shuffle.SensorMode{ - Enabled: os.Getenv("SHUFFLE_AGENT_SENSOR_MODE") == "true", - - SoftwareListEnabled: os.Getenv("SHUFFLE_SOFTWARE_LIST_ENABLED") == "true", - HdEncryptedCheck: os.Getenv("SHUFFLE_HD_ENCRYPTED_CHECK") == "true", - ScreenlockCheck: os.Getenv("SHUFFLE_SCREENLOCK_CHECK") == "true", - - LogForwarding: os.Getenv("SHUFFLE_LOG_FORWARDING"), - ResponseActions: os.Getenv("SHUFFLE_RESPONSE_ACTIONS"), - } - - ctx := context.Background() - workerTimeout := 600 - workerImage := fmt.Sprintf("ghcr.io/shuffle/shuffle-worker:%s", workerVersion) - if len(newWorkerImage) > 0 { - workerImage = newWorkerImage - } - - if len(orborusUuid) == 0 { - orborusUuid = uuid.NewV4().String() - } - - client := shuffle.GetExternalClient(baseUrl) - fullUrl := fmt.Sprintf("%s/api/v1/workflows/queue", baseUrl) - - // Increases default concurrency to 50 for swarm - if maxConcurrency < 50 && (swarmConfig == "run" || swarmConfig == "swarm") { - fullUrl += "?amount=50" - } - - if len(os.Getenv("SHUFFLE_ORBORUS_PULL_TIME")) > 0 { - log.Printf("[INFO] Trying to set Orborus sleep time between polls to %s", os.Getenv("SHUFFLE_ORBORUS_PULL_TIME")) - - tmpInt, err := strconv.Atoi(os.Getenv("SHUFFLE_ORBORUS_PULL_TIME")) - if err == nil { - sleepTime = tmpInt - } - } - - log.Println("[INFO] Setting up execution environment for env '%s'", environment) - if baseUrl == "" { - baseUrl = "https://uk.shuffler.io" - } - - //if orgId == "" { - // log.Printf("[ERROR] Org not defined. Set variable ORG_ID based on your org") - // os.Exit(3) - //} - if environment == "" { - log.Printf("[ERROR] Environment not defined. Set variable ENVIRONMENT_NAME to configure it.") - os.Exit(3) - } - - if timezone == "" { - timezone = "Europe/Amsterdam" - } - - log.Printf("[INFO] Using environment '%s' with timezone %s", environment, timezone) - - if sensorMode.Enabled { - - // Start high on purpose for now (no overloads) - if sleepTime < 15 { - - // For prod - if strings.Contains(baseUrl, "shuffler.io") || strings.Contains(baseUrl, ".run.app") { - log.Printf("[INFO] Running in hosted environment. Setting default sleep time to 30 seconds to avoid hitting rate limits. You can adjust this with SHUFFLE_ORBORUS_PULL_TIME.", sleepTime) - sleepTime = 15 - } - } - - if sensorMode.ResponseActions != "full" && sensorMode.ResponseActions != "controlled" { - log.Printf("[WARNING] Invalid response actions mode '%s'. Disabling. Valid options are 'full', 'controlled', or empty.", sensorMode.ResponseActions) - sensorMode.ResponseActions = "" - - } - - log.Printf("[INFO] Running in sensor/agent mode. Starting the agent.") - err := StartAgentSensor(sensorMode) - if err != nil { - log.Printf("[ERROR] Failed to start sensor/agent mode: %#v", err) - return - } - } else { - if os.Getenv("SHUFFLE_PIPELINE_STANDALONE") == "true" { - log.Printf("[INFO] Allowing use of standalone pipeline (tenzir). URL: %s", pipelineUrl) - - tenzirDisabled = false - os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") - os.Setenv("SHUFFLE_PIPELINE_ENABLED", "true") - } - - // Block until a signal is received - if shuffle.IsRunningInCluster() { - log.Printf("[INFO] Running inside k8s cluster") - } - - if isKubernetes == "true" { - fixk8sRoles() - } - - startupDelay := os.Getenv("SHUFFLE_ORBORUS_STARTUP_DELAY") - if len(startupDelay) > 0 { - log.Printf("[DEBUG] Setting startup delay to %#v", startupDelay) - - tmpInt, err := strconv.Atoi(startupDelay) - if err == nil { - time.Sleep(time.Duration(tmpInt) * time.Second) - } else { - log.Printf("[WARNING] Env SHUFFLE_ORBORUS_STARTUP_DELAY must be a number, not '%s'. Using default.", startupDelay) - } - } - - // Auto enables pipelines IF they are not mentioned - if len(os.Getenv("SHUFFLE_SKIP_PIPELINES")) == 0 { - tenzirDisabled = false - os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") - os.Setenv("SHUFFLE_PIPELINE_ENABLED", "true") - } - - if os.Getenv("SHUFFLE_SKIP_PIPELINES") != "true" && os.Getenv("SHUFFLE_PIPELINE_ENABLED") != "false" { - // Run in 15 seconds in a goroutine - go func() { - time.Sleep(15 * time.Second) - log.Printf("[INFO] Auto-downloading Sigma rules during startup") - ruleType := "sigma" - err := handleFileCategoryChange(ruleType) - if err != nil { - log.Printf("[WARNING] Failed downloading %s rules: %s", ruleType, err) - } - }() - } - - // Handle Cleanup - made it cleanup by default - if strings.ToLower(os.Getenv("SHUFFLE_CONTAINER_AUTO_CLEANUP")) != "false" && os.Getenv("CLEANUP") == "" { - cleanupEnv = "true" - } - - if len(cleanupEnv) > 0 { - log.Printf("[DEBUG] Verbose mode. NOT cleaning up. Cleanup env: %s", cleanupEnv) - } - - // Default to 120 instead of default 30 - if len(os.Getenv("SHUFFLE_APP_SDK_TIMEOUT")) == 0 { - os.Setenv("SHUFFLE_APP_SDK_TIMEOUT", "120") - } - - if workerTimeoutEnv != "" { - tmpInt, err := strconv.Atoi(workerTimeoutEnv) - if err == nil { - workerTimeout = tmpInt - } else { - log.Printf("[WARNING] Env SHUFFLE_ORBORUS_EXECUTION_TIMEOUT must be a number, not %s", workerTimeoutEnv) - } - - log.Printf("[INFO] Cleanup process running every %d seconds", workerTimeout) - } - - if concurrencyEnv != "" { - //var concurrencyEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY") - tmpInt, err := strconv.Atoi(concurrencyEnv) - if err == nil { - maxConcurrency = tmpInt - log.Printf("[INFO] Max workflow execution concurrency set to %d", maxConcurrency) - } else { - log.Printf("[WARNING] Env SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY must be a number, not %s. Defaulted to %d", workerTimeoutEnv, maxConcurrency) - } - } - - if len(os.Getenv("DOCKER_HOST")) > 0 { - log.Printf("[DEBUG] Running docker with socket proxy %s instead of default", os.Getenv("DOCKER_HOST")) - - } else { - log.Printf(`[DEBUG] Running docker with default socket /var/run/docker.sock or `) - } - - // Run by default from now - //commenting for now as its stoppoing minikube - - log.Printf("[INFO] Running towards %s (BASE_URL) with environment name %s", baseUrl, environment) - - if environment == "" { - environment = "onprem" - log.Printf("[WARNING] Defaulting to environment name %s. Set environment variable ENVIRONMENT_NAME to change. This should be the same as in the frontend action.", environment) - } - - if pipelineUrl == "" { - pipelineUrl = "http://localhost:5160" - - // Find the IP in baseUrl. Base format is http://: - if baseUrl != "" && !strings.Contains(baseUrl, "shuffle") && !strings.Contains(baseUrl, "localhost") && !strings.Contains(baseUrl, "run.app") { - urlSplit := strings.Split(baseUrl, "://") - if len(urlSplit) > 1 { - // Find the IP - ipSplit := strings.Split(urlSplit[1], ":") - if len(ipSplit) > 0 { - pipelineUrl = fmt.Sprintf("http://%s:5160", ipSplit[0]) - } - } - } - - if len(containerId) > 0 { - pipelineUrl = "http://tenzir-node:5160" - } - - log.Printf("[WARNING] SHUFFLE_PIPELINE_URL not set, falling back to default URL: %s. If BASE_URL is set, we use the external IP for that", pipelineUrl) - os.Setenv("SHUFFLE_PIPELINE_URL", pipelineUrl) - } - - // FIXME - during init, BUILD and/or LOAD worker and app_sdk - // Build/load app_sdk so it can be loaded as 127.0.0.1:5000/walkoff_app_sdk - log.Printf("[INFO] Setting up Docker environment. Downloading worker and App SDK!") - - initializeImages() - - if swarmConfig == "run" || swarmConfig == "swarm" || isKubernetes == "true" { - - if isKubernetes != "true" { - checkSwarmService(ctx) - } - - log.Printf("[DEBUG] Cleaning up containers from previous run") - cleanupExistingNodes(ctx) - time.Sleep(time.Duration(5) * time.Second) - - log.Printf("[DEBUG] Deploying worker image %s to swarm", workerImage) - - runString := "Run: \"docker service ls\" for more info" - - if isKubernetes != "true" { - deployServiceWorkers(workerImage) - - err := setBackendToSwarmNetwork(ctx) - if err != nil { - log.Printf("[WARNING] Failed setting backend to swarm network: %s", err) - } - - } else { - deployK8sWorker(workerImage, "shuffle-workers", []string{}) - runString = "Run: \"kubectl get pods\" for more info" - } - - log.Printf("[DEBUG] Waiting 45 seconds to ensure workers are deployed. %s", runString) - time.Sleep(time.Duration(45) * time.Second) - - //deployServiceWorkers(workerImage) - } - - zombiecheck(ctx, workerTimeout, sensorMode) - - if isKubernetes == "true" { - log.Printf("[INFO] Finished configuring kubernetes environment. Connecting to %s", fullUrl) - } else { - log.Printf("[INFO] Finished configuring docker environment. Connecting to %s", fullUrl) - } - } - - forwardData := bytes.NewBuffer([]byte{}) - forwardMethod := "POST" - - req, err := http.NewRequest( - forwardMethod, - fullUrl, - forwardData, - ) - - if err != nil { - log.Printf("[ERROR] Failed making request builder during init: %s", err) - return - } - - zombiecounter := 0 - - req.Header.Add("Content-Type", "application/json") - req.Header.Add("Org-Id", environment) - if len(auth) > 0 { - req.Header.Add("Authorization", auth) - } - - if len(org) > 0 { - req.Header.Add("Org", org) - } - - if len(orborusLabel) > 0 { - log.Printf("[DEBUG] Sending with Label '%s'", orborusLabel) - req.Header.Add("X-Orborus-Label", orborusLabel) - } - - if swarmConfig != "run" && swarmConfig != "swarm" { - req.Header.Add("X-Orborus-Runmode", "Default") - } else { - req.Header.Add("X-Orborus-Runmode", "Docker Swarm") - } - - if os.Getenv("SHUFFLE_MAX_CPU") != "" { - // parse - tmpInt, err := strconv.Atoi(os.Getenv("SHUFFLE_MAX_CPU")) - if err == nil { - maxCPUPercent = tmpInt - } - } - - swarmPollingTime := time.Now() - swarmRequestsMade := 0 - swarmControlMode := false - if os.Getenv("SHUFFLE_SWARM_CONTROL_MODE") == "true" { - swarmControlMode = true - } - - log.Printf("[INFO] Waiting for executions at %s with Environment %#v. Sensormode: %#v", fullUrl, environment, sensorMode.Enabled) - - hostname, err := getHostname() - hasStarted := false - for { - if req.Method == "POST" && !sensorMode.Enabled { - // Should find data to send (memory etc.) - - // Create timeout of max a few seconds just in case - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - // Marshal and set body - orborusStats := getOrborusStats(ctx, sensorMode) - - pipelinePayload, pipelineerr := sendPipelineHealthStatus(sensorMode) - - if pipelineerr != nil { - // Too verbose to be enabled. - //log.Printf("[ERROR] Failed sending pipeline health status: %s", pipelineerr) - } - - orborusStats.DataLake = pipelinePayload - jsonData, err := json.Marshal(orborusStats) - if err == nil { - req.Body = ioutil.NopCloser(bytes.NewBuffer(jsonData)) - } else { - log.Printf("[ERROR] Failed marshalling. Maybe max 4 second timeout? %s", err) - } - - if int(orborusStats.CPUPercent) > maxCPUPercent { - log.Printf("[DEBUG] CPU usage is at %f%%. This is more than the max limit the machine should be running at (%d). Waiting before continue.", orborusStats.CPUPercent, maxCPUPercent) - time.Sleep(time.Duration(sleepTime) * time.Second) - continue - } - } else if sensorMode.Enabled { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - orborusStats := getOrborusStats(ctx, sensorMode) - - jsonData, err := json.Marshal(orborusStats) - if err == nil { - req.Body = ioutil.NopCloser(bytes.NewBuffer(jsonData)) - } else { - log.Printf("[ERROR] Failed marshalling. Maybe max 4 second timeout? %s", err) - } - } - - newresp, err := client.Do(req) - if err != nil { - log.Printf("[WARNING] Failed making request to %s: %s", fullUrl, err) - - zombiecounter += 1 - if zombiecounter*sleepTime > workerTimeout { - go zombiecheck(ctx, workerTimeout, sensorMode) - zombiecounter = 0 - } - time.Sleep(time.Duration(sleepTime) * time.Second) - continue - } - - //defer newresp.Body.Close() - if newresp.StatusCode == 405 { - log.Printf("[WARNING] Received 405 from %s. This is likely due to a misconfigured base URL. Automatically swapping to GET request (backwards compatibility)", fullUrl) - - req.Method = "GET" - req.Body = nil - - //time.Sleep(time.Duration(sleepTime) * time.Second) - continue - } - - body, err := ioutil.ReadAll(newresp.Body) - if err != nil { - log.Printf("[ERROR] Failed reading body from Shuffle: %s", err) - zombiecounter += 1 - if zombiecounter*sleepTime > workerTimeout { - go zombiecheck(ctx, workerTimeout, sensorMode) - zombiecounter = 0 - } - time.Sleep(time.Duration(sleepTime) * time.Second) - continue - } - - // Controls Leader/Follower mode - if newresp.StatusCode == 409 { - log.Printf("[INFO] Another Orborus is already handling jobs. Polling every 30 seconds in case Leader stops. Resp: %s", string(body)) - time.Sleep(time.Duration(30) * time.Second) - continue - } else if newresp.StatusCode != 200 { - log.Printf("[ERROR] Backend connection failed for url '%s', or is missing (%d): %s", fullUrl, newresp.StatusCode, string(body)) - } else { - if !hasStarted { - log.Printf("[DEBUG] Starting iteration on environment %#v (default: Shuffle). Got statuscode %d from backend on first request", environment, newresp.StatusCode) - } - - if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" && os.Getenv("SHUFFLE_SCALE_REPLICAS") == "" { - //go AutoScale(ctx) - } - hasStarted = true - } - - var executionRequests shuffle.ExecutionRequestWrapper - err = json.Unmarshal(body, &executionRequests) - if err != nil { - log.Printf("[WARNING] Failed executionrequest in queue unmarshaling: %s", err) - if !sensorMode.Enabled { - sleepTime = 10 - } - - zombiecounter += 1 - if zombiecounter*sleepTime > workerTimeout { - go zombiecheck(ctx, workerTimeout, sensorMode) - zombiecounter = 0 - } - time.Sleep(time.Duration(sleepTime) * time.Second) - continue - } - - if hasStarted && len(executionRequests.Data) > 0 { - //log.Printf("[INFO] Body: %s", string(body)) - // Type string `json:"type"` - } - - // do things on behalf of backend - var toBeRemoved shuffle.ExecutionRequestWrapper - if len(executionRequests.Data) > 0 { - newrequests := []shuffle.ExecutionRequest{} - - // Deduplicating in case same job shows up multiple times - // This is specifically to handle data pipelines better - deduplicatedJobs := []shuffle.ExecutionRequest{} - for _, incRequest := range executionRequests.Data { - if !strings.Contains(incRequest.Type, "DOCKER") && !strings.Contains(incRequest.Type, "PIPELINE") && !strings.Contains(incRequest.Type, "SIGMA") && !strings.Contains(incRequest.Type, "TENZIR") { - deduplicatedJobs = append(deduplicatedJobs, incRequest) - continue - } - - found := false - for _, dedupRequest := range deduplicatedJobs { - if incRequest.ExecutionArgument == dedupRequest.ExecutionArgument && incRequest.Type == dedupRequest.Type { - found = true - break - } - } - - if found { - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - continue - } - - deduplicatedJobs = append(deduplicatedJobs, incRequest) - } - - executionRequests.Data = deduplicatedJobs - for _, incRequest := range executionRequests.Data { - - // Handles sensormode. ELSE handles Docker/K8s etc. - if sensorMode.Enabled { - - if len(incRequest.ExecutionSource) > 0 || len(incRequest.ExecutionArgument) == 0 { - parsedHostname := incRequest.ExecutionSource - if strings.Contains(parsedHostname, ".") { - parsedHostnameSplit := strings.Split(parsedHostname, ".") - parsedHostname = strings.ToUpper(parsedHostnameSplit[0]) - } - - if parsedHostname == hostname { - if debug { - log.Printf("[DEBUG] CORRECT HOSTNAME: %#v matches sensor hostname %#v. Removing from queue without processing.", parsedHostname, hostname) - } - - if sensorMode.ResponseActions != "" { - go shuffle.HandleSensorResponseAction(sensorMode, incRequest) - } - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - } else { - // Just ignore as other machines will handle it. - //log.Printf("[WARNING] Hostname '%s' from job does not match sensor hostname '%s'. Removing from queue without processing. Job: %#v", parsedHostname, hostname, incRequest) - } - } else { - // Invalid command - if debug { - log.Printf("[DEBUG] Removing invalid sensor command from queue: %#v", incRequest) - } - - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - } - - } else { - // Looking for specific jobs - if incRequest.Type == "PIPELINE_CREATE" || incRequest.Type == "PIPELINE_START" || incRequest.Type == "PIPELINE_STOP" || incRequest.Type == "PIPELINE_DELETE" || incRequest.Type == "PIPELINE_UPDATE" { - log.Printf("[INFO] Handling pipeline request from backend: '%s' with argument '%s'", incRequest.Type, incRequest.ExecutionArgument) - - os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") - os.Setenv("SHUFFLE_PIPELINE_ENABLED", "true") - tenzirDisabled = false - - // Running NEW or editing pipelines - err := handlePipeline(incRequest) - if err != nil { - log.Printf("[ERROR] Failed handling pipeline ('%s' '%s'): %s. Deleting job anyway.", incRequest.Type, incRequest.ExecutionSource, err) - } - - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - } else if incRequest.Type == "DOCKER_IMAGE_DOWNLOAD" { - log.Printf("[INFO] Re-downloading new image(s) due to backend request: %#v", incRequest.ExecutionArgument) - - if len(incRequest.ExecutionArgument) > 0 { - go handleBackendImageDownload(ctx, incRequest.ExecutionArgument) - } else { - log.Printf("[ERROR] No image name provided for download. Removing job from queue.") - } - - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - - } else if incRequest.Type == "CATEGORY_UPDATE" { - os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") - - tenzirDisabled = false - err = handleFileCategoryChange("sigma") - if err != nil { - log.Printf("[ERROR] Failed to download the file category: %s", err) - } - - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - - } else if incRequest.Type == "DISABLE_SIGMA_FOLDER" { - log.Printf("[INFO] Got job to disable sigma rules") - - err = removeFileCategory("sigma") - if err != nil { - log.Printf("[ERROR] Failed to disable the sigma rules: %s", err) - } - - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - - } else if incRequest.Type == "DISABLE_SIGMA_FILE" { - fileName := incRequest.ExecutionArgument - log.Printf("[INFO] Got job to disable sigma file %s", fileName) - - err = disableRule(fileName) - if err != nil { - log.Printf("[ERROR] Failed to disable the sigma file %s, reason: %s", fileName, err) - } - - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - - } else if incRequest.Type == "ENABLE_SIGMA_FILE" { - fileName := incRequest.ExecutionArgument - log.Printf("[INFO] Got job to enable sigma file %s", fileName) - - err = enableRule(fileName) - if err != nil { - log.Printf("[ERROR] Failed to disable the sigma file %s, reason: %s", fileName, err) - } - - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - } else if incRequest.Type == "START_TENZIR" { - log.Printf("[INFO] Got job to start tenzir") - - // Manual command = overrides to allow starting of Tenzir from the frontend anyway. - //os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") - tenzirDisabled = false - - // Removed either way - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - - err := deployTenzirNode() - if err != nil { - if strings.Contains(fmt.Sprintf("%s", err), "node available") { - // Disabling until UI is updated - //os.Setenv("SHUFFLE_SKIP_PIPELINES", "true") - //tenzirDisabled = true - - log.Printf("[ERROR] Failed to start tenzir, reason: %s", err) - err = shuffle.CreateOrgNotification( - ctx, - fmt.Sprintf("Failed to start Tenzir: %s", err), - fmt.Sprintf("Tenzir failed to start due to: %s", err), - fmt.Sprintf("/detections/Sigma"), - org, - true, - "LOW", - "TENZIR_START", - ) - - if err != nil { - log.Printf("[ERROR] Failed to send notification: %s", err) - return - } - } - } - - } else { - if debug { - log.Printf("[DEBUG] Passing execution ID request to normal queue: %#v", incRequest.ExecutionId) - } - - newrequests = append(newrequests, incRequest) - } - } - } - - if len(toBeRemoved.Data) > 0 { - err = sendRemoveRequest(client, toBeRemoved, baseUrl, environment, auth, org, sleepTime) - if err != nil { - log.Printf("[ERROR] Failed sending remove request: %s", err) - } else { - toBeRemoved.Data = []shuffle.ExecutionRequest{} - } - } - - // Remove the download image request - executionRequests.Data = newrequests - } - - // Skipping throttling with swarm - if sensorMode.Enabled { - // Pass - } else if swarmConfig != "run" && swarmConfig != "swarm" { - if len(executionRequests.Data) == 0 { - zombiecounter += 1 - if zombiecounter*sleepTime > workerTimeout { - go zombiecheck(ctx, workerTimeout, sensorMode) - zombiecounter = 0 - } - time.Sleep(time.Duration(sleepTime) * time.Second) - continue - } - - // Anything below here verifies concurrency - executionCount = getRunningWorkers(ctx, workerTimeout) - if executionCount >= maxConcurrency { - if zombiecounter*sleepTime > workerTimeout { - go zombiecheck(ctx, workerTimeout, sensorMode) - zombiecounter = 0 - } - time.Sleep(time.Duration(sleepTime) * time.Second) - continue - } - - allowed := maxConcurrency - executionCount - if len(executionRequests.Data) > allowed { - log.Printf("[WARNING] Throttle - Cutting down requests from %d to %d (MAX: %d, CUR: %d)", len(executionRequests.Data), allowed, maxConcurrency, executionCount) - executionRequests.Data = executionRequests.Data[0:allowed] - } - } else if swarmControlMode && (swarmConfig == "run" || swarmConfig == "swarm") { - // any reason it is not maxConcurrency instead of - // hardcoded 50? - if len(executionRequests.Data) > 50 { - executionRequests.Data = executionRequests.Data[0:50] - } - - if swarmRequestsMade > 100 && time.Since(swarmPollingTime).Seconds() > 5 { - log.Printf("[DEBUG] Swarm requests made: %d", swarmRequestsMade) - time.Sleep(time.Duration(sleepTime) * time.Second) - - swarmPollingTime = time.Now() - swarmRequestsMade = 0 - } - - swarmRequestsMade += len(executionRequests.Data) - } - - // New, abortable version. Should check executionid and remove everything else - for _, execution := range executionRequests.Data { - if len(execution.ExecutionArgument) > 0 { - log.Printf("[INFO] Argument: %s", execution.ExecutionArgument) - } - - if execution.Type == "schedule" { - log.Printf("[INFO] Schedule type! Weird deployment. Type: %s", execution.Type) - continue - } - - if len(execution.ExecutionId) == 0 { - log.Printf("[WARNING] Execution ID is empty: %#v", execution) - continue - } - - if execution.Status == "ABORT" || execution.Status == "FAILED" { - log.Printf("[INFO][%s] Executionstatus issue: ", execution.ExecutionId, execution.Status) - } - - if shuffle.ArrayContains(executionIds, execution.ExecutionId) { - log.Printf("[INFO][%s] Execution already handled (rerunning old execution)", execution.ExecutionId) - toBeRemoved.Data = append(toBeRemoved.Data, execution) - - // Should check when last this was ran, and if it's more than 10 minutes ago and it's not finished, we should run it again? - /* - if swarmConfig != "run" && swarmConfig != "swarm" { - continue - } - */ - } - - // Now, how do I execute this one? - containerName := fmt.Sprintf("worker-%s", execution.ExecutionId) - env := []string{ - fmt.Sprintf("AUTHORIZATION=%s", execution.Authorization), - fmt.Sprintf("EXECUTIONID=%s", execution.ExecutionId), - fmt.Sprintf("ENVIRONMENT_NAME=%s", environment), - fmt.Sprintf("BASE_URL=%s", baseUrl), - fmt.Sprintf("CLEANUP=%s", cleanupEnv), - fmt.Sprintf("TZ=%s", timezone), - fmt.Sprintf("SHUFFLE_PASS_APP_PROXY=%s", os.Getenv("SHUFFLE_PASS_APP_PROXY")), - fmt.Sprintf("SHUFFLE_SWARM_CONFIG=%s", os.Getenv("SHUFFLE_SWARM_CONFIG")), - fmt.Sprintf("SHUFFLE_LOGS_DISABLED=%s", os.Getenv("SHUFFLE_LOGS_DISABLED")), - fmt.Sprintf("SHUFFLE_BASE_IMAGE_NAME=%s", os.Getenv("SHUFFLE_BASE_IMAGE_NAME")), - fmt.Sprintf("SHUFFLE_ALLOW_PACKAGE_INSTAL=%s", os.Getenv("SHUFFLE_ALLOW_PACKAGE_INSTALL")), - } - - //log.Printf("Running worker with proxy? %s", os.Getenv("SHUFFLE_PASS_WORKER_PROXY")) - if strings.ToLower(os.Getenv("SHUFFLE_PASS_WORKER_PROXY")) == "true" { - env = append(env, fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY"))) - env = append(env, fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY"))) - env = append(env, fmt.Sprintf("NO_PROXY=%s", os.Getenv("NO_PROXY"))) - } - - if dockerApiVersion != "" { - env = append(env, fmt.Sprintf("DOCKER_API_VERSION=%s", dockerApiVersion)) - } - - if len(os.Getenv("DOCKER_HOST")) > 0 { - env = append(env, fmt.Sprintf("DOCKER_HOST=%s", os.Getenv("DOCKER_HOST"))) - } - - if len(os.Getenv("SHUFFLE_MEMCACHED")) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_MEMCACHED=%s", os.Getenv("SHUFFLE_MEMCACHED"))) - } - - if len(os.Getenv("SHUFFLE_CLOUDRUN_URL")) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_CLOUDRUN_URL=%s", os.Getenv("SHUFFLE_CLOUDRUN_URL"))) - } - - if len(os.Getenv("SHUFFLE_SKIPSSL_VERIFY")) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_SKIPSSL_VERIFY=%s", os.Getenv("SHUFFLE_SKIPSSL_VERIFY"))) - } - - if len(os.Getenv("SHUFFLE_DEBUG_MEMORY")) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_DEBUG_MEMORY=%s", os.Getenv("SHUFFLE_DEBUG_MEMORY"))) - } - - // Look for volume binds - if len(os.Getenv("SHUFFLE_VOLUME_BINDS")) > 0 { - //log.Printf("[DEBUG] Added volume binds: %s", os.Getenv("SHUFFLE_VOLUME_BINDS")) - env = append(env, fmt.Sprintf("SHUFFLE_VOLUME_BINDS=%s", os.Getenv("SHUFFLE_VOLUME_BINDS"))) - } - - if len(os.Getenv("SHUFFLE_APP_SDK_TIMEOUT")) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_APP_SDK_TIMEOUT=%s", os.Getenv("SHUFFLE_APP_SDK_TIMEOUT"))) - } - - // Setting up internal proxy config for Shuffle -> shuffle comms - overrideHttpProxy := os.Getenv("SHUFFLE_INTERNAL_HTTP_PROXY") - overrideHttpsProxy := os.Getenv("SHUFFLE_INTERNAL_HTTPS_PROXY") - if len(overrideHttpProxy) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_INTERNAL_HTTP_PROXY=%s", overrideHttpProxy)) - } - - if len(overrideHttpsProxy) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_INTERNAL_HTTPS_PROXY=%s", overrideHttpsProxy)) - } - - if len(os.Getenv("SHUFFLE_MAX_SWARM_NODES")) > 0 { - env = append(env, fmt.Sprintf("SHUFFLE_MAX_SWARM_NODES=%s", os.Getenv("SHUFFLE_MAX_SWARM_NODES"))) - } - - err = deployWorker(workerImage, containerName, env, execution) - zombiecounter += 1 - if err == nil { - //log.Printf("[DEBUG] ExecutionID %s was deployed and to be removed from queue.", execution.ExecutionId) - toBeRemoved.Data = append(toBeRemoved.Data, execution) - executionIds = append(executionIds, execution.ExecutionId) - } else { - log.Printf("[WARNING][%s] Failed to deploy: %s", execution.ExecutionId, err) - - if strings.Contains(err.Error(), "already exists") { - toBeRemoved.Data = append(toBeRemoved.Data, execution) - executionIds = append(executionIds, execution.ExecutionId) - } else if strings.Contains(err.Error(), "No such image") { - // Download the image - - if isKubernetes == "true" { - log.Printf("[DEBUG] Skipping image pull of '%s' because Kubernetes does it in realtime instead", workerImage) - } else { - log.Printf("[DEBUG] Re-pulling image %s as it doesn't exist, and is necessary for worker to run (autofix)", workerImage) - pullOptions := image.PullOptions{} - _, err = dockercli.ImagePull(ctx, workerImage, pullOptions) - if err != nil { - log.Printf("[ERROR] Failed to pull image %s: %s", workerImage, err) - } - } - } - } - } - - // Removes handled workflows (worker is made) - //log.Printf("\n\n[INFO] Removing %d executions from queue\n\n", len(toBeRemoved.Data)) - if len(toBeRemoved.Data) > 0 { - - err = sendRemoveRequest(client, toBeRemoved, baseUrl, environment, auth, org, sleepTime) - if err != nil { - log.Printf("[ERROR] Failed to remove executions from queue: %s", err) - } - - } - time.Sleep(time.Duration(sleepTime) * time.Second) - } -} - -// Tenzir command samples -// docker pull ghcr.io/dominiklohmann/tenzir-arm64:latest -// docker tag ghcr.io/dominiklohmann/tenzir-arm64:latest tenzir/tenzir:latest - -// Read from Cache and send it to a webhook -// docker run tenzir/tenzir:latest 'from http://192.168.86.44:5002/api/v1/orgs/7e9b9007-5df2-4b47-bca5-c4d267ef2943/cache/CIDR%20ranges?type=text&authorization=cec9d01f-09b2-4419-8a0a-76c6046e3fef read lines | to http://192.168.86.44:5002/api/v1/hooks/webhook_665ace5f-f27b-496a-a365-6e07eb61078c write lines' -func handlePipeline(incRequest shuffle.ExecutionRequest) error { - - log.Printf("[INFO] Pipeline: '%s' with source '%s'", incRequest.Type, incRequest.ExecutionSource) - - err := deployTenzirNode() - if err != nil { - log.Printf("[ERROR] Failed to deploy the pipeline, reason: %s", err) - return err - } - - // no need of execution arguments for STOP and DELETE - if (incRequest.Type != "PIPELINE_STOP" && incRequest.Type != "PIPELINE_DELETE") && len(incRequest.ExecutionArgument) == 0 { - log.Printf("[ERROR] No execution argument found for pipeline type %s. Skipping", incRequest.Type) - - return errors.New("no execution argument found for pipeline create. Skipping") - } - - identifier := strings.ToLower(strings.ReplaceAll(incRequest.ExecutionSource, " ", "-")) - if !strings.HasPrefix(strings.ToLower(incRequest.ExecutionSource), "shuffle") { - identifier = fmt.Sprintf("shuffle-%s", strings.ToLower(strings.ReplaceAll(incRequest.ExecutionSource, " ", "-"))) - } - - command := incRequest.ExecutionArgument - pipelines = []shuffle.PipelineInfo{} - if incRequest.Type == "PIPELINE_CREATE" { - log.Printf("[INFO] Should delete -> recreate new pipeline with id %#v", identifier) - //err := deployPipeline(image, identifier, command) - _, err := createPipeline(command, identifier) - if err != nil { - log.Printf("[ERROR] Failed to create pipeline: %s", err) - return err - } - } else if incRequest.Type == "PIPELINE_DELETE" || incRequest.Type == "PIPELINE_STOP" { - pipelineId := incRequest.ExecutionId - log.Printf("[INFO] Should delete pipeline %#v. PipelineID: %s", identifier, pipelineId) - //pipelineId, err := searchPipeline(identifier) - //if err != nil { - //} - - err = deletePipeline(pipelineId) - if err != nil { - log.Printf("[ERROR] Failed Deleting Pipeline %s", err) - return err - } - - /* - } else if incRequest.Type == "PIPELINE_STOP" { - log.Printf("[INFO] Should stop the pipeline %#v", identifier) - pipelineId, err := searchPipeline(identifier) - if err != nil { - log.Printf("[ERROR] Failed searching for Pipeline with name %s reason:%s ", identifier, err) - toBeRemoved.Data = append(toBeRemoved.Data, incRequest) - return err - } - _, err = updatePipelineState(command, pipelineId, "stop") - if err != nil { - log.Printf("[ERROR] Failed to stop Pipeline: %s reason:%s ", pipelineId, err) - return err - } else { - log.Printf("[INFO] Successfully stopped the Pipeline: %s", pipelineId) - } - */ - - } else if incRequest.Type == "PIPELINE_START" { - log.Printf("[INFO] Should start the pipeline %#v", identifier) - pipelineId, err := searchPipeline(identifier) - if err != nil { - if err.Error() == "no existing pipeline found with name" { - log.Printf("[INFO] Starting a new pipeline with command '%s' and identifier '%s'", command, identifier) - var createErr error - pipelineId, createErr = createPipeline(command, identifier) - if createErr != nil { - return createErr - } - } else { - log.Printf("[ERROR] Failed searching for Pipeline with name %s reason:%s ", identifier, err) - return err - } - } - - log.Printf("[INFO] Starting existing pipeline with ID %s", pipelineId) - _, err = updatePipelineState(command, pipelineId, "start") - if err != nil { - log.Printf("[ERROR] Failed to start Pipeline: %s reason:%s ", pipelineId, err) - return err - } else { - log.Printf("[INFO] Successfully started pipeline: %s", pipelineId) - } - - } else { - log.Printf("[ERROR] Unknown type for pipeline: %s", incRequest.Type) - return errors.New("unknown type for pipeline") - } - - return nil -} - -func deployTenzirNode() error { - // Specifically for standalone tenzir - if os.Getenv("SHUFFLE_PIPELINE_STANDALONE") == "true" { - return nil - } - - // Disabled all pipeline features - if os.Getenv("SHUFFLE_SKIP_PIPELINES") == "true" { - return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES (1)") - } - - if isKubernetes == "true" { - return errors.New("Tenzir not implemented for k8s") - } - - err := checkTenzirNode() - if err == nil { - return nil - } - - ctx := context.Background() - cacheKey := "tenzir-key" - _, err = shuffle.GetCache(ctx, cacheKey) - if err == nil { - return nil - } - - //imageName := "frikky/shuffle:tenzir" - imageName := "tenzir/tenzir:main" - if os.Getenv("TENZIR_IMAGE_NAME") != "" { - imageName = os.Getenv("TENZIR_IMAGE_NAME") - log.Printf("[INFO] Using custom Tenzir image name: %s", imageName) - } - - containerName := "tenzir-node" - containerStartOptions := container.StartOptions{} - - containerInfo, err := dockercli.ContainerInspect(ctx, containerName) - if err != nil { - if dockerclient.IsErrNotFound(err) { - // Create network if it doesn't exist - networkName := "tenzir-network" - networkSubnet := "192.168.102.0/24" - networkGateway := "192.168.102.1" - - err = createNetworkIfNotExists(ctx, networkName, networkSubnet, networkGateway) - if err != nil { - log.Printf("[ERROR] Failed to create network %s: %s", networkName, err) - //return err - } - - // Trying to connect orborus to the tenzir network as well - err = dockercli.NetworkConnect(ctx, networkName, containerId, nil) - if err != nil { - log.Printf("[ERROR] Error connecting tenzir container to network: %s", err) - } - - // Check if image exists - _, _, err := dockercli.ImageInspectWithRaw(ctx, imageName) - if dockerclient.IsErrNotFound(err) { - log.Printf("[DEBUG] Pulling image %s. This may take a while.", imageName) - pullOptions := image.PullOptions{} - out, err := dockercli.ImagePull(ctx, imageName, pullOptions) - if err != nil { - log.Printf("[ERROR] Failed to pull the Tenzir image: %s", err) - return err - } - defer out.Close() - - io.Copy(io.Discard, out) - } else if err != nil { - return err - } - - err = createAndStartTenzirNode(ctx, containerName, imageName, containerStartOptions) - if err != nil { - return err - } - } else { - return err - } - } else { - if !containerInfo.State.Running { - log.Printf("[DEBUG] Tenzir Node exists, but is not running. Restarting it.") - err := dockercli.ContainerStart(ctx, containerName, containerStartOptions) - if err != nil { - log.Printf("[ERROR] Failed to start Tenzir Node container: %v", err) - return err - } - - time.Sleep(10 * time.Second) - log.Printf("[INFO] Waiting for Tenzir to become available ...") - err = checkTenzirNode() - if err != nil { - return err - } - } - } - - tenzirStatus := struct { - ContainerStatus string `json:"container_status"` - }{ - ContainerStatus: "running", - } - - cacheData, err := json.Marshal(tenzirStatus) - if err != nil { - log.Printf("[WARNING] Failed marshalling execution: %s", err) - } - - err = shuffle.SetCache(ctx, cacheKey, cacheData, 1) - if err != nil { - log.Printf("[WARNING] Failed updating cache for tenzir: %s", err) - } - - return nil -} - -func createAndStartTenzirNode(ctx context.Context, containerName, imageName string, containerStartOptions container.StartOptions) error { - healthconfig := &container.HealthConfig{ - Test: []string{"tenzir --connection-timeout=30s --connection-retry-delay=1s 'api /ping'"}, - Interval: 30 * time.Second, - Retries: 1, - } - - // Ensure restart policy is there - config := &container.Config{ - Hostname: containerName, - Cmd: []string{"--commands=web server --mode=dev --bind=0.0.0.0"}, - Image: imageName, - Healthcheck: healthconfig, - ExposedPorts: nat.PortSet{ - "5160/tcp": struct{}{}, - "1514/udp": struct{}{}, - "1514/tcp": struct{}{}, - }, - Entrypoint: []string{containerName}, - Env: []string{}, - } - - tenzirApikey := os.Getenv("TENZIR_PLUGINS__PLATFORM__API_KEY") - tenzirControlEndpoint := os.Getenv("TENZIR_PLUGINS__PLATFORM__CONTROL_ENDPOINT") - tenzirPluginsPlatform := os.Getenv("TENZIR_PLUGINS__PLATFORM__TENANT_ID") - - anyFound := false - if len(tenzirApikey) > 0 { - config.Env = append(config.Env, fmt.Sprintf("TENZIR_PLUGINS__PLATFORM__API_KEY=%s", tenzirApikey)) - anyFound = true - } - - if len(tenzirControlEndpoint) > 0 { - config.Env = append(config.Env, fmt.Sprintf("TENZIR_PLUGINS__PLATFORM__CONTROL_ENDPOINT=%s", tenzirControlEndpoint)) - anyFound = true - } - - if len(tenzirPluginsPlatform) > 0 { - config.Env = append(config.Env, fmt.Sprintf("TENZIR_PLUGINS__PLATFORM__TENANT_ID=%s", tenzirPluginsPlatform)) - anyFound = true - } - - tenzirStorageFolder := os.Getenv("SHUFFLE_STORAGE_FOLDER") - if len(tenzirStorageFolder) > 0 { - tenzirStorageFolder = tenzirStorageFolder - - if !strings.HasSuffix(tenzirStorageFolder, "/") { - tenzirStorageFolder = tenzirStorageFolder + "/" - } - } else { - tenzirStorageFolder = "/tmp/" - log.Printf("[DEBUG] Using base folder %s for Tenzir storage. Change it using environment variable SHUFFLE_STORAGE_FOLDER=/filepath/", tenzirStorageFolder) - } - - if !anyFound { - //log.Printf("[DEBUG] No Tenzir Plugin environment variables found.") - } else { - //log.Printf("[DEBUG] Attempting Tenzir connection with app.tenzir.com tenant '%s'", tenzirPluginsPlatform) - } - - hostConfig := &container.HostConfig{ - PortBindings: nat.PortMap{ - "1514/tcp": []nat.PortBinding{{HostPort: "1514"}}, - "1514/udp": []nat.PortBinding{{HostPort: "1514"}}, - "5160/tcp": []nat.PortBinding{{HostPort: "5160"}}, - }, - Mounts: []mount.Mount{ - { - Type: "bind", - Source: tenzirStorageFolder, - Target: "/tmp", - }, - /* - { - Type: "bind", - Source: tenzirStorageFolder, - Target: "/var/log/tenzir/", - }, - { - Type: "bind", - Source: tenzirStorageFolder, - Target: "/var/cache/tenzir/", - }, - */ - }, - VolumeDriver: "local", - RestartPolicy: container.RestartPolicy{ - Name: "always", - }, - } - - if os.Getenv("SHUFFLE_DISABLE_SYSLOG") == "true" { - hostConfig.PortBindings = nat.PortMap{ - "5160/tcp": []nat.PortBinding{{HostPort: "5160"}}, - } - } - - if skipPipelineMount { - hostConfig.Mounts = []mount.Mount{} - } - - //networkingConfig := &network.NetworkingConfig{ - // EndpointsConfig: map[string]*network.EndpointSettings{ - // "tenzir-network": { - // IPAMConfig: &network.EndpointIPAMConfig{ - // IPv4Address: "192.168.102.100", - // }, - // }, - // }, - //} - - networkingConfig := &network.NetworkingConfig{ - EndpointsConfig: map[string]*network.EndpointSettings{ - "tenzir-network": { - IPAMConfig: nil, - Aliases: []string{"tenzir-node"}, - }, - }, - } - - // FIXME: Is this necessary? Seems to screw up networking: - // conflicting options: hostname and the network mode - /* - if isKubernetes != "true" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" { - hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) - } - */ - - resp, err := dockercli.ContainerCreate(ctx, config, hostConfig, networkingConfig, nil, containerName) - if err != nil { - if strings.Contains(err.Error(), "path does not exist") { - log.Printf("[ERROR] Not using permanent pipeline storage as storage folder %s does not exist. If you want permanent storage, create the %s folder then restart Orborus (1). Raw: %s", tenzirStorageFolder, tenzirStorageFolder, err) - skipPipelineMount = true - } else { - log.Printf("[ERROR] Failed to create Tenzir Node container: %v", err) - } - - return err - } - - if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" { - networkName := "shuffle_swarm_executions" - err = dockercli.NetworkConnect(ctx, networkName, resp.ID, nil) - if err != nil { - log.Printf("[ERROR] Error connecting tenzir container to network: %s", err) - } - } - - err = dockercli.ContainerStart(ctx, containerName, containerStartOptions) - if err != nil { - if strings.Contains(err.Error(), "path does not exist") { - log.Printf("[ERROR] Not using permanent pipeline storage as storage folder %s does not exist. If you want permanent storage, create the %s folder then restart Orborus (2). Raw: %s", tenzirStorageFolder, tenzirStorageFolder, err) - skipPipelineMount = true - } else { - log.Printf("[ERROR] Failed to START Tenzir Node container: %v", err) - } - - return err - } - - log.Printf("[INFO] Tenzir Node container started successfully. Waiting for it to become available..") - time.Sleep(20 * time.Second) - err = checkTenzirNode() - if err != nil { - log.Printf("[ERROR] Tenzir connection not available: %s. IF the URL seems wrong, set SHUFFLE_PIPELINE_URL=http://:5160", err) - return err - } - - log.Printf("[INFO] Successfully deployed Tenzir Node! Setting up default syslog listener on TCP/1514 AND UDP/1514") - - command := `load_tcp "0.0.0.0:1514" { read_syslog } | import` - _, err = createPipeline(command, "default-syslog-tcp-514") - if err != nil { - log.Printf("[ERROR] Failed to create tcp syslog pipeline: %s", err) - return nil - } - - command = `load_udp "0.0.0.0:1514", insert_newlines=true | read_syslog | import` - _, err = createPipeline(command, "default-syslog-udp-514") - if err != nil { - log.Printf("[ERROR] Failed to create udp syslog pipeline: %s", err) - return nil - } - - return nil -} - -func createNetworkIfNotExists(ctx context.Context, networkName, subnet, gateway string) error { - listOptions := network.ListOptions{} - networks, err := dockercli.NetworkList(ctx, listOptions) - if err != nil { - return err - } - - for _, network := range networks { - if network.Name == networkName { - // Network exists - return nil - } - } - - ipamConfig := &network.IPAM{ - Config: []network.IPAMConfig{ - { - Subnet: subnet, - Gateway: gateway, - }, - }, - } - - networkCreate := network.CreateOptions{ - //CheckDuplicate: true, - Driver: "bridge", - IPAM: ipamConfig, - } - - _, err = dockercli.NetworkCreate(ctx, networkName, networkCreate) - if err != nil { - return err - } - - return nil -} - -func checkTenzirNode() error { - if tenzirDisabled && os.Getenv("SHUFFLE_SKIP_PIPELINES") == "true" && os.Getenv("SHUFFLE_PIPELINE_ENABLED") == "false" { - return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES (2)") - } - - url := fmt.Sprintf("%s/api/v0/ping", pipelineUrl) - forwardMethod := "POST" - - client := http.Client{ - Timeout: 1 * time.Second, - } - req, err := http.NewRequest(forwardMethod, url, nil) - if err != nil { - log.Printf("[ERROR] Failed to create HTTP request: %s", err) - return err - } - - resp, err := client.Do(req) - if err == nil && resp.StatusCode == http.StatusOK { - return nil - } - - return fmt.Errorf("Tenzir node is not available due to: %s", err) -} - -func createPipeline(command, identifier string) (string, error) { - - //toBeDeleted := false - /* - // Pre-checked. No point here - pipelineId, err := searchPipeline(identifier) - if err != nil { - return "", err - } - */ - - url := fmt.Sprintf("%s/api/v0/pipeline/create", pipelineUrl) - forwardMethod := "POST" - - /* - if err != nil { - if strings.Contains(fmt.Sprintf("%s", err), "no existing pipeline found") { - log.Printf("[INFO] No existing pipeline found with id: %s. Creating a new one!", identifier) - } else { - log.Printf("[ERROR] Failed to search for existing pipeline but continuing anyway : %s", err) - } - } else { - log.Printf("[INFO] an existing pipeline found with ID: %s. it will be deleted", pipelineId) - toBeDeleted = true - } - */ - - // if strings.Contains(command, "shuffler.io") { - - // } else { - // var scheme string - // if strings.Contains(command, "http://") { - // scheme = "http://" - // } else if strings.Contains(command, "https://") { - // scheme = "https://" - // } - - // startIndex := strings.Index(command, scheme) - // if startIndex != -1 { - // endIndex := startIndex + len(scheme) - // endIndex += strings.Index(command[endIndex:], "/") - - // command = command[:startIndex] + baseUrl + command[endIndex:] - // } - // } - - //command = "from file /var/lib/tenzir/sysmon_logs.ndjson read json | sigma /var/lib/tenzir/rule.yaml" - //command = "from file /var/lib/tenzir/sysmon_logs.ndjson read json | import" - - // Make sure to escape them - //if strings.Contains(command, "/") { - // command = strings.ReplaceAll("\\\"", "", command) - // command = strings.ReplaceAll(command, "\"", "") - //} - - requestBody := map[string]interface{}{ - "definition": command, - "name": identifier, - "hidden": false, - "retry_delay": "500.0ms", - "unstoppable": true, - } - - requestBodyJSON, err := json.Marshal(requestBody) - if err != nil { - log.Printf("[ERROR] failed marshalling body: %s", err) - return "", err - } - - forwardData := bytes.NewBuffer(requestBodyJSON) - req, err := http.NewRequest( - forwardMethod, - url, - forwardData, - ) - - if err != nil { - log.Printf("[ERROR] Failed to create HTTP request: %s", err) - return "", err - } - - req.Header.Set("Content-Type", "application/json") - client := &http.Client{Timeout: 10 * time.Second} - resp, err := client.Do(req) - if err != nil { - log.Printf("[ERROR] Failed to send HTTP request: %s", err) - return "", err - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - log.Printf("[ERROR] Failed reading response body: %s", err) - return "", err - } - - if strings.Contains(string(body), "error") { - log.Printf("[ERROR] Pipeline creation error resp (%d): %s", resp.StatusCode, string(body)) - } else { - log.Printf("[DEBUG] Pipeline creation debug (%d): %s", resp.StatusCode, string(body)) - } - - defer resp.Body.Close() - if resp.StatusCode != 200 { - log.Printf("[DEBUG] status code is %d instead of 200", resp.StatusCode) - return "", fmt.Errorf("got the status code %d instead of 200", resp.StatusCode) - } - - type PipelineResponse struct { - ID string `json:"id"` - Message string `json:"message"` - Severity string `json:"severity"` - } - - var response PipelineResponse - if err := json.Unmarshal(body, &response); err != nil { - log.Printf("[ERROR] Failed unmarshalling response: %s", err) - return "", err - } - - if response.ID == "" { - log.Printf("[ERROR] ID not found or empty in response. Severity: %#v, Message: %#v", response.Severity, response.Message) - return "", errors.New("Pipeline ID not found or empty in the response. See error logs.") - } - - return response.ID, nil -} - -func updatePipelineState(command, pipelineId, action string) (string, error) { - - url := fmt.Sprintf("%s/api/v0/pipeline/update", pipelineUrl) - forwardMethod := "POST" - requestBody := map[string]interface{}{ - "id": pipelineId, - "action": action, - - /* - "autostart": map[string]bool{ - "created": true, - "completed": false, - "failed": false, - }, - "autodelete": map[string]bool{ - "completed": false, - "failed": false, - "stopped": false, - }, - */ - } - - requestBodyJSON, err := json.Marshal(requestBody) - if err != nil { - return "", err - } - - log.Printf("[INFO] Updating pipeline %s with action %s to ensure it starts. Body: %s", pipelineId, action, string(requestBodyJSON)) - - forwardData := bytes.NewBuffer(requestBodyJSON) - req, err := http.NewRequest( - forwardMethod, - url, - forwardData, - ) - if err != nil { - log.Printf("[ERROR] Failed to update HTTP request: %s", err) - return "", err - } - - req.Header.Set("Content-Type", "application/json") - client := &http.Client{Timeout: 10 * time.Second} - resp, err := client.Do(req) - if err != nil { - log.Printf("[ERROR] Failed to send HTTP request: %s", err) - return "", err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("got the status code %d instead of 200", resp.StatusCode) - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return "", err - } - - var responseData struct { - Pipeline struct { - State string `json:"state"` - } `json:"pipeline"` - } - if err := json.Unmarshal(body, &responseData); err != nil { - return "", err - } - - return responseData.Pipeline.State, nil -} - -func deletePipeline(pipelineId string) error { - requestBody := map[string]string{ - "id": pipelineId, - } - - url := fmt.Sprintf("%s/api/v0/pipeline/delete", pipelineUrl) - forwardMethod := "POST" - - requestBodyJSON, err := json.Marshal(requestBody) - if err != nil { - log.Println("[ERROR] failed marshalling request body:", err) - return err - } - - forwardData := bytes.NewBuffer(requestBodyJSON) - - req, err := http.NewRequest( - forwardMethod, - url, - forwardData, - ) - if err != nil { - log.Printf("[ERROR] Failed to delete HTTP request: %s", err) - return err - } - req.Header.Set("Content-Type", "application/json") - - client := &http.Client{Timeout: 10 * time.Second} - resp, err := client.Do(req) - if err != nil { - log.Printf("[ERROR] Failed to send HTTP request: %s", err) - return err - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - log.Printf("[DEBUG] The deletion of pipeline with ID: %s is unsucessful as status code is NOT 200 !!!", pipelineId) - return fmt.Errorf("got the status code %d instead of 200", resp.StatusCode) - } - - log.Printf("[INFO] Pipeline with ID: %s deleted successfully", pipelineId) - - pipelines = []shuffle.PipelineInfo{} - return nil -} - -// Lists the pipelines from the API exactly as they are. Definition is set up in Shuffle structs -func listPipelines() ([]shuffle.PipelineInfo, error) { - responseData := shuffle.PipelineInfoWrapper{} - - if tenzirDisabled { - return responseData.Pipelines, errors.New("Tenzir is disabled") - } - - var reqBody []byte - url := fmt.Sprintf("%s/api/v0/pipeline/list", pipelineUrl) - client := http.Client{ - Timeout: 2 * time.Second, - } - - req, err := http.NewRequest( - "POST", - url, - bytes.NewBuffer(reqBody), - ) - - if err != nil { - return responseData.Pipelines, err - } - - req.Header.Set("Content-Type", "application/json") - resp, err := client.Do(req) - if err != nil { - return responseData.Pipelines, err - } - - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return responseData.Pipelines, fmt.Errorf("Got the status code %d instead of 200 from Pipeline node", resp.StatusCode) - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return responseData.Pipelines, err - } - - if err := json.Unmarshal(body, &responseData); err != nil { - return responseData.Pipelines, err - } - - return responseData.Pipelines, nil -} - -func searchPipeline(identifier string) (string, error) { - allPipelines, err := listPipelines() - if err != nil { - return "", err - } - - for _, pipeline := range allPipelines { - if pipeline.Name == identifier { - return pipeline.ID, nil - } - } - - return "", errors.New("no existing pipeline found with name") -} - -func handleFileCategoryChange(ruleType string) error { - apiEndpoint := fmt.Sprintf("%s/api/v1/files/namespaces/%s", baseUrl, ruleType) - req, err := http.NewRequest("GET", apiEndpoint, nil) - if err != nil { - return err - } - - if len(pipelineApikey) == 0 { - //var auth = os.Getenv("AUTH") - //var org = os.Getenv("ORG") - - if len(auth) > 0 && len(org) > 0 { - pipelineApikey = auth - } else { - return errors.New("Shuffle API-key not set for Pipelines: SHUFFLE_PIPELINE_AUTH=") - } - } - - req.Header.Add("Authorization", "Bearer "+pipelineApikey) - if len(org) > 0 { - req.Header.Add("Org-Id", org) - } - - client := shuffle.GetExternalClient(apiEndpoint) - resp, err := client.Do(req) - if err != nil { - return err - } - - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("Received non-200 response '%d' from backend URL %s. ", resp.StatusCode, apiEndpoint) - } - - out, err := os.Create("files.zip") - if err != nil { - return err - } - - defer out.Close() - defer os.Remove("files.zip") - _, err = io.Copy(out, resp.Body) - if err != nil { - log.Printf("[ERROR] Failed to io.Copy ZIP file content: %s", err) - return err - } - - //log.Println("[DEBUG] ZIP file downloaded successfully.") - tenzirStorageFolder := os.Getenv("SHUFFLE_STORAGE_FOLDER") - if len(tenzirStorageFolder) == 0 { - tenzirStorageFolder = "/tmp/" - } - - tenzirStorageFolder = strings.TrimRight(tenzirStorageFolder, "/") - sigmaPath := fmt.Sprintf("%s/%s_rules", tenzirStorageFolder, ruleType) - err = extractZIP("files.zip", sigmaPath) - if err != nil { - log.Printf("[ERROR] Failed to extract ZIP file: %s", err) - return err - } - - log.Printf("[DEBUG] Detection files copied to '%s' successfully.", sigmaPath) - - return nil -} - -func extractZIP(zipFile, destDir string) error { - r, err := zip.OpenReader(zipFile) - if err != nil { - return err - } - - // FInd size of the zip - var totalSize uint64 - for _, f := range r.File { - totalSize += f.UncompressedSize64 - } - - log.Printf("[DEBUG] Total size of the ZIP file: %d bytes", totalSize) - defer r.Close() - if err := os.MkdirAll(destDir, 0755); err != nil { - return err - } - - log.Printf("[DEBUG] Total files to extract: %d", len(r.File)) - for _, f := range r.File { - // Fix path traversal - if strings.Contains(f.Name, "..") { - return fmt.Errorf("illegal file name: %s", f.Name) - } - - err := extractFile(f, destDir) - if err != nil { - return err - } - } - - return nil -} - -func extractFile(f *zip.File, destDir string) error { - rc, err := f.Open() - if err != nil { - return err - } - - defer rc.Close() - path := filepath.Join(destDir, f.Name) - out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) - if err != nil { - return err - } - - defer out.Close() - _, err = io.Copy(out, rc) - return err -} - -func copyToTenzir(srcPath, destPath string) error { - containerName := "tenzir-node" - - checkCmd := exec.Command("docker", "exec", containerName, "test", "-d", destPath) - if err := checkCmd.Run(); err == nil { - rmCmd := exec.Command("docker", "exec", "-u", "root", containerName, "rm", "-rf", destPath) - if err := rmCmd.Run(); err != nil { - return fmt.Errorf("error removing existing directory in container: %v", err) - } - } - - cpCmd := exec.Command("docker", "cp", srcPath, fmt.Sprintf("%s:%s", containerName, destPath)) - var out bytes.Buffer - cpCmd.Stdout = &out - cpCmd.Stderr = &out - - err := cpCmd.Run() - if err != nil { - return fmt.Errorf("error copying files: %v, output: %s", err, out.String()) - } - - return nil -} - -func removeFileCategory(ruleType string) error { - tenzirStorageFolder := os.Getenv("SHUFFLE_STORAGE_FOLDER") - if len(tenzirStorageFolder) == 0 { - tenzirStorageFolder = "/tmp/" - } - - tenzirStorageFolder = strings.TrimRight(tenzirStorageFolder, "/") - - //sigmaPath := "/var/lib/tenzir/sigma_rules/*" - rulePath := fmt.Sprintf("%s/%s_rules", tenzirStorageFolder, ruleType) - - err := os.RemoveAll(rulePath) - if err != nil { - return fmt.Errorf("Error removing category files in %s: %v", rulePath, err) - } - - log.Printf("[INFO] Removed all local category data in %s", rulePath) - - return nil -} - -// curl https://get.tenzir.app | sh -func removeFile(fileName string) error { - containerName := "tenzir-node" - srcPath := fmt.Sprintf("/var/lib/tenzir/sigma_rules/%s", fileName) - - checkSrcCmd := exec.Command("docker", "exec", containerName, "sh", "-c", fmt.Sprintf("test -f %s", srcPath)) - if err := checkSrcCmd.Run(); err != nil { - // If the file does not exist, simply return nil - if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 1 { - log.Printf("[ERROR] No such file: %s, nothing to delete\n", srcPath) - return nil - } - return fmt.Errorf("error checking source file: %v", err) - } - - return removePath(containerName, srcPath) -} - -func removePath(containerName, path string) error { - // rmCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("rm -rf %s", path)) - rmCmd := exec.Command("docker", "exec", "-u", "root", containerName, "rm", "-rf", path) - output, err := rmCmd.CombinedOutput() - if err != nil { - return fmt.Errorf("error removing path: %v, output: %s", err, output) - } - return nil -} - -func sendPipelineHealthStatus(sensorMode shuffle.SensorMode) (shuffle.LakeConfig, error) { - if sensorMode.Enabled { - return shuffle.LakeConfig{}, nil - } - - pipelinePayload := shuffle.LakeConfig{ - Enabled: false, - Pipelines: []shuffle.PipelineInfo{}, - } - - if tenzirDisabled { - return pipelinePayload, nil - } - - // To not spam down the list API too much - randint := rand.Intn(5) - if len(pipelines) == 0 || randint == 0 { - pipelineDef, err := listPipelines() - - if err == nil || len(pipelines) > 0 { - pipelines = pipelineDef - pipelinePayload.Pipelines = pipelines - } - } else { - pipelinePayload.Pipelines = pipelines - } - - err := deployTenzirNode() - if err != nil { - if (!strings.Contains(err.Error(), "SHUFFLE_SKIP_PIPELINES") && !strings.Contains(err.Error(), "Kubernetes not implemented for Tenzir node")) && !strings.Contains(err.Error(), "Tenzir Node is already running") && !strings.Contains(err.Error(), "docker daemon") { - log.Printf("[ERROR] Tenzir node connection problem: %s", err) - - } else { - //tenzirDisabled = true - if debug { - log.Printf("[WARNING] Disabling pipelines: %s. You will need to restart the Orborus to fix this.", err) - } - - } - - return pipelinePayload, err - } - - pipelinePayload.Enabled = true - - // No direct sending. - return pipelinePayload, nil -} - -func disableRule(fileName string) error { - containerName := "tenzir-node" - srcPath := fmt.Sprintf("/var/lib/tenzir/sigma_rules/%s", fileName) - destDir := "/var/lib/tenzir/disabled_rules" - destPath := fmt.Sprintf("%s/%s", destDir, fileName) - - // checkSrcCmd := exec.Command("docker", "exec", containerName, "sh", "-c", fmt.Sprintf("test -f %s", srcPath)) - checkSrcCmd := exec.Command("docker", "exec", containerName, "test", "-f", srcPath) - if err := checkSrcCmd.Run(); err != nil { - if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 1 { - fmt.Printf("File does not exist: %s\n", srcPath) - return nil // Nothing to disable - } - return fmt.Errorf("error checking source file: %v", err) - } - - // checkDestDirCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mkdir -p %s", destDir)) - checkDestDirCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mkdir", "-p", "--", destDir) - if err := checkDestDirCmd.Run(); err != nil { - return fmt.Errorf("error ensuring destination directory exists: %v", err) - } - - // moveCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mv %s %s", srcPath, destPath)) - moveCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mv", "--", srcPath, destPath) - if err := moveCmd.Run(); err != nil { - return fmt.Errorf("error moving file: %v", err) - } - - fmt.Printf("File %s moved to %s successfully.\n", fileName, destDir) - return nil -} - -func enableRule(fileName string) error { - containerName := "tenzir-node" - srcPath := fmt.Sprintf("/var/lib/tenzir/disabled_rules/%s", fileName) - destDir := "/var/lib/tenzir/sigma_rules" - destPath := fmt.Sprintf("%s/%s", destDir, fileName) - - // checkSrcCmd := exec.Command("docker", "exec", containerName, "sh", "-c", fmt.Sprintf("test -f %s", srcPath)) - checkSrcCmd := exec.Command("docker", "exec", containerName, "test", "-f", srcPath) - if err := checkSrcCmd.Run(); err != nil { - if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 1 { - fmt.Printf("File does not exist: %s\n", srcPath) - return nil // Nothing to enable - } - return fmt.Errorf("error checking source file: %v", err) - } - - // checkDestDirCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mkdir -p %s", destDir)) - checkDestDirCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mkdir", "-p", "--", destDir) - if err := checkDestDirCmd.Run(); err != nil { - return fmt.Errorf("error ensuring destination directory exists: %v", err) - } - // moveCmd := exec.Command("docker", "exec", "-u", "root", containerName, "sh", "-c", fmt.Sprintf("mv %s %s", srcPath, destPath)) - moveCmd := exec.Command("docker", "exec", "-u", "root", containerName, "mv", "--", srcPath, destPath) - if err := moveCmd.Run(); err != nil { - return fmt.Errorf("error moving file: %v", err) - } - - fmt.Printf("[DEBUG] File %s moved to %s successfully.\n", fileName, destDir) - return nil -} - -// Is this ok to do with Docker? idk :) -func getRunningWorkers(ctx context.Context, workerTimeout int) int { - //log.Printf("[DEBUG] Getting running workers with API version %s", dockerApiVersion) - counter := 0 - if isKubernetes == "true" { - log.Printf("[INFO] Getting running workers in kubernetes") - - thresholdTime := time.Now().Add(time.Duration(-workerTimeout) * time.Second) - - clientset, _, err := shuffle.GetKubernetesClient() - if err != nil { - log.Printf("[ERROR] Failed getting kubernetes client: %s", err) - return 0 - } - - pods, podErr := clientset.CoreV1().Pods(kubernetesNamespace).List(ctx, metav1.ListOptions{ - LabelSelector: "app.kubernetes.io/name=shuffle-worker", - }) - if podErr != nil { - log.Printf("[ERROR] Failed getting running workers: %s", podErr) - return 0 - } - - for _, pod := range pods.Items { - if pod.Status.Phase == "Running" && pod.CreationTimestamp.Time.After(thresholdTime) { - counter++ - } - } - - if counter > 0 { - log.Printf("[INFO] Found %d running workers in Orborus", counter) - } - } else { - - containers, err := dockercli.ContainerList(ctx, container.ListOptions{ - All: true, - }) - - // Automatically updates the version - if err != nil { - log.Printf("[ERROR] Error getting containers from Docker: %s", err) - - newVersionSplit := strings.Split(fmt.Sprintf("%s", err), "version is") - if len(newVersionSplit) > 1 { - //dockerApiVersion = strings.TrimSpace(newVersionSplit[1]) - log.Printf("[DEBUG] WANT to change the API version to default to %s?", strings.TrimSpace(newVersionSplit[1])) - } - - return maxConcurrency - } - - currenttime := time.Now().Unix() - - for _, container := range containers { - // Skip random containers. Only handle things related to Shuffle. - if !strings.Contains(container.Image, baseimagename) { - shuffleFound := false - for _, item := range container.Labels { - if item == "shuffle" { - shuffleFound = true - break - } - } - - // Check image name - if !shuffleFound { - continue - } - //} else { - // log.Printf("NAME: %s", container.Image) - } - - for _, name := range container.Names { - // FIXME - add name_version_uid_uid regex check as well - if !strings.HasPrefix(name, "/worker") { - continue - } - - //log.Printf("Time: %d - %d", currenttime-container.Created, int64(workerTimeout)) - if container.State == "running" && currenttime-container.Created < int64(workerTimeout) { - counter += 1 - break - } - } - } - } - return counter -} - -// FIXME - add this to remove exited workers -// Should it check what happened to the execution? idk -func zombiecheck(ctx context.Context, workerTimeout int, sensorMode shuffle.SensorMode) error { - if sensorMode.Enabled { - return nil - } - - isK8s := isKubernetes == "true" - - executionIds = []string{} - if swarmConfig == "run" || swarmConfig == "swarm" || isK8s { - //log.Printf("[DEBUG] Skipping Zombie check due to new execution model (swarm)") - return nil - } - - log.Println("[INFO] Looking for old containers to remove") - containers, err := dockercli.ContainerList(ctx, container.ListOptions{ - All: true, - }) - - if err != nil { - log.Printf("[ERROR] Failed creating Containerlist: %s", err) - return err - } - - containerNames := map[string]string{} - stopContainers := []string{} - removeContainers := []string{} - log.Printf("[INFO] Baseimage: %s, Workertimeout: %d", baseimagename, int64(workerTimeout)) - //baseString := `/bin/sh -c 'python app.py --log-level DEBUG'` - baseString := `python app.py` - for _, container := range containers { - // Skip random containers. Only handle things related to Shuffle. - if !strings.Contains(container.Image, baseimagename) && !strings.Contains(container.Command, baseString) && !strings.Contains(container.Command, "walkoff") && container.Command != "./worker" { - shuffleFound := false - for _, item := range container.Labels { - if item == "shuffle" { - shuffleFound = true - break - } - } - - // Check image name - if !shuffleFound { - //log.Printf("[DEBUG] Zombie container skip: %#v, %s", container.Labels, container.Image) - continue - } - //} else { - // log.Printf("NAME: %s", container.Image) - } else { - //log.Printf("Img: %s", container.Image) - //log.Printf("Names: %s", container.Names) - } - - for _, name := range container.Names { - // FIXME - add name_version_uid_uid regex check as well - if strings.HasPrefix(name, "/shuffle") && !strings.HasPrefix(name, "/shuffle-subflow") { - continue - } - - currenttime := time.Now().Unix() - //log.Printf("[INFO] (%s) NAME: %s. TIME: %d", container.State, name, currenttime-container.Created) - - // Need to check time here too because a container can be removed the same instant as its created - if container.State != "running" && currenttime-container.Created > int64(workerTimeout) { - removeContainers = append(removeContainers, container.ID) - containerNames[container.ID] = name - } - - // stopcontainer & removecontainer - //log.Printf("Time: %d - %d", currenttime-container.Created, int64(workerTimeout)) - if container.State == "running" && currenttime-container.Created > int64(workerTimeout) { - stopContainers = append(stopContainers, container.ID) - containerNames[container.ID] = name - } - } - } - - // FIXME - add killing of apps with same execution ID too - log.Printf("[INFO] Should STOP and remove %d containers.", len(stopContainers)) - var options container.StopOptions - for _, containername := range stopContainers { - log.Printf("[INFO] Stopping and removing container %s", containerNames[containername]) - go dockercli.ContainerStop(ctx, containername, options) - removeContainers = append(removeContainers, containername) - } - - removeOptions := container.RemoveOptions{ - RemoveVolumes: true, - Force: true, - } - - log.Printf("[INFO] Should REMOVE %d containers.", len(removeContainers)) - for _, containername := range removeContainers { - dockercli.ContainerRemove(ctx, containername, removeOptions) - } - - return nil -} - -func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest, image string, env []string) error { - parsedRequest := shuffle.OrborusExecutionRequest{ - ExecutionId: workflowExecution.ExecutionId, - Authorization: workflowExecution.Authorization, - BaseUrl: os.Getenv("BASE_URL"), - EnvironmentName: os.Getenv("ENVIRONMENT_NAME"), - Timezone: os.Getenv("TZ"), - Cleanup: os.Getenv("CLEANUP"), - HTTPProxy: os.Getenv("HTTP_PROXY"), - HTTPSProxy: os.Getenv("HTTPS_PROXY"), - ShufflePassProxyToApp: os.Getenv("SHUFFLE_PASS_APP_PROXY"), - WorkerServerUrl: os.Getenv("SHUFFLE_WORKER_SERVER_URL"), - } - - parsedBaseurl := baseUrl - if strings.Contains(baseUrl, ":") { - baseUrlSplit := strings.Split(baseUrl, ":") - if len(baseUrlSplit) >= 3 { - parsedBaseurl = strings.Join(baseUrlSplit[0:2], ":") - } - } - - data, err := json.Marshal(parsedRequest) - if err != nil { - log.Printf("[ERROR] Failed marshalling worker request: %s", err) - return err - } - - streamUrl := fmt.Sprintf("http://shuffle-workers:33333/api/v1/execute") - if containerId == "" || containerId == "shuffle-orborus" { - streamUrl = fmt.Sprintf("%s:33333/api/v1/execute", parsedBaseurl) - } - - identifier := "shuffle-workers" - if isKubernetes == "true" { - // FIXME: Do we need this to map the cluster? - //if shuffle.IsRunningInCluster() { - //log.Printf("[INFO] Running in Kubernetes cluster") - // try getting the k8s worker server url - //} - } - - if strings.Contains(streamUrl, "shuffler.io") || strings.Contains(streamUrl, "localhost") || strings.Contains(streamUrl, "127.0.0.1") || strings.Contains(streamUrl, "shuffle-backend") { - - // Specific to debugging - if len(workerServerUrl) == 0 { - if debug { - log.Printf("[INFO] Using default worker server url as previous is invalid: %s. Swapping to shuffle-workers:33333", streamUrl) - } - } - - streamUrl = fmt.Sprintf("http://shuffle-workers:33333/api/v1/execute") - } - - if len(workerServerUrl) > 0 { - // Check if a port is supplied or not - if strings.Contains(workerServerUrl, "/api/v1/execute") { - streamUrl = workerServerUrl - } else { - streamUrl = fmt.Sprintf("%s/api/v1/execute", workerServerUrl) - if !strings.Contains(workerServerUrl, ":") { - streamUrl = fmt.Sprintf("%s:33333/api/v1/execute", workerServerUrl) - } - } - } - - client := &http.Client{ - //Transport: &http.Transport{ - // TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - //}, - Timeout: time.Duration(120 * time.Second), - } - - if debug { - log.Printf("[DEBUG][%s] Worker request to be sent to URL: %s", workflowExecution.ExecutionId, streamUrl) - } - - req, err := http.NewRequest( - "POST", - streamUrl, - bytes.NewBuffer([]byte(data)), - ) - - if err != nil { - log.Printf("[ERROR] Failed creating worker request: %s", err) - if strings.Contains(fmt.Sprintf("%s", err), "connection refused") || strings.Contains(fmt.Sprintf("%s", err), "EOF") { - workerImage := fmt.Sprintf("ghcr.io/shuffle/shuffle-worker:%s", workerVersion) - if len(newWorkerImage) > 0 { - workerImage = newWorkerImage - } - - if isKubernetes == "true" { - deployK8sWorker(workerImage, identifier, env) - } else { - deployServiceWorkers(workerImage) - } - - time.Sleep(time.Duration(10) * time.Second) - //err = sendWorkerRequest(executionRequest) - } - - return err - } - - newresp, err := client.Do(req) - if err != nil { - // Connection refused? - if !strings.Contains(fmt.Sprintf("%s", err), "timeout") { - log.Printf("[ERROR][%s] Error running worker request to %s (1): %s", workflowExecution.ExecutionId, streamUrl, err) - } - - if strings.Contains(fmt.Sprintf("%s", err), "connection refused") || strings.Contains(fmt.Sprintf("%s", err), "EOF") { - workerImage := fmt.Sprintf("ghcr.io/shuffle/shuffle-worker:%s", workerVersion) - if len(newWorkerImage) > 0 { - workerImage = newWorkerImage - } - - if isKubernetes == "true" { - deployK8sWorker(workerImage, identifier, env) - } else { - deployServiceWorkers(workerImage) - } - - time.Sleep(time.Duration(10) * time.Second) - //err = sendWorkerRequest(executionRequest) - } - - return err - } - - defer newresp.Body.Close() - body, err := ioutil.ReadAll(newresp.Body) - if err != nil { - log.Printf("[ERROR] Failed reading body in worker request body to worker on %s: %s", streamUrl, err) - return err - } - window.AddEvent(time.Now()) - - if newresp.StatusCode != 200 { - log.Printf("[WARNING] POTENTIAL error running worker request (2) - status code is %d for %s, not 200. Body: %s", newresp.StatusCode, streamUrl, string(body)) - - // In case of old executions - if strings.Contains(strings.ToLower(string(body)), "bad status ") { - return nil - } - - if strings.Contains(strings.ToLower(string(body)), "no apps to handle") { - return nil - } - - return errors.New(fmt.Sprintf("Bad statuscode from worker: %d - expecting 200", newresp.StatusCode)) - } - - _ = body - - debugCommand := fmt.Sprintf("docker service logs shuffle-workers 2>&1 -f | grep %s", workflowExecution.ExecutionId) - if isKubernetes == "true" { - debugCommand = fmt.Sprintf("kubectl logs -n %s deployment/shuffle-workers | grep %s", kubernetesNamespace, workflowExecution.ExecutionId) - } - - log.Printf("[DEBUG][%s] Ran worker from requests. Worker URL: %s. DEBUGGING:\n%s", workflowExecution.ExecutionId, streamUrl, debugCommand) - return nil -} - -// 0x0elliot: -// let's never increase worker replicas. -// in our tests, workers replicas mattered a lot less. -// edge-case: subflows are helped with when worker replicas are higher. -func AutoScale(ctx context.Context) { - if os.Getenv("SHUFFLE_SCALE_REPLICAS") != "" { - return - } - - ticker := time.NewTicker(1 * time.Second) - coolDownPeriod := 10 * time.Second - queuePerMinuteInt = 20 - if os.Getenv("SHUFFLE_QUEUE_PER_MINUTE") != "" { - var err error - queuePerMinuteInt, err = strconv.Atoi(os.Getenv("SHUFFLE_QUEUE_PER_MINUTE")) - if err != nil { - log.Printf("[WARNING] Cannot convert %s to int. Using default value for it: %d", queuePerMinute, queuePerMinuteInt) - } - } - - lastScaleTime := time.Now() - currentWorkers := currentWokerCount(ctx, dockercli) - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - if time.Since(lastScaleTime) < (coolDownPeriod) { - continue - } - currentRequestCount := window.CountEvents(time.Now()) - requiredReplicas := 0 - if currentRequestCount >= queuePerMinuteInt*currentWorkers { - // FIXME: Hardcoded Max Replicas should be 6 - requiredReplicas = int(math.Min(float64(6), float64(currentRequestCount/queuePerMinuteInt)+1)) - } - - if requiredReplicas > 0 { - err := scaleService(ctx, dockercli, uint64(requiredReplicas)) - if err != nil { - log.Printf("[ERROR] Failed to scale service: %s", err) - } else { - lastScaleTime = time.Now() - currentWorkers = currentWokerCount(ctx, dockercli) - } - } - } - } -} - -func scaleService(ctx context.Context, client *dockerclient.Client, replicas uint64) error { - service, _, err := client.ServiceInspectWithRaw(ctx, "shuffle-workers", types.ServiceInspectOptions{}) - if err != nil { - return err - } - - if service.Spec.Mode.Replicated == nil { - return errors.New("Service cannot be replicated") - } - - if *service.Spec.Mode.Replicated.Replicas >= replicas { - return nil - } - - service.Spec.Mode.Replicated.Replicas = &replicas - - _, err = dockercli.ServiceUpdate(ctx, service.ID, service.Version, service.Spec, types.ServiceUpdateOptions{}) - if err != nil { - return err - } - - log.Printf("[INFO] Scaled shuffle-worker to %d replicas", replicas) - return nil -} - -func currentWokerCount(ctx context.Context, client *dockerclient.Client) int { - service, _, err := client.ServiceInspectWithRaw(ctx, "shuffle-workers", types.ServiceInspectOptions{}) - if err != nil { - return 0 - } - - if service.Spec.Mode.Replicated == nil { - return 0 - } - - return int(*service.Spec.Mode.Replicated.Replicas) -} - -func queueScaleFactor(numQueue int, queuePerMin int) float64 { - if numQueue > queuePerMin { - queuePressure := float64(numQueue) / float64(queuePerMin) - return 1.0 + math.Min(queuePressure-1.0, 1.0) - } - - return 1.0 -} - -func checkMemcached(ctx context.Context, dockercli *dockerclient.Client) (bool, error) { - containerName := "shuffle-cache" - continer, err := dockercli.ContainerInspect(context.Background(), containerName) - if err != nil { - if dockerclient.IsErrNotFound(err) { - return false, nil - } - return false, err - } - networkName := "shuffle_swarm_executions" - err = dockercli.NetworkConnect(ctx, networkName, containerName, nil) - if err != nil { - log.Printf("[WARNING] Failed connecting memcached container to network: %s", err) - } - - if continer.State.Running == false { - log.Printf("[INFO] Container %s exists but is not running. Attempting to start it.", containerName) - err = dockercli.ContainerStart(ctx, containerName, container.StartOptions{}) - if err != nil { - log.Printf("[ERROR] Failed to start container %s: %v", containerName, err) - return false, err - } - log.Printf("[INFO] Successfully started container %s.", containerName) - return true, nil - } - - return continer.State.Running, nil -} - -func deployMemcached(dockercli *dockerclient.Client) error { - if os.Getenv("SHUFFLE_MEMCACHED") != "" { - return errors.New("Memcached already running") - } - - defaultMem := "1024" - log.Printf("[INFO] Spanning a default memcached container to handle the distribution between cache across different workers. Default memory assigned %s", defaultMem) - - ctx := context.Background() - - memcachedImage := "docker.io/library/memcached:latest" - containerConfig := &container.Config{ - Image: memcachedImage, - Cmd: []string{"-m", defaultMem}, - } - - hostConfig := &container.HostConfig{ - PortBindings: nat.PortMap{ - "11211/tcp": []nat.PortBinding{{HostPort: "11211"}}, - }, - } - - _, _, err := dockercli.ImageInspectWithRaw(ctx, memcachedImage) - if dockerclient.IsErrNotFound(err) { - log.Printf("[DEBUG] Pulling image %s. This may take a while.", memcachedImage) - pullOptions := image.PullOptions{} - out, err := dockercli.ImagePull(ctx, memcachedImage, pullOptions) - if err != nil { - log.Printf("[ERROR] Failed to pull the memcached image: %s", err) - return err - } - defer out.Close() - - io.Copy(io.Discard, out) - } else if err != nil { - return err - } - - containerName := "shuffle-cache" - resp, err := dockercli.ContainerCreate(ctx, containerConfig, hostConfig, nil, nil, containerName) - if err != nil { - log.Printf("[ERROR] Error spanning memcached continer: %s", err) - return err - } - - if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" { - networkName := "shuffle_swarm_executions" - err = dockercli.NetworkConnect(ctx, networkName, resp.ID, nil) - if err != nil { - log.Printf("[ERROR] Error connecting tenzir container to network: %s", err) - } - } - - err = dockercli.ContainerStart(ctx, resp.ID, container.StartOptions{}) - if err != nil { - log.Printf("[ERROR] Error starting memcached continer: %s", err) - return err - } - - networkName := "shuffle_swarm_executions" - err = dockercli.NetworkConnect(ctx, networkName, resp.ID, nil) - if err != nil { - log.Printf("[ERROR] Error connecting memcached container to network: %s", err) - } - - log.Printf("[INFO] Memcached container started successfully at port 11211") - - return nil -} - -// How do we get the cpu usage? maybe just get the number of requests (much more useful for apps) -/* -func nodesResourceUsage(ctx context.Context, client *dockerclient.Client) error { - nodes, err := client.NodeList(ctx, types.NodeListOptions{}) - if err != nil { - return err - } - for _, node := range nodes { - res := node.Description.Resources - } - - return nil -} -*/ - -/* -func numberOfReplicas(ctx context.Context, queueLength int, config shuffle.ScalingConfig) (int, int) { - queueScaleFactor := queueScaleFactor(queueLength, config) - numReplicas := int(float64(queueLength) * queueScaleFactor) - serviceName := "shuffle-workers" - nodes, err := dockercli.NodeList(ctx, types.NodeListOptions{}) - if err != nil { - log.Printf("[ERROR] Cannot find any nodes in the swarm network") - } - - filterArgs := filters.NewArgs() - filterArgs.Add("service", serviceName) - filterArgs.Add("desired-state", "running") - - tasks, err := dockercli.TaskList(context.Background(), types.TaskListOptions{ - Filters: filterArgs, - }) - if err != nil { - log.Fatalf("[WARNING] Failed to list tasks for service %s: %s", serviceName, err) - } - - runningReplicas := len(tasks) - if numReplicas > runningReplicas*len(nodes) { - maxIncrease := config.MaxScaleUpStep - if numReplicas > (runningReplicas*len(nodes) + maxIncrease) { - numReplicas = runningReplicas + maxIncrease - } - } - - if numReplicas < config.MinReplicas { - numReplicas = config.MinReplicas - } - if numReplicas > config.MaxReplicas { - numReplicas = config.MaxReplicas - } - - return numReplicas, runningReplicas -} -*/ - -// TODO: Currently we use number of request made for the worker to run a execution as it is much -// easier to track in a window time frame. But this could be useful. -func collectMetrics(ctx context.Context, dockerClient *dockerclient.Client) (int, error) { - client := shuffle.GetExternalClient(baseUrl) - fullUrl := fmt.Sprintf("%s/api/v1/workflows/queue", baseUrl) - req, err := http.NewRequest("GET", fullUrl, nil) - if err != nil { - log.Printf("[ERROR] Failed to send a request to %s: %s", fullUrl, err) - return 0, err - } - - req.Header.Add("Content-Type", "application/json") - req.Header.Add("Org-Id", environment) - if len(auth) > 0 { - req.Header.Add("Authorization", auth) - } - - if len(org) > 0 { - req.Header.Add("Org", org) - } - - if len(orborusLabel) > 0 { - log.Printf("[DEBUG] Sending with Label '%s'", orborusLabel) - req.Header.Add("X-Orborus-Label", orborusLabel) - } - - if swarmConfig != "run" && swarmConfig != "swarm" { - req.Header.Add("X-Orborus-Runmode", "Default") - } else { - req.Header.Add("X-Orborus-Runmode", "Docker Swarm") - } - - resp, err := client.Do(req) - if err != nil { - return 0, err - } - - var executionRequests shuffle.ExecutionRequestWrapper - body, err := ioutil.ReadAll(resp.Body) - - json.Unmarshal(body, &executionRequests) - - return len(executionRequests.Data), nil -} - -func setBackendToSwarmNetwork(ctx context.Context) error { - containerId := "" - filterArgs := filters.NewArgs() - filterArgs.Add("name", "shuffle-backend") - - containers, err := dockercli.ContainerList(ctx, container.ListOptions{ - All: true, - Filters: filterArgs, - }) - if err != nil { - return err - } - if len(containers) == 0 { - return errors.New("No containers found with name shuffle-backend") - } - - containerId = containers[0].ID - networkName := "shuffle_swarm_executions" - err = dockercli.NetworkConnect(ctx, networkName, containerId, nil) - if err != nil { - log.Printf("[ERROR] Error connecting backend container to network: %s", err) - } - - return nil -} diff --git a/functions/onprem/orborus/orborus.yaml b/functions/onprem/orborus/orborus.yaml deleted file mode 100644 index 0ff50ef7..00000000 --- a/functions/onprem/orborus/orborus.yaml +++ /dev/null @@ -1,82 +0,0 @@ ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - namespace: default - name: pod-manager -rules: -- apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list", "create", "update", "delete"] -- apiGroups: ["batch"] - resources: ["jobs"] - verbs: ["create", "get", "list", "watch", "delete"] - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: pod-manager-binding - namespace: default -subjects: -- kind: ServiceAccount - name: default - namespace: default -roleRef: - kind: Role - name: pod-manager - apiGroup: rbac.authorization.k8s.io - ---- - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: orborus - annotations: - kompose.cmd: kompose convert -f docker-compose.yml - kompose.version: 1.26.0 (40646f47) - labels: - io.kompose.service: orborus -spec: - replicas: 1 - selector: - matchLabels: - io.kompose.service: orborus - strategy: {} - template: - metadata: - annotations: - kompose.cmd: kompose convert -f docker-compose.yml - kompose.version: 1.26.0 (40646f47) - creationTimestamp: null - labels: - io.kompose.network/shuffle: "true" - io.kompose.service: orborus - spec: - dnsPolicy: "Default" - containers: - - env: - - name: BASE_URL - value: "https://shuffler.io" - - name: SHUFFLE_SCALE_REPLICAS - value: "7" - - name: IS_KUBERNETES - value: "true" - - name: ENVIRONMENT_NAME - value: "environment test" - - name: ORG - value: "9c938e5b-d812-40d9-92f0-93783f43ec0d" - - name: AUTH - value: "3663a270-bb3a-4678-a365-d879601a1a0c" - - name: SHUFFLE_WORKER_IMAGE - value: "ghcr.io/shuffle/shuffle-worker:nightly" - - image: ghcr.io/shuffle/shuffle-orborus:nightly - imagePullPolicy: Always - name: shuffle-orborus - resources: {} - hostname: shuffle-orborus - restartPolicy: Always diff --git a/functions/onprem/orborus/proxy_server.py b/functions/onprem/orborus/proxy_server.py deleted file mode 100644 index 8020616c..00000000 --- a/functions/onprem/orborus/proxy_server.py +++ /dev/null @@ -1,54 +0,0 @@ -# -# curl -H "Org-id: Shuffle" --proxy "http://192.168.86.45:8081" http://192.168.86.45:5001/api/v1/workflows/queue - -import SocketServer -import SimpleHTTPServer -import requests -import json - -PORT = 8082 - -class MyProxy(SimpleHTTPServer.SimpleHTTPRequestHandler): - def do_GET(self): - url=self.path[:] - allheaders = {} - for item in ("%s" % self.headers).split("\n"): - headersplit = item.split(":") - if len(headersplit) == 2: - allheaders[headersplit[0]] = (headersplit[1][:-1]).strip() - - ret = requests.get(url, headers=allheaders) - print("RESP (%s) - %d - %s" % (url, ret.status_code, ret.text)) - - self.send_response(ret.status_code) - self.end_headers() - self.wfile.write(ret.text) - - def do_POST(self): - url=self.path[:] - allheaders = {} - for item in ("%s" % self.headers).split("\n"): - headersplit = item.split(":") - if len(headersplit) == 2: - allheaders[headersplit[0]] = (headersplit[1][:-1]).strip() - - length = int(self.headers.getheader('content-length')) - try: - message = json.loads(self.rfile.read(length)) - print("Got message: %s" % message) - ret = requests.post(url, headers=allheaders, json=message) - except: - message = self.rfile.read(length) - print("Got message: %s" % message) - ret = requests.post(url, headers=allheaders, data=message) - - print("RESP (%s) - %d - %s" % (url, ret.status_code, ret.text)) - - self.send_response(ret.status_code) - self.end_headers() - self.wfile.write(ret.text) - -httpd = SocketServer.ForkingTCPServer(('', PORT), MyProxy) -print("Now serving at %d" % PORT) -httpd.serve_forever() - diff --git a/functions/onprem/orborus/run.sh b/functions/onprem/orborus/run.sh deleted file mode 100755 index d71ac59c..00000000 --- a/functions/onprem/orborus/run.sh +++ /dev/null @@ -1,19 +0,0 @@ -#docker run \ -# --env DOCKER_API_VERSION=1.40 \ -# --env ENVIRONMENT_NAME="Shuffle" \ -# --env BASE_URL="http://192.168.86.45:5001" \ -# --env HTTP_PROXY="http://192.168.86.45:8082" \ -# --env HTTPS_PROXY="https://192.168.86.45:8082" \ -# --env SHUFFLE_PASS_WORKER_PROXY=true \ -# --env SHUFFLE_PASS_APP_PROXY=true \ -# -v /var/run/docker.sock:/var/run/docker.sock \ -# ghcr.io/frikky/shuffle-orborus:nightly - -docker run \ - --env DOCKER_API_VERSION=1.40 \ - --env ENVIRONMENT_NAME="Another env" \ - --env ORG="2e7b6a08-b63b-4fc2-bd70-718091509db1" \ - --env AUTH="env auth" \ - --env BASE_URL="https://shuffler.io" \ - -v /var/run/docker.sock:/var/run/docker.sock \ - ghcr.io/frikky/shuffle-orborus:nightly From f1d39b427dcddf9835adc1aeb2843f74adba540e Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Thu, 16 Apr 2026 01:45:51 +0530 Subject: [PATCH 39/61] shuffle-shared bump --- backend/go-app/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 6f8bdfbf..67624b0d 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -26,7 +26,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v1.2.24 + github.com/shuffle/shuffle-shared v1.2.33 github.com/shuffle/singul v0.0.29 golang.org/x/crypto v0.45.0 google.golang.org/api v0.236.0 From 70d3dc3b6a0e9229e7e86cde5274c48f2c26d9d8 Mon Sep 17 00:00:00 2001 From: Lalit Deore Date: Thu, 16 Apr 2026 18:13:23 +0530 Subject: [PATCH 40/61] licensing changes --- frontend/src/components/Billing.jsx | 8 ++++---- frontend/src/components/CloudSyncTab.jsx | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/frontend/src/components/Billing.jsx b/frontend/src/components/Billing.jsx index 1a5d375d..0a4a03e4 100644 --- a/frontend/src/components/Billing.jsx +++ b/frontend/src/components/Billing.jsx @@ -81,7 +81,7 @@ const ProductionStatus = ({ selectedOrganization, userdata, isCloud, theme }) => } const themeMode = theme.palette.mode; - const workflowActive = selectedOrganization?.sync_features?.workflow_executions?.active; + const workflowActive = selectedOrganization?.sync_features?.app_executions?.active; const multiTenantActive = selectedOrganization?.sync_features?.multi_tenant?.active; const multiEnvActive = selectedOrganization?.sync_features?.multi_env?.active; const brandingActive = selectedOrganization?.sync_features?.branding?.active; @@ -104,9 +104,9 @@ const ProductionStatus = ({ selectedOrganization, userdata, isCloud, theme }) => const features = [ { icon: ZapIcon, - label: 'Workflow Executions', - licensed: `${selectedOrganization?.sync_features?.workflow_executions?.limit}/month limit`, - unlicensed: '10,000/month limit', + label: 'App Runs', + licensed: `${selectedOrganization?.sync_features?.app_executions?.limit}/month limit`, + unlicensed: '25,000/month limit', isActive: workflowActive, }, { diff --git a/frontend/src/components/CloudSyncTab.jsx b/frontend/src/components/CloudSyncTab.jsx index e02c7783..66bcdac4 100644 --- a/frontend/src/components/CloudSyncTab.jsx +++ b/frontend/src/components/CloudSyncTab.jsx @@ -777,6 +777,10 @@ const CloudSyncTab = (props) => { newname = "app runs" } + if (key === "onprem_app_executions" && userdata.support !== true && !isCloud) { + return null + } + const griditem = { primary: newkey, secondary: From 16615cac8bce16b7720f09005dd0a13b8fec56e7 Mon Sep 17 00:00:00 2001 From: Lalit Deore Date: Thu, 16 Apr 2026 19:05:45 +0530 Subject: [PATCH 41/61] fix - minor issue --- frontend/src/components/CloudSyncTab.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/CloudSyncTab.jsx b/frontend/src/components/CloudSyncTab.jsx index 66bcdac4..9d12168a 100644 --- a/frontend/src/components/CloudSyncTab.jsx +++ b/frontend/src/components/CloudSyncTab.jsx @@ -777,7 +777,7 @@ const CloudSyncTab = (props) => { newname = "app runs" } - if (key === "onprem_app_executions" && userdata.support !== true && !isCloud) { + if ((key === "onprem_app_executions" && userdata.support !== true) || !isCloud) { return null } From 12644241dbf6cd09fe8faceb2fc6a08e6704360a Mon Sep 17 00:00:00 2001 From: Lalit Deore Date: Mon, 20 Apr 2026 16:00:35 +0530 Subject: [PATCH 42/61] fix - ui bug --- frontend/src/components/CloudSyncTab.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/CloudSyncTab.jsx b/frontend/src/components/CloudSyncTab.jsx index 9d12168a..3c197677 100644 --- a/frontend/src/components/CloudSyncTab.jsx +++ b/frontend/src/components/CloudSyncTab.jsx @@ -777,7 +777,7 @@ const CloudSyncTab = (props) => { newname = "app runs" } - if ((key === "onprem_app_executions" && userdata.support !== true) || !isCloud) { + if (key === "onprem_app_executions" && userdata.support !== true) { return null } From e311afb892a53b9f0a7a089d9f3e2595895a6e9d Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Thu, 23 Apr 2026 16:19:44 +0530 Subject: [PATCH 43/61] chore: bumping shuffle-shared to 1.2.40 --- backend/go-app/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 67624b0d..41c4c2b7 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -26,7 +26,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v1.2.33 + github.com/shuffle/shuffle-shared v1.2.40 github.com/shuffle/singul v0.0.29 golang.org/x/crypto v0.45.0 google.golang.org/api v0.236.0 From c6ed73315125c4de9b268e4dca7688cd93bfba00 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Fri, 1 May 2026 13:11:15 +0530 Subject: [PATCH 44/61] stability improvement over subflows and workflow executions --- functions/onprem/worker/worker.go | 132 ++++++++++++++++++++++++++++-- 1 file changed, 127 insertions(+), 5 deletions(-) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 28ca9dc4..e40e356f 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -1581,6 +1581,11 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { return } + if workflowExecution.Status == "WAITING" { + log.Printf("[DEBUG][%s] Execution is WAITING. Skipping action dispatch until a new result updates state.", workflowExecution.ExecutionId) + return + } + startAction, extra, children, parents, visited, executed, nextActions, environments := shuffle.GetExecutionVariables(ctx, workflowExecution.ExecutionId) var dockercli *dockerclient.Client @@ -2280,11 +2285,6 @@ func handleSubflowPoller(ctx context.Context, workflowExecution shuffle.Workflow } } - if len(data) == 0 { - log.Printf("[WARNING] Stream result missing execution ID and authorization; injecting them from workflow execution") - data = fmt.Sprintf(`{"execution_id": "%s", "authorization": "%s"}`, workflowExecution.ExecutionId, workflowExecution.Authorization) - } - key := fmt.Sprintf("%s:%s", workflowExecution.ExecutionId, subflowId) cacheKey := fmt.Sprintf("workflowexecution_%s", workflowExecution.ExecutionId) usedCache := false @@ -2337,6 +2337,11 @@ func handleSubflowPoller(ctx context.Context, workflowExecution shuffle.Workflow return errors.New("Subflow status not found yet (cache)") } + if len(data) == 0 { + log.Printf("[WARNING] Stream result missing execution ID and authorization; injecting them from workflow execution") + data = fmt.Sprintf(`{"execution_id": "%s", "authorization": "%s"}`, workflowExecution.ExecutionId, workflowExecution.Authorization) + } + req, err := http.NewRequest( "POST", streamResultUrl, @@ -2601,6 +2606,112 @@ func arrayContains(visited []string, id string) bool { return found } +func isTerminalResultStatus(status string) bool { + return status == "SUCCESS" || status == "FINISHED" || status == "FAILURE" || status == "ABORTED" +} + +func hasWaitForResultParameter(params []shuffle.WorkflowAppActionParameter) bool { + for _, param := range params { + if param.Name == "check_result" && strings.ToLower(param.Value) == "true" { + return true + } + } + + return false +} + +func getSubflowBarrierActionIDs(workflowExecution shuffle.WorkflowExecution) map[string]struct{} { + actionIDs := map[string]struct{}{} + + for _, action := range workflowExecution.Workflow.Actions { + if action.AppName != "shuffle-subflow" && action.AppName != "shuffle-subflow-v2" && action.AppName != "Shuffle Workflow" { + continue + } + + if !hasWaitForResultParameter(action.Parameters) { + continue + } + + actionIDs[action.ID] = struct{}{} + } + + for _, trigger := range workflowExecution.Workflow.Triggers { + if trigger.AppName != "shuffle-subflow" && trigger.AppName != "shuffle-subflow-v2" && trigger.AppName != "Shuffle Workflow" { + continue + } + + if !hasWaitForResultParameter(trigger.Parameters) { + continue + } + + actionIDs[trigger.ID] = struct{}{} + } + + return actionIDs +} + +func getSubflowBarrierProgress(workflowExecution shuffle.WorkflowExecution) (int, int, int) { + barrierActions := getSubflowBarrierActionIDs(workflowExecution) + expected := len(barrierActions) + if expected == 0 { + return 0, 0, 0 + } + + terminalByAction := map[string]string{} + for _, result := range workflowExecution.Results { + if _, ok := barrierActions[result.Action.ID]; !ok { + continue + } + + if !isTerminalResultStatus(result.Status) { + continue + } + + terminalByAction[result.Action.ID] = result.Status + } + + completed := len(terminalByAction) + failed := 0 + for _, status := range terminalByAction { + if status == "FAILURE" || status == "ABORTED" { + failed += 1 + } + } + + return expected, completed, failed +} + +func enforceSubflowBarrier(workflowExecution *shuffle.WorkflowExecution) (int, int, int) { + if workflowExecution == nil { + return 0, 0, 0 + } + + if workflowExecution.Status == "FINISHED" || workflowExecution.Status == "FAILURE" || workflowExecution.Status == "ABORTED" { + return 0, 0, 0 + } + + expected, completed, failed := getSubflowBarrierProgress(*workflowExecution) + if expected == 0 { + return 0, 0, 0 + } + + if completed < expected { + workflowExecution.Status = "WAITING" + return expected, completed, failed + } + + if workflowExecution.Status == "WAITING" { + if failed > 0 { + workflowExecution.Status = "FAILURE" + workflowExecution.Result = fmt.Sprintf("Subflow barrier failed: %d of %d subflows failed or aborted", failed, expected) + } else { + workflowExecution.Status = "EXECUTING" + } + } + + return expected, completed, failed +} + func getResult(workflowExecution shuffle.WorkflowExecution, id string) shuffle.ActionResult { for _, actionResult := range workflowExecution.Results { if actionResult.Action.ID == id { @@ -2976,6 +3087,12 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl workflowExecution, dbSave, err := shuffle.ParsedExecutionResult(ctx, *workflowExecution, actionResult, true, 0) if err == nil { + expectedSubflows, completedSubflows, failedSubflows := enforceSubflowBarrier(workflowExecution) + if expectedSubflows > 0 { + dbSave = true + log.Printf("[DEBUG][%s] Subflow barrier progress: %d/%d completed (failed=%d). Status=%s", workflowExecution.ExecutionId, completedSubflows, expectedSubflows, failedSubflows, workflowExecution.Status) + } + if workflowExecution.Status != "EXECUTING" && workflowExecution.Status != "WAITING" { log.Printf("[WARNING][%s] Execution is not executing, but %s. Stopping Transaction update.", workflowExecution.ExecutionId, workflowExecution.Status) if resp != nil { @@ -3040,6 +3157,11 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl return } else { log.Printf("[DEBUG][%s] Successfully got ParsedExecution with %d results!", workflowExecution.ExecutionId, len(workflowExecution.Results)) + expectedSubflows, completedSubflows, failedSubflows := enforceSubflowBarrier(workflowExecution) + if expectedSubflows > 0 { + dbSave = true + log.Printf("[DEBUG][%s] Subflow barrier progress: %d/%d completed (failed=%d). Status=%s", workflowExecution.ExecutionId, completedSubflows, expectedSubflows, failedSubflows, workflowExecution.Status) + } } } else { log.Printf("[ERROR][%s] Failed execution of parsedexecution: %s", workflowExecution.ExecutionId, err) From fb4c1edb7bac279a40887b52ddf0029136e3fea1 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Fri, 1 May 2026 13:12:48 +0530 Subject: [PATCH 45/61] fix: ignore duplicate action results during execution updates --- backend/go-app/walkoff.go | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index b8abdf8d..e6167cb4 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -593,6 +593,15 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl return } + if hasEquivalentActionResult(*workflowExecution, actionResult) { + log.Printf("[DEBUG][%s] Ignoring duplicate action result for action %s with status %s", workflowExecutionId, actionResult.Action.ID, actionResult.Status) + if resp != nil { + resp.WriteHeader(http.StatusOK) + resp.Write([]byte(`{"success": true, "reason": "duplicate result ignored"}`)) + } + return + } + workflowExecution, dbSave, err := shuffle.ParsedExecutionResult(ctx, *workflowExecution, actionResult, false, 0) if err != nil { b, suberr := json.Marshal(actionResult) @@ -628,6 +637,32 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } +func hasEquivalentActionResult(workflowExecution shuffle.WorkflowExecution, incoming shuffle.ActionResult) bool { + for _, existing := range workflowExecution.Results { + if existing.Action.ID != incoming.Action.ID { + continue + } + + if existing.Status != incoming.Status { + continue + } + + if existing.CompletedAt > 0 && incoming.CompletedAt > 0 && existing.CompletedAt == incoming.CompletedAt { + return true + } + + if existing.StartedAt > 0 && incoming.StartedAt > 0 && existing.StartedAt == incoming.StartedAt { + return true + } + + if len(existing.Result) > 0 && existing.Result == incoming.Result { + return true + } + } + + return false +} + func JSONCheck(str string) bool { var jsonStr interface{} return json.Unmarshal([]byte(str), &jsonStr) == nil From e4101f991aa5254a790dc38551d97cc8097a3124 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Fri, 1 May 2026 13:14:51 +0530 Subject: [PATCH 46/61] Revert "fix: ignore duplicate action results during execution updates" This reverts commit fb4c1edb7bac279a40887b52ddf0029136e3fea1. --- backend/go-app/walkoff.go | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index e6167cb4..b8abdf8d 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -593,15 +593,6 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl return } - if hasEquivalentActionResult(*workflowExecution, actionResult) { - log.Printf("[DEBUG][%s] Ignoring duplicate action result for action %s with status %s", workflowExecutionId, actionResult.Action.ID, actionResult.Status) - if resp != nil { - resp.WriteHeader(http.StatusOK) - resp.Write([]byte(`{"success": true, "reason": "duplicate result ignored"}`)) - } - return - } - workflowExecution, dbSave, err := shuffle.ParsedExecutionResult(ctx, *workflowExecution, actionResult, false, 0) if err != nil { b, suberr := json.Marshal(actionResult) @@ -637,32 +628,6 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } -func hasEquivalentActionResult(workflowExecution shuffle.WorkflowExecution, incoming shuffle.ActionResult) bool { - for _, existing := range workflowExecution.Results { - if existing.Action.ID != incoming.Action.ID { - continue - } - - if existing.Status != incoming.Status { - continue - } - - if existing.CompletedAt > 0 && incoming.CompletedAt > 0 && existing.CompletedAt == incoming.CompletedAt { - return true - } - - if existing.StartedAt > 0 && incoming.StartedAt > 0 && existing.StartedAt == incoming.StartedAt { - return true - } - - if len(existing.Result) > 0 && existing.Result == incoming.Result { - return true - } - } - - return false -} - func JSONCheck(str string) bool { var jsonStr interface{} return json.Unmarshal([]byte(str), &jsonStr) == nil From 0ba8ac1eea2890948d554f7712af045adba706a8 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Fri, 1 May 2026 18:04:14 +0530 Subject: [PATCH 47/61] remove the waiting stage it can confuse users --- functions/onprem/worker/worker.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index e40e356f..e34d50d4 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -1581,11 +1581,6 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { return } - if workflowExecution.Status == "WAITING" { - log.Printf("[DEBUG][%s] Execution is WAITING. Skipping action dispatch until a new result updates state.", workflowExecution.ExecutionId) - return - } - startAction, extra, children, parents, visited, executed, nextActions, environments := shuffle.GetExecutionVariables(ctx, workflowExecution.ExecutionId) var dockercli *dockerclient.Client @@ -3087,12 +3082,6 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl workflowExecution, dbSave, err := shuffle.ParsedExecutionResult(ctx, *workflowExecution, actionResult, true, 0) if err == nil { - expectedSubflows, completedSubflows, failedSubflows := enforceSubflowBarrier(workflowExecution) - if expectedSubflows > 0 { - dbSave = true - log.Printf("[DEBUG][%s] Subflow barrier progress: %d/%d completed (failed=%d). Status=%s", workflowExecution.ExecutionId, completedSubflows, expectedSubflows, failedSubflows, workflowExecution.Status) - } - if workflowExecution.Status != "EXECUTING" && workflowExecution.Status != "WAITING" { log.Printf("[WARNING][%s] Execution is not executing, but %s. Stopping Transaction update.", workflowExecution.ExecutionId, workflowExecution.Status) if resp != nil { @@ -3157,11 +3146,6 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl return } else { log.Printf("[DEBUG][%s] Successfully got ParsedExecution with %d results!", workflowExecution.ExecutionId, len(workflowExecution.Results)) - expectedSubflows, completedSubflows, failedSubflows := enforceSubflowBarrier(workflowExecution) - if expectedSubflows > 0 { - dbSave = true - log.Printf("[DEBUG][%s] Subflow barrier progress: %d/%d completed (failed=%d). Status=%s", workflowExecution.ExecutionId, completedSubflows, expectedSubflows, failedSubflows, workflowExecution.Status) - } } } else { log.Printf("[ERROR][%s] Failed execution of parsedexecution: %s", workflowExecution.ExecutionId, err) From 7f7f93069da9a84dcab4f0139e86f91f140cf350 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Fri, 1 May 2026 18:41:24 +0530 Subject: [PATCH 48/61] return 200 before the workflow transaction --- functions/onprem/worker/worker.go | 108 +++++++++++------------------- 1 file changed, 38 insertions(+), 70 deletions(-) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index e34d50d4..b3607c9e 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -5038,7 +5038,7 @@ func main() { } } -func checkUnfinished(resp http.ResponseWriter, request *http.Request, execRequest shuffle.OrborusExecutionRequest) { +func checkUnfinished(execRequest shuffle.OrborusExecutionRequest) { // Meant as a function that periodically checks whether previous executions have finished or not. // Should probably be based on executedIds and finishedIds // Schedule a check in the future instead? @@ -5067,38 +5067,16 @@ func checkUnfinished(resp http.ResponseWriter, request *http.Request, execReques sendResult(*exec, data) } -func handleRunExecution(resp http.ResponseWriter, request *http.Request) { - defer request.Body.Close() - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Printf("[WARNING] Failed reading body for stream result queue") - resp.WriteHeader(400) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return - } - - //log.Printf("[DEBUG] In run execution with body length %d", len(body)) - var execRequest shuffle.OrborusExecutionRequest - err = json.Unmarshal(body, &execRequest) - if err != nil { - log.Printf("[WARNING] Failed shuffle.WorkflowExecution unmarshaling: %s", err) - resp.WriteHeader(400) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return - } - +func processRunExecution(execRequest shuffle.OrborusExecutionRequest) { // Checks if a workflow is done 30 seconds later, and sends info to backend no matter what go func() { time.Sleep(time.Duration(30) * time.Second) - checkUnfinished(resp, request, execRequest) + checkUnfinished(execRequest) }() window.AddEvent(time.Now()) ctx := context.Background() - // FIXME: This should be PER EXECUTION - //if strings.ToLower(os.Getenv("SHUFFLE_PASS_APP_PROXY")) == "true" { - // Is it ok if these are standard? Should they be update-able after launch? Hmm if len(execRequest.HTTPProxy) > 0 { log.Printf("[DEBUG] Sending proxy info to child process") os.Setenv("SHUFFLE_PASS_APP_PROXY", execRequest.ShufflePassProxyToApp) @@ -5128,9 +5106,7 @@ func handleRunExecution(resp http.ResponseWriter, request *http.Request) { baseUrl = execRequest.BaseUrl } - // Setting to just have an auth available. if len(execRequest.Authorization) > 0 && len(os.Getenv("AUTHORIZATION")) == 0 { - //log.Printf("[DEBUG] Sending proxy info to child process") os.Setenv("AUTHORIZATION", execRequest.Authorization) } @@ -5143,71 +5119,47 @@ func handleRunExecution(resp http.ResponseWriter, request *http.Request) { ) if err != nil { - log.Printf("[ERROR][%s] Failed to create a new request", execRequest.ExecutionId) - resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + log.Printf("[ERROR][%s] Failed to create stream results request: %s", execRequest.ExecutionId, err) return } client := shuffle.GetExternalClient(streamResultUrl) newresp, err := client.Do(req) if err != nil { - log.Printf("[ERROR] Failed making request (2): %s", err) - resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + log.Printf("[ERROR][%s] Failed making stream results request: %s", execRequest.ExecutionId, err) return } defer newresp.Body.Close() - body, err = ioutil.ReadAll(newresp.Body) + body, err := ioutil.ReadAll(newresp.Body) if err != nil { - log.Printf("[ERROR][%s] Failed reading body (2): %s", execRequest.ExecutionId, err) - resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + log.Printf("[ERROR][%s] Failed reading stream results response body: %s", execRequest.ExecutionId, err) return } if newresp.StatusCode != 200 { - log.Printf("[ERROR][%s] Bad statuscode: %d, %s", execRequest.ExecutionId, newresp.StatusCode, string(body)) - - if strings.Contains(string(body), "Workflowexecution is already finished") { - log.Printf("[DEBUG] Shutting down (19)") - //shutdown(workflowExecution, "", "", true) - } - - resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad statuscode: %d"}`, newresp.StatusCode))) + log.Printf("[ERROR][%s] Bad statuscode from stream results: %d, %s", execRequest.ExecutionId, newresp.StatusCode, string(body)) return } err = json.Unmarshal(body, &workflowExecution) if err != nil { - log.Printf("[ERROR] Failed workflowExecution unmarshal: %s", err) - resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + log.Printf("[ERROR][%s] Failed workflowExecution unmarshal: %s", execRequest.ExecutionId, err) return } - //err = shuffle.SetWorkflowExecution(ctx, workflowExecution, true) err = setWorkflowExecution(ctx, workflowExecution, true) if err != nil { - log.Printf("[ERROR] Failed initializing execution saving for %s: %s", workflowExecution.ExecutionId, err) + log.Printf("[ERROR][%s] Failed initializing execution saving: %s", workflowExecution.ExecutionId, err) } if workflowExecution.Status == "FINISHED" || workflowExecution.Status == "SUCCESS" { - log.Printf("[DEBUG] Workflow %s is finished. Exiting worker.", workflowExecution.ExecutionId) - log.Printf("[DEBUG] Shutting down (20)") - - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad status for execution - already %s. Returning with 200 OK"}`, workflowExecution.Status))) + log.Printf("[DEBUG] Workflow %s is finished before dispatch. Exiting async run setup.", workflowExecution.ExecutionId) return } - //startAction, extra, children, parents, visited, executed, nextActions, environments := shuffle.GetExecutionVariables(ctx, workflowExecution.ExecutionId) - extra := 0 for _, trigger := range workflowExecution.Workflow.Triggers { - //log.Printf("Appname trigger (0): %s", trigger.AppName) if trigger.AppName == "User Input" || trigger.AppName == "Shuffle Workflow" { extra += 1 } @@ -5216,15 +5168,10 @@ func handleRunExecution(resp http.ResponseWriter, request *http.Request) { log.Printf("[INFO][%s] (1) Status: %s, Results: %d, actions: %d", workflowExecution.ExecutionId, workflowExecution.Status, len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)+extra) if workflowExecution.Status != "EXECUTING" { - log.Printf("[WARNING] Exiting as worker execution has status %s!", workflowExecution.Status) - log.Printf("[DEBUG] Shutting down (38)") - resp.WriteHeader(400) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad status %s for the workflow execution %s"}`, workflowExecution.Status, workflowExecution.ExecutionId))) + log.Printf("[WARNING][%s] Exiting async run as execution status is %s", workflowExecution.ExecutionId, workflowExecution.Status) return } - //log.Printf("[DEBUG] Starting execution :O") - cacheKey := fmt.Sprintf("workflowexecution_%s", workflowExecution.ExecutionId) execData, err := json.Marshal(workflowExecution) if err != nil { @@ -5239,15 +5186,36 @@ func handleRunExecution(resp http.ResponseWriter, request *http.Request) { err = executionInit(workflowExecution) if err != nil { log.Printf("[DEBUG][%s] Shutting down (30) - Workflow setup failed: %s", workflowExecution.ExecutionId, err) - resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error in execution init: %s"}`, err))) return - //shutdown(workflowExecution, "", "", true) } handleExecutionResult(workflowExecution) - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) +} + +func handleRunExecution(resp http.ResponseWriter, request *http.Request) { + defer request.Body.Close() + body, err := ioutil.ReadAll(request.Body) + if err != nil { + log.Printf("[WARNING] Failed reading body for stream result queue") + resp.WriteHeader(400) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + return + } + + //log.Printf("[DEBUG] In run execution with body length %d", len(body)) + var execRequest shuffle.OrborusExecutionRequest + err = json.Unmarshal(body, &execRequest) + if err != nil { + log.Printf("[WARNING] Failed shuffle.WorkflowExecution unmarshaling: %s", err) + resp.WriteHeader(400) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + return + } + + resp.WriteHeader(http.StatusAccepted) + resp.Write([]byte(`{"success": true, "accepted": true}`)) + + go processRunExecution(execRequest) } func handleDownloadImage(resp http.ResponseWriter, request *http.Request) { From 20e87f85c47e1842109b0631e60568dcb9b5ed6c Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Fri, 1 May 2026 18:49:30 +0530 Subject: [PATCH 49/61] shuffle-shared bumo --- functions/onprem/worker/go.mod | 2 +- functions/onprem/worker/go.sum | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 183552c1..76be5021 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -11,7 +11,7 @@ require ( github.com/docker/docker v28.3.3+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v1.2.24 + github.com/shuffle/shuffle-shared v1.2.41 github.com/shuffle/singul v0.0.30 k8s.io/api v0.34.2 k8s.io/apimachinery v0.34.2 diff --git a/functions/onprem/worker/go.sum b/functions/onprem/worker/go.sum index 1eb817a4..652b3222 100644 --- a/functions/onprem/worker/go.sum +++ b/functions/onprem/worker/go.sum @@ -66,6 +66,8 @@ github.com/adrg/strutil v0.3.1 h1:OLvSS7CSJO8lBii4YmBt8jiK9QOtB9CzCzwl4Ic/Fz4= github.com/adrg/strutil v0.3.1/go.mod h1:8h90y18QLrs11IBffcGX3NW/GFBXCMcNg4M7H6MspPA= github.com/algolia/algoliasearch-client-go/v3 v3.31.4 h1:UJhx6AhZCYf0qZygDz2c1x1+1q2q2sfzsRaQM6yswWk= github.com/algolia/algoliasearch-client-go/v3 v3.31.4/go.mod h1:i7tLoP7TYDmHX3Q7vkIOL4syVse/k5VJ+k0i8WqFiJk= +github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= +github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= @@ -93,8 +95,6 @@ github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151X github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/coreos/go-oidc/v3 v3.17.0 h1:hWBGaQfbi0iVviX4ibC7bk8OKT5qNr4klBaCHVNvehc= -github.com/coreos/go-oidc/v3 v3.17.0/go.mod h1:wqPbKFrVnE90vty060SB40FCJ8fTHTxSwyXJqZH+sI8= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s= github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= @@ -130,8 +130,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0= github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4= -github.com/frikky/schemaless v0.0.28 h1:gdurMqBwtvY4Y/5pcxn8bdGCJn/eolKGz+c5DcidLkI= -github.com/frikky/schemaless v0.0.28/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= +github.com/frikky/schemaless v0.0.33 h1:5Soj6VQc+ozqLh4R6MatWOl/atAeNpdon+nV5EKwjOI= +github.com/frikky/schemaless v0.0.33/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -317,10 +317,10 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shuffle/opensearch-go/v4 v4.0.0 h1:Mh85CD1MwOgXiFFYlzS1llnvdqL3CztRdR1ZT/SLIjU= github.com/shuffle/opensearch-go/v4 v4.0.0/go.mod h1:gVLZKQE5khQWMb68XBtgKrhu78oLGL2zHwAGnFMDwC0= -github.com/shuffle/shuffle-shared v0.9.87 h1:INA1cZ18MKcMs8kaVBJoQqrdNuqLHfge9elJ17hZfVc= -github.com/shuffle/shuffle-shared v0.9.87/go.mod h1:AkXajlWWB16WfWjCw9K7y38L8JKABJzVxcX6KI/J1H4= -github.com/shuffle/singul v0.0.26 h1:P2uZ8YIYQUN4qNfQujoW3Lhod91X6TAfrRHGWiUPNI8= -github.com/shuffle/singul v0.0.26/go.mod h1:S8GszXL+fT2mTnh7j57V0r/tY5Y/mSW8QowQEnADs3k= +github.com/shuffle/shuffle-shared v1.2.41 h1:p2bW08L9jZaRLRmlTDgTTmPgYFryEgwRF5ceqRZtONk= +github.com/shuffle/shuffle-shared v1.2.41/go.mod h1:RSKyexqkGDB+WbboGfWMj1Sfl4e49MI2CvW3pFmirg4= +github.com/shuffle/singul v0.0.30 h1:xYTpGHWzZ9lX1P6/kJIz6kng5q6gUdua3IMhsK399e8= +github.com/shuffle/singul v0.0.30/go.mod h1:2dGXQMk8q4QOivomuJ9QkrpevpmTM1QYemJixbY5jW0= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -364,6 +364,8 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= From 0bc92c9fa73106d6359c3aa885a3e028eba1a5d4 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Thu, 7 May 2026 05:03:27 +0530 Subject: [PATCH 50/61] pass proxy if worker don't have them either --- functions/onprem/worker/worker.go | 53 ++++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index b3607c9e..8598626c 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -193,6 +193,40 @@ func restoreActionConfig(ctx context.Context, executionID string, action *shuffl } } +func getAppProxyValue(primaryKey, fallbackKey string) string { + value := strings.TrimSpace(os.Getenv(primaryKey)) + if len(value) > 0 { + return value + } + + return strings.TrimSpace(os.Getenv(fallbackKey)) +} + +func appendAppProxyEnv(env []string) []string { + httpProxy := getAppProxyValue("HTTP_PROXY", "SHUFFLE_APP_HTTP_PROXY") + httpsProxy := getAppProxyValue("HTTPS_PROXY", "SHUFFLE_APP_HTTPS_PROXY") + noProxy := getAppProxyValue("NO_PROXY", "SHUFFLE_APP_NO_PROXY") + noProxyLower := getAppProxyValue("no_proxy", "SHUFFLE_APP_no_proxy") + + if len(httpProxy) > 0 { + env = append(env, fmt.Sprintf("HTTP_PROXY=%s", httpProxy)) + } + + if len(httpsProxy) > 0 { + env = append(env, fmt.Sprintf("HTTPS_PROXY=%s", httpsProxy)) + } + + if len(noProxy) > 0 { + env = append(env, fmt.Sprintf("NO_PROXY=%s", noProxy)) + } + + if len(noProxyLower) > 0 { + env = append(env, fmt.Sprintf("no_proxy=%s", noProxyLower)) + } + + return env +} + // Images to be autodeployed in the latest version of Shuffle. var autoDeploy = map[string]string{ "http:1.4.0": "frikky/shuffle:http_1.4.0", @@ -1784,10 +1818,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { if strings.ToLower(os.Getenv("SHUFFLE_PASS_APP_PROXY")) == "true" { //log.Printf("APPENDING PROXY TO THE APP!") - env = append(env, fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY"))) - env = append(env, fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY"))) - env = append(env, fmt.Sprintf("NO_PROXY=%s", os.Getenv("NO_PROXY"))) - env = append(env, fmt.Sprintf("no_proxy=%s", os.Getenv("no_proxy"))) + env = appendAppProxyEnv(env) } overrideHttpProxy := os.Getenv("SHUFFLE_INTERNAL_HTTP_PROXY") @@ -3818,10 +3849,7 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl } if strings.ToLower(os.Getenv("SHUFFLE_PASS_APP_PROXY")) == "true" { - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY"))) - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY"))) - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("NO_PROXY=%s", os.Getenv("NO_PROXY"))) - serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("no_proxy=%s", os.Getenv("no_proxy"))) + serviceSpec.TaskTemplate.ContainerSpec.Env = appendAppProxyEnv(serviceSpec.TaskTemplate.ContainerSpec.Env) } overrideHttpProxy := os.Getenv("SHUFFLE_INTERNAL_HTTP_PROXY") @@ -4342,6 +4370,10 @@ func sendAppRequest(ctx context.Context, incomingUrl, appName string, port int, callbackUrl := os.Getenv("SHUFFLE_WORKER_SERVER_URL") if len(callbackUrl) > 0 { parsedRequest.BaseUrl = callbackUrl + if parsedRequest.Action.AppName == "shuffle-subflow" || parsedRequest.Action.AppName == "shuffle-subflow-v2" || parsedRequest.Action.AppName == "User Input" { + parsedRequest.BaseUrl = fmt.Sprintf("http://%s:%d", hostname, baseport) + //parsedRequest.Url = parsedRequest.BaseUrl + } } else if len(hostname) > 0 { // Run with proper hostname, but set to shuffle-worker to avoid specific host target. // This means running with VIP instead. @@ -4568,10 +4600,7 @@ func baseDeploy() { if strings.ToLower(os.Getenv("SHUFFLE_PASS_APP_PROXY")) == "true" { //log.Printf("APPENDING PROXY TO THE APP!") - env = append(env, fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY"))) - env = append(env, fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY"))) - env = append(env, fmt.Sprintf("NO_PROXY=%s", os.Getenv("NO_PROXY"))) - env = append(env, fmt.Sprintf("no_proxy=%s", os.Getenv("no_proxy"))) + env = appendAppProxyEnv(env) } if len(os.Getenv("SHUFFLE_APP_SDK_TIMEOUT")) > 0 { From 4cff6addde3d7f38d5ea10d4bbb68ba04af72064 Mon Sep 17 00:00:00 2001 From: Lalit Deore Date: Thu, 7 May 2026 15:27:32 +0530 Subject: [PATCH 51/61] shaffuru files sync --- frontend/public/images/icons/aws_logo.svg | 3 + frontend/src/components/AdminNavBar.jsx | 4 + frontend/src/components/ApiExplorer.jsx | 2 +- frontend/src/components/AppAuthTab.jsx | 193 +- frontend/src/components/AppCreationModal.jsx | 2 +- frontend/src/components/AppGrid.jsx | 233 +- frontend/src/components/AppModal.jsx | 2 +- frontend/src/components/AppSearch1.jsx | 2 +- frontend/src/components/Appsearch.jsx | 2 +- frontend/src/components/Billing.jsx | 460 +++- frontend/src/components/BillingStats.jsx | 16 + frontend/src/components/CacheView.jsx | 255 +- .../src/components/CollectIngestModal.jsx | 4 +- frontend/src/components/ConfigureWorkflow.jsx | 2 +- frontend/src/components/CreatorGrid.jsx | 103 +- .../src/components/DeleteConfirmDialog.jsx | 61 + frontend/src/components/DiscordChat.jsx | 103 +- frontend/src/components/DocsGrid.jsx | 114 +- frontend/src/components/EnvironmentTab.jsx | 66 +- frontend/src/components/Files.jsx | 230 +- frontend/src/components/HealthPage.jsx | 1288 +++++++--- frontend/src/components/LeftSideBar.jsx | 226 +- frontend/src/components/LicencePopup.jsx | 157 +- frontend/src/components/Oauth2Auth.jsx | 122 +- .../src/components/OrgHeaderexpandedNew.jsx | 6 +- frontend/src/components/ParsedActionNew.jsx | 894 +++++-- .../src/components/PartnersUsecasesTab.jsx | 1 + frontend/src/components/RuntimeDebugger.jsx | 3 + frontend/src/components/SearchContactForm.jsx | 121 + frontend/src/components/SearchData.jsx | 2 +- frontend/src/components/Searchfield.jsx | 3 +- .../src/components/ShuffleCodeEditor1.jsx | 42 +- .../components/SubOrgDistributionDialog.jsx | 265 ++ frontend/src/components/TenantsTab.jsx | 2288 ++++++++--------- frontend/src/components/UserManagmentTab.jsx | 163 +- frontend/src/components/WorkflowGrid.jsx | 118 +- frontend/src/components/Workflowsearch.jsx | 2 +- frontend/src/components/ssoTab.jsx | 35 +- frontend/src/theme.jsx | 180 +- frontend/src/views/AgentUI.jsx | 168 +- frontend/src/views/AngularWorkflow.jsx | 572 ++++- frontend/src/views/ApiExplorerWrapper.jsx | 128 +- frontend/src/views/AppCreator.jsx | 38 +- frontend/src/views/AppExplorer.jsx | 118 +- frontend/src/views/Apps.jsx | 18 +- frontend/src/views/Apps2.jsx | 130 +- frontend/src/views/Docs.jsx | 19 +- frontend/src/views/LoginPage.jsx | 35 +- frontend/src/views/NewDashboard.jsx | 4 +- frontend/src/views/RunWorkflow.jsx | 174 +- frontend/src/views/UpdateAuthentication.jsx | 46 +- frontend/src/views/Usecases2.jsx | 5 +- frontend/src/views/Welcome.jsx | 266 +- frontend/src/views/Workflows2.jsx | 50 +- 54 files changed, 5857 insertions(+), 3687 deletions(-) create mode 100644 frontend/public/images/icons/aws_logo.svg create mode 100644 frontend/src/components/DeleteConfirmDialog.jsx create mode 100644 frontend/src/components/SearchContactForm.jsx create mode 100644 frontend/src/components/SubOrgDistributionDialog.jsx diff --git a/frontend/public/images/icons/aws_logo.svg b/frontend/public/images/icons/aws_logo.svg new file mode 100644 index 00000000..f023cba5 --- /dev/null +++ b/frontend/public/images/icons/aws_logo.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/frontend/src/components/AdminNavBar.jsx b/frontend/src/components/AdminNavBar.jsx index 6a771afd..38aff384 100644 --- a/frontend/src/components/AdminNavBar.jsx +++ b/frontend/src/components/AdminNavBar.jsx @@ -142,6 +142,10 @@ const AdminNavBar = (props) => { // navigate(`?tab=organization`, { replace: true }); // } // } + const adminTab = queryParams.get('admin_tab'); + if (adminTab) { + setSelectedItem("Organization"); + } if (partnerTab) { setSelectedItem("Partner"); } diff --git a/frontend/src/components/ApiExplorer.jsx b/frontend/src/components/ApiExplorer.jsx index 3bf6785b..2e5009fa 100644 --- a/frontend/src/components/ApiExplorer.jsx +++ b/frontend/src/components/ApiExplorer.jsx @@ -1977,7 +1977,7 @@ const Action = memo(( const actionId = action.name.replace(/ /g, "-").replace(/_/g, "-"); window.history.pushState(null, "", `#${actionId}`); setExampleBody(action?.example_response); - document.getElementById(`action-list-${nextSelectedActionIndex}`).scrollIntoView({ behavior: "smooth", block: "center" }); + document.getElementById(`action-list-${nextSelectedActionIndex}`)?.scrollIntoView({ behavior: "smooth", block: "center" }); } }, 300); } diff --git a/frontend/src/components/AppAuthTab.jsx b/frontend/src/components/AppAuthTab.jsx index 7c404e1c..af7fbef2 100644 --- a/frontend/src/components/AppAuthTab.jsx +++ b/frontend/src/components/AppAuthTab.jsx @@ -20,6 +20,7 @@ import { isMobile } from "react-device-detect" import PaperComponent from "../components/PaperComponent.jsx"; import { CodeHandler, Img, OuterLink, } from '../views/Docs.jsx' import { v4 as uuidv4} from "uuid"; +import DeleteConfirmDialog from "./DeleteConfirmDialog.jsx"; import { Divider, @@ -63,10 +64,11 @@ import { } from "react-instantsearch-dom"; import aa from "search-insights"; import { Context } from '../context/ContextApi.jsx'; +import SubOrgDistributionDialog from './SubOrgDistributionDialog.jsx'; const searchClient = algoliasearch( "JNSS5CFDZZ", - "c8f882473ff42d41158430be09ec2b4e" + "33e4e3564f4f060e96e0531957bed552" ) const AppAuthTab = memo((props) => { @@ -89,9 +91,13 @@ const AppAuthTab = memo((props) => { const [searchQuery, setSearchQuery] = React.useState(""); const [showAppModal, setShowAppModal] = useState(false) const [selectedAuthId, setSelectedAuthId] = useState(""); + const [selectedAuthName, setSelectedAuthName] = useState(""); const [showDistributionPopup, setShowDistributionPopup] = useState(false); const [showAuthenticationLoader, setShowAuthenticationLoader] = useState(true) const [showAppAuthGroupLoader, setShowAppAuthGroupLoader] = useState(true) + const [deleteConfirmOpen, setDeleteConfirmOpen] = useState(false); + const [deleteConfirmTarget, setDeleteConfirmTarget] = useState(null); + const [distribOrgOrder, setDistribOrgOrder] = useState([]); const { themeMode, supportEmail, brandColor } = useContext(Context) const theme = getTheme(themeMode, brandColor) @@ -183,7 +189,7 @@ const AppAuthTab = memo((props) => { }; const deleteAuthentication = (data) => { - toast("Deleting auth " + data.label); + toast("Deleting auth " + data?.label); // Just use this one? const url = globalUrl + "/api/v1/apps/authentication/" + data.id; @@ -213,33 +219,6 @@ const AppAuthTab = memo((props) => { }); }; - const handleSelectSubOrg = (id, action) => { - if (action === "all") { - const childOrgs = userdata.orgs.filter( - (data) => data.creator_org === userdata.active_org.id - ); - setSelectedSubOrg((prev) => { - if (prev.length === childOrgs.length) { - // If all child orgs are already selected, clear the selection - return []; - } else { - // Otherwise, select all child org IDs - return childOrgs.map((data) => data.id); - } - }); - } else if (action === "none") { - setSelectedSubOrg([]); - } else { - setSelectedSubOrg((prev) => { - if (prev.includes(id)) { - return prev.filter((data) => data !== id); - } else { - return [...prev, id]; - } - }); - } - }; - const editAuthenticationConfig = (id, parentAction, selectedSuborgs) => { const data = { id: id, @@ -284,100 +263,22 @@ const AppAuthTab = memo((props) => { editAuthenticationConfig(id, "suborg_distribute", [...new Set(selectedSubOrg)]) } - - - const cacheDistributionModal = showDistributionPopup ? ( - {setShowDistributionPopup(false);setSelectedAuthId("")}} - PaperProps={{ - sx: { - borderRadius: theme?.palette?.DialogStyle?.borderRadius, - border: theme?.palette?.DialogStyle?.border, - fontFamily: theme?.typography?.fontFamily, - backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, - zIndex: 1000, - minWidth: "600px", - minHeight: "320px", - overflow: "auto", - '& .MuiDialogContent-root': { - backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, - }, - '& .MuiDialogTitle-root': { - backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, - }, - '& .MuiDialogActions-root': { - backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, - }, - }, - }} - > - - - Select sub-org to distribute Datastore key - - - - {handleSelectSubOrg(null, "none")}}>None - {handleSelectSubOrg(null, "all")}}>All - {userdata.orgs.map((data, index) => { - if (data.creator_org !== userdata.active_org.id) { - return null; - } - - const imagesize = 22; - const imageStyle = { - width: imagesize, - height: imagesize, - pointerEvents: "none", - marginRight: 10, - marginLeft: data.id === userdata.active_org.id ? 0 : 20, - }; - - const image = data.image === "" ? ( - {data.name} - ) : ( - {data.name} - ); - return ( - handleSelectSubOrg(data.id)} - style={{ display: "flex", alignItems: "center" }} - > - - {image} - {data.name} - - ); - })} - -
- - -
- - - ) : null; + const deleteConfirmDialog = ( + { setDeleteConfirmOpen(false); setDeleteConfirmTarget(null); }} + onConfirm={() => { + deleteAuthentication(deleteConfirmTarget); + setDeleteConfirmOpen(false); + setDeleteConfirmTarget(null); + }} + title="Delete Authentication?" + description={<>Are you sure you want to delete {deleteConfirmTarget?.app?.name} authentication?} + warningText="This cannot be undone. Any workflows using this authentication will lose access." + /> + ); + const editAuthenticationModal = selectedAuthenticationModalOpen ? ( { return (
{appModal} - {cacheDistributionModal} + { setShowDistributionPopup(false); setSelectedAuthId(""); setSelectedAuthName(""); }} + title="Distribute App Auth to Sub-Organizations" + extraInfo={selectedAuthName ? `Selected Auth: ${selectedAuthName}` : null} + orgs={distribOrgOrder.map(id => (userdata?.orgs || []).find(o => o.id === id)).filter(Boolean)} + selectedOrgIds={selectedSubOrg} + onSelectionChange={setSelectedSubOrg} + onSave={(ids) => { changeDistribution(selectedAuthId, ids); }} + /> + {deleteConfirmDialog}
@@ -1362,10 +1273,10 @@ const AppAuthTab = memo((props) => { { - deleteAuthentication(data); + setDeleteConfirmTarget(data); + setDeleteConfirmOpen(true); }} > delete icon @@ -1400,21 +1311,27 @@ const AppAuthTab = memo((props) => { color="secondary" onClick={() => { setShowDistributionPopup(true) - if(data?.suborg_distribution?.length > 0){ - setSelectedSubOrg(data.suborg_distribution) - }else{ - setSelectedSubOrg([]) - } - setSelectedAuthId(data.id) + let initialSelected = []; if (data?.suborg_distributed) { - const allSuborg = userdata?.orgs?.map((data, index) => { - if (data.creator_org !== userdata.active_org.id) { - return null; - } - return data.id; - }) - setSelectedSubOrg(allSuborg.filter((data) => data !== null)) + const allSuborg = userdata?.orgs?.map((d) => { + if (d.creator_org !== userdata.active_org.id) return null; + return d.id; + }); + initialSelected = allSuborg.filter((d) => d !== null); + } else if (data?.suborg_distribution?.length > 0) { + initialSelected = data.suborg_distribution; } + setSelectedSubOrg(initialSelected); + setSelectedAuthId(data.id); + setSelectedAuthName(data?.app?.name); + const suborgs = (userdata?.orgs || []).filter(o => o.creator_org === userdata?.active_org?.id); + const sorted = [...suborgs].sort((a, b) => { + const aS = initialSelected.includes(a.id); + const bS = initialSelected.includes(b.id); + if (aS !== bS) return bS - aS; + return a.name.localeCompare(b.name); + }); + setDistribOrgOrder(sorted.map(o => o.id)); }} /> diff --git a/frontend/src/components/AppCreationModal.jsx b/frontend/src/components/AppCreationModal.jsx index 6a66f0e0..e1e62e28 100644 --- a/frontend/src/components/AppCreationModal.jsx +++ b/frontend/src/components/AppCreationModal.jsx @@ -708,7 +708,7 @@ const AppCreationModal = ({ open, onClose, theme, globalUrl, isCloud, startOpenA fontWeight: 500, fontFamily: theme?.typography?.fontFamily, }}> - Generate an app based on documentation (beta) + Generate an app based on documentation { diff --git a/frontend/src/components/AppGrid.jsx b/frontend/src/components/AppGrid.jsx index cd37fb23..68b4f7db 100644 --- a/frontend/src/components/AppGrid.jsx +++ b/frontend/src/components/AppGrid.jsx @@ -1,10 +1,9 @@ -import React, { useEffect, useState, useRef } from "react"; +import React, { useEffect, useState, useRef, useMemo } from "react"; import theme from "../theme.jsx"; import ReactGA from "react-ga4"; import { Link } from "react-router-dom"; import { removeQuery } from "../components/ScrollToTop.jsx"; -import { useMemo } from "react"; import { Tabs, Tab, Collapse } from "@mui/material"; import ExpandMoreIcon from "@mui/icons-material/ExpandMore"; @@ -27,7 +26,7 @@ import { InstantSearch, Configure, connectSearchBox, - connectHits, + connectInfiniteHits, connectHitInsights, RefinementList, ClearRefinements, @@ -39,6 +38,7 @@ import aa from "search-insights"; import { useLocation } from 'react-router-dom'; import "./FilterCSS.css"; +import SearchContactForm from "../components/SearchContactForm.jsx"; import { Zoom, @@ -54,7 +54,7 @@ import { const searchClient = algoliasearch( "JNSS5CFDZZ", - "c8f882473ff42d41158430be09ec2b4e" + "eb5fd80aa6ed5ab4730d836cff3ea283" ); //const searchClient = algoliasearch("L55H18ZINA", "a19be455e7e75ee8f20a93d26b9fc6d6") @@ -77,62 +77,13 @@ const AppGrid = (props) => { const xs = parsedXs === undefined || parsedXs === null ? (isMobile ? 6 : 3) : parsedXs; - const [formMail, setFormMail] = React.useState(""); - const [message, setMessage] = React.useState(""); - const [formMessage, setFormMessage] = React.useState(""); const [deactivatedIndexes, setDeactivatedIndexes] = React.useState([]); - const buttonStyle = { - borderRadius: 30, - height: 50, - width: 220, - margin: isMobile ? "15px auto 15px auto" : 20, - fontSize: 18, - }; const innerColor = "rgba(255,255,255,0.65)"; const borderRadius = 3; window.title = "Shuffle | Apps | Find and integrate any app"; const noImage = "/public/no_image.png"; - const submitContact = (email, message) => { - const data = { - firstname: "", - lastname: "", - title: "", - companyname: "", - email: email, - phone: "", - message: message, - }; - - const errorMessage = - "Something went wrong. Please contact frikky@shuffler.io directly."; - - fetch(globalUrl + "/api/v1/contact", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(data), - }) - .then((response) => response.json()) - .then((response) => { - if (response?.success === true) { - setFormMessage(response.reason); - //toast("Thanks for submitting!") - } else { - setFormMessage(errorMessage); - } - - setFormMail(""); - setMessage(""); - }) - .catch((error) => { - setFormMessage(errorMessage); - console.log(error); - }); - }; - const SearchBox = ({ currentRefinement, refine, isSearchStalled, searchQuery, setSearchQuery }) => { var defaultSearch = ""; @@ -148,10 +99,11 @@ const AppGrid = (props) => { const params = Object.fromEntries(urlSearchParams.entries()); const foundQuery = params["q"]; if (foundQuery !== null && foundQuery !== undefined) { - console.log("Got query: ", foundQuery); refine(foundQuery); defaultSearch = foundQuery; - searchQuery = foundQuery + if (searchQuery !== foundQuery) { + setSearchQuery(foundQuery); + } } } //}, []) @@ -234,7 +186,13 @@ const AppGrid = (props) => { onChange={(event) => { const value = event.currentTarget.value; setSearchQuery(value); - removeQuery("q"); + const urlSearchParams = new URLSearchParams(window.location.search); + if (value) { + urlSearchParams.set("q", value); + } else { + urlSearchParams.delete("q"); + } + window.history.replaceState({}, '', `${window.location.pathname}?${urlSearchParams.toString()}`); debouncedRefine(value); }} onKeyDown={(event) => { @@ -252,6 +210,21 @@ const AppGrid = (props) => { const [currTab, setCurrTab] = useState(0); const location = useLocation(); + const conditionalSearchClient = useMemo(() => ({ + ...searchClient, + search(requests) { + if (currTab !== 0) { + return Promise.resolve({ + results: requests.map(() => ({ + hits: [], nbHits: 0, page: 0, nbPages: 0, hitsPerPage: 0, + processingTimeMS: 0, exhaustiveNbHits: true, query: "", params: "", + })), + }); + } + return searchClient.search(requests); + }, + }), [currTab]); + useEffect(() => { const queryParams = new URLSearchParams(location.search); const tabParam = queryParams.get('tab'); @@ -269,7 +242,6 @@ const AppGrid = (props) => { const newQueryParam = newTab === 0 ? 'all_apps' : newTab === 1 ? 'org_apps' : 'my_apps'; const queryParams = new URLSearchParams(location.search); queryParams.set('tab', newQueryParam); - queryParams.delete('q'); window.history.replaceState({}, '', `${location.pathname}?${queryParams.toString()}`); }; @@ -281,6 +253,8 @@ const AppGrid = (props) => { // Component to fetch all public app from the algolia. const Hits = ({ hits, + hasMore, + refineNext, insights, setIsAnyAppActivated, searchQuery @@ -288,6 +262,32 @@ const AppGrid = (props) => { const [mouseHoverIndex, setMouseHoverIndex] = useState(-1); var counted = 0; const [hoverEffect, setHoverEffect] = useState(-1); + const [isLoadingMore, setIsLoadingMore] = useState(false); + const loadMoreRef = useRef(null); + const scrollContainerRef = useRef(null); + const isFetchingMore = useRef(false); + + useEffect(() => { + isFetchingMore.current = false; + setIsLoadingMore(false); + }, [hits.length]); + + useEffect(() => { + return; // infinite scroll disabled + if (!loadMoreRef.current || !scrollContainerRef.current) return; + const observer = new IntersectionObserver( + (entries) => { + if (entries[0].isIntersecting && hasMore && !isFetchingMore.current) { + isFetchingMore.current = true; + setIsLoadingMore(true); + refineNext(); + } + }, + { root: scrollContainerRef.current, rootMargin: "200px" } + ); + observer.observe(loadMoreRef.current); + return () => observer.disconnect(); + }, [hasMore, refineNext]); const normalizedString = (name) => { if (typeof name === 'string') { @@ -359,11 +359,11 @@ const AppGrid = (props) => { } else { //toast.success(`App ${type}d Successfully!`); if (type === 'activate') { - setAllActivatedAppIds(prev => [...prev, data.objectID]); + setAllActivatedAppIds(prev => [...(prev || []), data.objectID]); setIsAnyAppActivated(true); } if (type === 'deactivate') { - const updatedIds = allActivatedAppIds.filter(id => id !== data.objectID); + const updatedIds = (allActivatedAppIds || []).filter(id => id !== data.objectID); setAllActivatedAppIds(updatedIds); } } @@ -373,6 +373,16 @@ const AppGrid = (props) => { }); } + const sortedHits = useMemo(() => { + const list = [...(hits || [])]; + if (!allActivatedAppIds?.length) return list; + return list.sort((a, b) => { + const aActive = allActivatedAppIds.includes(a.objectID) ? 1 : 0; + const bActive = allActivatedAppIds.includes(b.objectID) ? 1 : 0; + return bActive - aActive; + }); + }, [hits, hits?.length, allActivatedAppIds]); + let workflowDelay = 0; const isHeader = true; const paperStyle = { @@ -404,6 +414,7 @@ const AppGrid = (props) => { ) : (
{ scrollbarColor: "#494949 #2f2f2f", }} > - {hits?.map((data, index) => { + {sortedHits.map((data, index) => { const appUrl = isCloud === true ? `/apps/${data.objectID}` @@ -629,6 +640,12 @@ const AppGrid = (props) => { ); }) } +
+ {isLoadingMore && ( +
+ +
+ )}
)} @@ -926,7 +943,7 @@ const AppGrid = (props) => { //Component to display all apps. const AllApps = ({ setIsAnyAppActivated }) => { - var [searchQuery, setSearchQuery] = useState(""); + var [searchQuery, setSearchQuery] = useState(() => new URLSearchParams(window.location.search).get('q') || ""); return (
{ }} onClick={() => { setSearchQuery(''); + removeQuery("q"); }} /> )} @@ -1015,7 +1033,15 @@ const AppGrid = (props) => { placeholder="Search your Activated or self-built apps" id="shuffle_search_field" onChange={(event) => { - setSearchQuery(event.currentTarget.value); + const value = event.currentTarget.value; + setSearchQuery(value); + const urlSearchParams = new URLSearchParams(window.location.search); + if (value) { + urlSearchParams.set("q", value); + } else { + urlSearchParams.delete("q"); + } + window.history.replaceState({}, '', `${window.location.pathname}?${urlSearchParams.toString()}`); }} onKeyDown={(event) => { if(event.key === "Enter") { @@ -1592,7 +1618,7 @@ const AppGrid = (props) => { //Component to fetch all apps created by user and Org const UserAndOrgApps = ({ selectedCategoryForUsersAndOgsApps, selectedTagsForUserAndOrgApps, selectedOptionOfCreatedWith, setselectedCategoryForUsersAndOgsApps, setSelectedTagsForUserAndOrgApps, setSelectedOptionOfCreatedWith }) => { - const [searchQuery, setSearchQuery] = useState(""); + const [searchQuery, setSearchQuery] = useState(() => new URLSearchParams(window.location.search).get('q') || ""); const [appsToShow, setAppsToShow] = useState([]); useEffect(() => { if (currTab === 1) { @@ -2003,7 +2029,7 @@ const AppGrid = (props) => { }; const CustomSearchBox = connectSearchBox(SearchBox); - const CustomHits = connectHits(Hits); + const CustomHits = connectInfiniteHits(Hits); const DisplayAllAppsTab = () => { const [selectedCategoryForUsersAndOgsApps, setselectedCategoryForUsersAndOgsApps] = useState([]); @@ -2012,7 +2038,7 @@ const AppGrid = (props) => { return (
- +
{currTab === 0 ? ( @@ -2037,7 +2063,7 @@ const AppGrid = (props) => { />
- + {currTab === 0 && }
); @@ -2060,80 +2086,7 @@ const AppGrid = (props) => { > {showSuggestion === true ? ( -
- - Can't find what you're looking for? - -
- setFormMail(e.target.value)} - /> - setMessage(e.target.value)} - /> -
- - - {formMessage} - -
+ ) : null}
diff --git a/frontend/src/components/AppModal.jsx b/frontend/src/components/AppModal.jsx index 38c8581f..b0f8728f 100644 --- a/frontend/src/components/AppModal.jsx +++ b/frontend/src/components/AppModal.jsx @@ -35,7 +35,7 @@ import { Context } from '../context/ContextApi.jsx'; const searchClient = algoliasearch( "JNSS5CFDZZ", - "c8f882473ff42d41158430be09ec2b4e" + "33e4e3564f4f060e96e0531957bed552" );; const AppModal = ({ open, onClose, app, globalUrl, getApps}) => { diff --git a/frontend/src/components/AppSearch1.jsx b/frontend/src/components/AppSearch1.jsx index 449ad925..d551841d 100644 --- a/frontend/src/components/AppSearch1.jsx +++ b/frontend/src/components/AppSearch1.jsx @@ -13,7 +13,7 @@ import { InputAdornment, Typography, } from '@mui/material'; -const searchClient = algoliasearch("JNSS5CFDZZ", "c8f882473ff42d41158430be09ec2b4e") +const searchClient = algoliasearch("JNSS5CFDZZ", "33e4e3564f4f060e96e0531957bed552") const Appsearch = props => { const { maxRows, showName, showSuggestion, isMobile, globalUrl, parsedXs, newSelectedApp, setNewSelectedApp, defaultSearch, showSearch, ConfiguredHits, userdata, cy, isCreatorPage, actionImageList, setActionImageList, setUserSpecialzedApp, placeholder, diff --git a/frontend/src/components/Appsearch.jsx b/frontend/src/components/Appsearch.jsx index 904cf7c2..5b529d9d 100644 --- a/frontend/src/components/Appsearch.jsx +++ b/frontend/src/components/Appsearch.jsx @@ -20,7 +20,7 @@ import { } from '@mui/material'; import aa from 'search-insights' -const searchClient = algoliasearch("JNSS5CFDZZ", "c8f882473ff42d41158430be09ec2b4e") +const searchClient = algoliasearch("JNSS5CFDZZ", "33e4e3564f4f060e96e0531957bed552") const Appsearch = props => { const { maxRows, showName, showSuggestion, isMobile, globalUrl, parsedXs, newSelectedApp, setNewSelectedApp, defaultSearch, showSearch, ConfiguredHits, userdata, cy, isCreatorPage, actionImageList, setActionImageList, setUserSpecialzedApp, inputHeight, apps, } = props const { themeMode } = useContext(Context) diff --git a/frontend/src/components/Billing.jsx b/frontend/src/components/Billing.jsx index 0a4a03e4..959ce875 100644 --- a/frontend/src/components/Billing.jsx +++ b/frontend/src/components/Billing.jsx @@ -52,6 +52,7 @@ import { Cancel as CancelIcon, Shield as ShieldIcon, Cancel as XCircleIcon, + LockOutlined as LockIcon, FlashOn as ZapIcon, People as UsersIcon, FmdGoodOutlined as FmdGoodOutlinedIcon, @@ -203,7 +204,7 @@ const ProductionStatus = ({ selectedOrganization, userdata, isCloud, theme }) => > {isProdStatusOn ? 'Your organization has full access to all enterprise features and capabilities.' - : 'View your current limits and available features. Upgrade to unlock enterprise capabilities.'} + : 'Your organization is running on the open-source plan. Upgrade to Enterprise to remove limits and unlock advanced capabilities.'} {/* Features Grid */} @@ -223,8 +224,8 @@ const ProductionStatus = ({ selectedOrganization, userdata, isCloud, theme }) => }) .map((feature, index) => { const Icon = feature.icon; - const isAvailable = isProdStatusOn && feature.isActive; - const statusColor = isAvailable ? colors.success : colors.disabled; + const isAvailable = isProdStatusOn; + const statusColor = isAvailable ? colors.success : colors.warning; const bgColor = isAvailable ? themeMode === "dark" ? "#212121" : "#ffffff" : colors.disabledBg; return ( @@ -235,19 +236,34 @@ const ProductionStatus = ({ selectedOrganization, userdata, isCloud, theme }) => alignItems: 'center', gap: 14, padding: '14px 16px', + paddingLeft: !isAvailable ? 20 : 16, borderRadius: 10, background: bgColor, border: `1px solid ${isAvailable ? colors.success + '40' : colors.border}`, transition: 'all 0.2s', + position: 'relative', + overflow: 'hidden', }} > + {!isAvailable && ( +
+ )} + {/* Icon */}
fontSize: 15, fontWeight: 600, color: colors.textPrimary, - marginBottom: 4, + marginBottom: 3, }} > {feature.label} @@ -272,7 +288,7 @@ const ProductionStatus = ({ selectedOrganization, userdata, isCloud, theme }) =>
@@ -285,8 +301,8 @@ const ProductionStatus = ({ selectedOrganization, userdata, isCloud, theme }) => style={{ color: colors.success, fontSize: 20, flexShrink: 0 }} /> ) : ( - )}
@@ -304,80 +320,323 @@ const ProductionStatus = ({ selectedOrganization, userdata, isCloud, theme }) => /> {!isProdStatusOn && ( -
- )} - {/* CTA Section */} - {!isProdStatusOn && ( - )}
); }; +const AppRunsQueueCard = memo(({ environment, isAirGapped, isCloudSynching, totalRuns, limit, theme, navigate }) => { + + const usagePct = limit > 0 ? (totalRuns / limit) * 100 : 0; + const hardPauseLimit = limit * 2; + const hardPausePct = hardPauseLimit > 0 ? Math.min((totalRuns / hardPauseLimit) * 100, 100) : 0; + const mainBarPct = Math.min(usagePct, 100); + + const queueSize = environment?.queue !== undefined && environment?.queue !== null + ? Math.max(0, environment.queue) + : 0; + + const isThrottled = isCloudSynching ? false : (isAirGapped ? hardPausePct >= 100 : usagePct >= 100); + + let status, statusColor, statusBg; + if (isThrottled) { + status = 'Throttled'; + statusColor = '#ef4444'; + statusBg = 'rgba(239, 68, 68, 0.12)'; + } else if (!isCloudSynching && usagePct >= 80) { + status = 'Warning'; + statusColor = '#f59e0b'; + statusBg = 'rgba(245, 158, 11, 0.12)'; + } else { + status = 'Healthy'; + statusColor = theme.palette.green; + statusBg = `${theme.palette.green}1f`; + } + + const throttleRate = isThrottled ? '1/min' : '\u2014'; + const estClearTime = isThrottled && queueSize > 0 ? `${queueSize} min` : '\u2014'; + const mainBarColor = isThrottled ? '#ef4444' : usagePct >= 80 && !isCloudSynching ? '#f59e0b' : theme.palette.green; + + const envTypeLabel = environment?.run_type === 'cloud' ? 'Cloud' : 'On-prem'; + const envName = environment?.Name || environment?.name || 'Default'; + + const borderColor = theme.palette.slateGrayColor; + const trackBg = theme.palette.slateGrayColor; + const mutedText = theme.palette.text.secondary; + + return ( +
+ {/* Title row */} +
+ + {envTypeLabel} - {envName} · App runs / month + +
+ + {status} +
+
+ + {/* Main number */} +
+ + {totalRuns.toLocaleString()} + + + / {limit.toLocaleString()} + +
+ + {isAirGapped && !isCloudSynching && ( + + No throttle until {hardPauseLimit.toLocaleString()} runs · 2× your plan limit + + )} + + {/* Main usage bar */} +
+
+ {/* 80% threshold marker */} +
+
+
+ 0 + 80% threshold + {limit.toLocaleString()} +
+ + {/* Throttle limit row */} + {isAirGapped && ( + <> +
+ + Burst throttle threshold (2× limit) · workflows throttle to 1/min above this + + + {totalRuns.toLocaleString()} / {hardPauseLimit.toLocaleString()} + +
+ +
+
+
+ + )} + + {/* Alert box for Warning / Throttled */} + {status !== 'Healthy' && ( +
+ + {isThrottled ? 'Running slow \u2014 workflows are still running' : 'Approaching your monthly limit'} + + + {isThrottled + ? isAirGapped + ? `You've exceeded the burst threshold of ${hardPauseLimit.toLocaleString()} runs (2\u00d7 your plan limit). Your workflows are still running \u2014 there is no hard stop. Executions slow to 1 per minute until next month.` + : `You've exceeded your ${limit.toLocaleString()} monthly limit. Your instance keeps running \u2014 executions slow to 1 per minute until next month. Nothing is lost. You can view or clear the queue from the Locations tab.` + : isAirGapped + ? `You've used ${totalRuns.toLocaleString()} of ${limit.toLocaleString()} app runs. Workflows run normally \u2014 slowdown only begins at ${hardPauseLimit.toLocaleString()} runs (2\u00d7 your plan limit). No action needed.` + : `You've used ${totalRuns.toLocaleString()} of ${limit.toLocaleString()} app runs (${Math.max(0, limit - totalRuns).toLocaleString()} remaining). If you reach 100%, executions continue at a reduced rate of 1 per minute, nothing stops or is lost.` + } + + +
+ )} + + {/* Stats row */} +
+ {[ + { label: 'Queued jobs', value: queueSize }, + { label: 'Throttle rate', value: throttleRate }, + { label: 'Est. clear time', value: estClearTime }, + ].map((stat, i) => ( +
+ + {stat.label} + + + {stat.value} + +
+ ))} +
+
+ ); +}); + const Billing = memo((props) => { const { globalUrl, userdata, serverside, billingInfo, stripeKey,isLoaded, selectedOrganization, handleGetOrg, clickedFromOrgTab, removeCookie} = props; //const alert = useAlert(); @@ -413,7 +672,7 @@ const Billing = memo((props) => { const [statistics, setStatistics] = useState([]) const [monthlyAppRunsParent, setMonthlyAppRunsParent] = useState(0) const [monthlyAllSuborgExecutions, setMonthlyAllSuborgExecutions] = useState(0) - + const [billingEnvironments, setBillingEnvironments] = useState([]) useEffect(() => { if (monthlyAppRunsParent > 0 || monthlyAllSuborgExecutions > 0) { const percentage = ((monthlyAppRunsParent + monthlyAllSuborgExecutions) / userdata.app_execution_limit) * 100; @@ -525,6 +784,29 @@ const Billing = memo((props) => { }, []) + const getBillingEnvironments = () => { + fetch(globalUrl + "/api/v1/getenvironments", { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) return; + return response.json(); + }) + .then((responseJson) => { + if (responseJson && Array.isArray(responseJson)) { + setBillingEnvironments(responseJson); + } + }) + .catch((error) => { + console.log("Error fetching environments for billing:", error); + }); + }; + const getStats = (orgid) => { if (orgid === undefined || orgid === null) { @@ -563,6 +845,9 @@ const Billing = memo((props) => { useEffect(() => { if (selectedOrganization && selectedOrganization?.id?.length > 0) { getStats(selectedOrganization.id); + if (!isCloud) { + getBillingEnvironments(); + } } }, [selectedOrganization]); @@ -2387,6 +2672,17 @@ const Billing = memo((props) => { const isChildOrg = userdata?.active_org?.creator_org !== "" && userdata?.active_org?.creator_org !== undefined && userdata?.active_org?.creator_org !== null + const activeQueueEnvs = Array.isArray(billingEnvironments) ? billingEnvironments.filter(env => env != null && !env.archived && env.Type !== 'cloud') : []; + const totalQueueSize = activeQueueEnvs.reduce((sum, env) => sum + Math.max(0, env?.queue || 0), 0); + const aggregatedQueueEnv = { + Name: `${activeQueueEnvs.length} Runtime Location${activeQueueEnvs.length !== 1 ? 's' : ''}`, + run_type: 'on-prem', + queue: totalQueueSize, + }; + const appExecLimit = selectedOrganization?.sync_features?.app_executions?.limit ?? 0; + const isAirGapped = selectedOrganization != null && (selectedOrganization.cloud_sync_active === true || selectedOrganization.cloud_sync === true) ? false : appExecLimit < 300000 ? false : true; + const isCloudSynching = selectedOrganization != null && selectedOrganization.cloud_sync === true && appExecLimit >= 300000; + useEffect(() => { if (isChildOrg && currentTab === 0) { setCurrentTab(1); @@ -2800,6 +3096,28 @@ const Billing = memo((props) => {
) : null*/} + + {/* Queue Management */} + {!isCloud && activeQueueEnvs.length > 0 && !isChildOrg && ( +
+ + Queue Management + + + Real-time status of your app run usage and workflow queue across all runtime locations. + + +
+ )} + {!isChildOrg && isCloud && (
diff --git a/frontend/src/components/BillingStats.jsx b/frontend/src/components/BillingStats.jsx index d2d2e304..05796cbd 100644 --- a/frontend/src/components/BillingStats.jsx +++ b/frontend/src/components/BillingStats.jsx @@ -92,6 +92,11 @@ const AppStats = (defaultprops) => { const statKey = syncStats === true ? "onprem_stats" : "daily_statistics" const dailyStats = inputdata[statKey] if (dailyStats === undefined || dailyStats === null) { + setAppruns(undefined) + setWorkflowRuns(undefined) + setSubflowRuns(undefined) + setChildOrgsAppRuns(undefined) + setApprunCosts(undefined) return } @@ -378,6 +383,17 @@ const AppStats = (defaultprops) => { return } + if (syncStats && (statistics[statKey] === undefined || statistics[statKey] === null)) { + setOnpremAppRuns(0) + setFilteredStatistics(statistics) + setAppruns(undefined) + setWorkflowRuns(undefined) + setSubflowRuns(undefined) + setChildOrgsAppRuns(undefined) + setApprunCosts(undefined) + return + } + // Calculate month to date cost var mtd_cost = 0 for (let key in statistics[statKey]) { diff --git a/frontend/src/components/CacheView.jsx b/frontend/src/components/CacheView.jsx index ef5719e7..4c9fd46c 100644 --- a/frontend/src/components/CacheView.jsx +++ b/frontend/src/components/CacheView.jsx @@ -1,6 +1,8 @@ import React, { useState, useEffect, useContext, memo } from "react"; import { makeStyles } from "@mui/styles"; import { getTheme } from "../theme.jsx"; +import SubOrgDistributionDialog from "./SubOrgDistributionDialog.jsx"; +import DeleteConfirmDialog from "./DeleteConfirmDialog.jsx"; import { toast } from 'react-toastify'; import ReactJson from "react-json-view-ssr"; @@ -17,8 +19,6 @@ import { Button, Tabs, Tab, - List, - ListItem, ListItemText, IconButton, Dialog, @@ -82,6 +82,7 @@ import { Hub as HubIcon, Key as KeyIcon, FlashOn as FlashOnIcon, + Search as SearchIcon, } from "@mui/icons-material"; import { Context } from "../context/ContextApi.jsx"; @@ -127,6 +128,9 @@ const CacheView = memo((props) => { const [showDistributionPopup, setShowDistributionPopup] = useState(false); const [selectedSubOrg, setSelectedSubOrg] = useState([]); const [selectedCacheKey, setSelectedCacheKey] = useState(""); + const [deleteConfirmOpen, setDeleteConfirmOpen] = useState(false); + const [deleteConfirmTarget, setDeleteConfirmTarget] = useState(null); + const [distribOrgOrder, setDistribOrgOrder] = useState([]); const [totalAmount, setTotalAmount] = useState(0); const [page, setPage] = useState(0); const [pageSize, setPageSize] = useState(50) @@ -233,7 +237,7 @@ const CacheView = memo((props) => { { "name": "Enrich", - "description": "Enriches the data. Only runs on valid JSON data AND if the 'enrichment' field does not exist.", + "description": "Enriches the data. Uses regex keys and runs a workflow in the background. Added to the 'enrichments' key.", "type": "singul", "options": [{ "key": "", @@ -331,6 +335,14 @@ const CacheView = memo((props) => { // In order to make linking weird urls from workflow page work. if (urlParams.get("src") == "workflow") { + if (categoryParam === "OCSF") { + const newParam = "shuffle-security incidents" + + urlParams.set("category", newParam) + window.history.replaceState({}, '', `${window.location.pathname}?${urlParams.toString()}`) + categoryParam = newParam + } + if (categoryParam?.toLowerCase().startsWith("list")) { const newParam = categoryParam.substring(5).replaceAll("%20", "_") @@ -570,17 +582,26 @@ const CacheView = memo((props) => { .then((response) => { if (response.status === 200) { if (refreshList === undefined || refreshList === null || refreshList === true) { - toast.success("Deleted datastore entry"); setTimeout(() => { listOrgCache(orgId, selectedCategory, 0, pageSize, page) }, 1000); } } else { + if (refreshList === undefined || refreshList === null || refreshList === true) { + setTimeout(() => { + listOrgCache(orgId, selectedCategory, 0, pageSize, page) + }, 1000); + } toast.error(`Failed deleting entry ${key} in category ${itemCategory || selectedCategory}. If this persists, please contact support@shuffler.io.`) } }) .catch((error) => { + if (refreshList === undefined || refreshList === null || refreshList === true) { + setTimeout(() => { + listOrgCache(orgId, selectedCategory, 0, pageSize, page) + }, 1000); + } toast(error.toString()); }); }; @@ -831,6 +852,11 @@ const CacheView = memo((props) => { Category: {dataValue.category} : null} + {dataValue?.enrichments !== undefined && dataValue?.enrichments !== null && dataValue.enrichments.length > 0 ? + + Enrichments: {dataValue.enrichments.length} + + : null} {dataValue?.tags !== undefined && dataValue?.tags !== null && dataValue?.tags?.length > 0 ?
@@ -900,33 +926,6 @@ const CacheView = memo((props) => { ); - const handleSelectSubOrg = (id, action) => { - if (action === "all") { - const childOrgs = userdata.orgs.filter( - (data) => data.creator_org === userdata.active_org.id - ); - setSelectedSubOrg((prev) => { - if (prev.length === childOrgs.length) { - // If all child orgs are already selected, clear the selection - return []; - } else { - // Otherwise, select all child org IDs - return childOrgs.map((data) => data.id); - } - }); - } else if (action === "none") { - setSelectedSubOrg([]); - } else { - setSelectedSubOrg((prev) => { - if (prev.includes(id)) { - return prev.filter((data) => data !== id); - } else { - return [...prev, id]; - } - }); - } - }; - const changeDistribution = (id, selectedSubOrg) => { editFileConfig(id, [...new Set(selectedSubOrg)], selectedCategory) @@ -940,8 +939,6 @@ const CacheView = memo((props) => { selected_suborgs: selectedSubOrg, category: category === undefined || category === "" || category === "default" ? "" : category, } - - console.log("data: ", data); const url = `${globalUrl}/api/v1/orgs/${orgId}/cache/config`; @@ -975,98 +972,41 @@ const CacheView = memo((props) => { }; - const cacheDistributionModal = showDistributionPopup ? ( - setShowDistributionPopup(false)} - PaperProps={{ - sx: { - borderRadius: theme?.palette?.DialogStyle?.borderRadius, - border: theme?.palette?.DialogStyle?.border, - fontFamily: theme?.typography?.fontFamily, - backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, - zIndex: 1000, - minWidth: "600px", - minHeight: "320px", - overflow: "auto", - '& .MuiDialogContent-root': { - backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, - }, - '& .MuiDialogTitle-root': { - backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, - }, - '& .MuiDialogActions-root': { - backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, - }, - }, - }} - > - - - Select sub-org to distribute Datastore key - - - - {handleSelectSubOrg(null, "none")}}>None - {handleSelectSubOrg(null, "all")}}>All - {userdata.orgs.map((data, index) => { - if (data.creator_org !== userdata.active_org.id) { - return null; - } - const imagesize = 22; - const imageStyle = { - width: imagesize, - height: imagesize, - pointerEvents: "none", - marginRight: 10, - marginLeft: data.id === userdata.active_org.id ? 0 : 20, - }; - const image = data.image === "" ? ( - {data.name} - ) : ( - {data.name} - ); + const deleteConfirmDialog = ( + { setDeleteConfirmOpen(false); setDeleteConfirmTarget(null); }} + onConfirm={() => { + if (deleteConfirmTarget?.bulk) { + const itemsToDelete = selectedRows.map(rowId => + listCache.find(item => `${item.key}_${item.category || ""}` === rowId) + ).filter(Boolean); - return ( - handleSelectSubOrg(data.id)} - style={{ display: "flex", alignItems: "center" }} - > - - {image} - {data.name} - - ); - })} + const count = itemsToDelete.length; + setSelectedRows([]); + itemsToDelete.forEach(item => deleteEntry(orgId, item.key, item.category, false)); -
- - -
- - - ) : null; + setTimeout(() => { + listOrgCache(orgId, selectedCategory, 0, pageSize, page); + toast.success("Deleted " + count + " keys from datastore"); + }, 3000); + } else { + deleteEntry(orgId, deleteConfirmTarget.key, deleteConfirmTarget.category); + } + setDeleteConfirmOpen(false); + setDeleteConfirmTarget(null); + }} + title={deleteConfirmTarget?.bulk ? `Delete ${selectedRows?.length} Key${selectedRows?.length > 1 ? "s" : ""}?` : "Delete Key?"} + description={ + deleteConfirmTarget?.bulk + ? <>Are you sure you want to delete {selectedRows?.length} key{selectedRows?.length > 1 ? "s" : ""}? + : <>Are you sure you want to delete {deleteConfirmTarget?.key}? + } + warningText="This cannot be undone. Any workflows using these keys will lose access." + /> + ); const saveAutomation = (allAutomation, settings) => { // Check if icon is a string. Otherwise make it empty. @@ -1759,6 +1699,10 @@ const CacheView = memo((props) => { enableClipboard={(copy) => { handleReactJsonClipboard(copy) }} + onSelect={(select) => { + //currentParams.set("category", selectedCategory); + //HandleJsonCopy(validate.result, select, "exec"); + }} collapseStringsAfterLength={theme.palette.jsonCollapseStringsAfterLength} iconStyle={theme.palette.jsonIconStyle} displayDataTypes={false} @@ -1935,6 +1879,7 @@ const CacheView = memo((props) => { "workflow_id": data.workflow_id, "category": data.category, "tags": data.tags, + "enrichments": data.enrichments, }) setValue(newvalue) setModalOpen(true) @@ -2019,7 +1964,8 @@ const CacheView = memo((props) => { onClick={(e) => { e.preventDefault() e.stopPropagation() - deleteEntry(orgId, data.key, data.category) + setDeleteConfirmTarget({ key: data.key, category: data.category }) + setDeleteConfirmOpen(true) }} > { style={{ margin: "auto" }} color="secondary" onClick={() => { - setShowDistributionPopup(true) + setShowDistributionPopup(true); + let initialSelected = []; if(data?.suborg_distribution?.length > 0){ - setSelectedSubOrg(data.suborg_distribution) - }else{ - setSelectedSubOrg([]) + initialSelected = data.suborg_distribution; } - setSelectedCacheKey(data.key) + setSelectedSubOrg(initialSelected); + setSelectedCacheKey(data.key); + const suborgs = (userdata?.orgs || []).filter(o => o.creator_org === userdata?.active_org?.id); + const sorted = [...suborgs].sort((a, b) => { + const aS = initialSelected.includes(a.id); + const bS = initialSelected.includes(b.id); + if (aS !== bS) return bS - aS; + return a.name.localeCompare(b.name); + }); + setDistribOrgOrder(sorted.map(o => o.id)); }} /> @@ -2116,6 +2070,7 @@ const CacheView = memo((props) => { var previousgroup = "" const isAutomating = categoryAutomations?.find((automation) => automation.enabled) !== undefined + const isAutomatingAccess = categoryConfig?.settings?.timeout >= 60 || categoryConfig?.settings?.public === true ? true : false return (
{ apps={apps} /> - {cacheDistributionModal} + { setShowDistributionPopup(false); setSelectedCacheKey(""); }} + title="Distribute Datastore Key to Sub-Organizations" + extraInfo={selectedCacheKey ? `Selected Key: ${selectedCacheKey}` : null} + orgs={distribOrgOrder.map(id => (userdata?.orgs || []).find(o => o.id === id)).filter(Boolean)} + selectedOrgIds={selectedSubOrg} + onSelectionChange={setSelectedSubOrg} + onSave={(ids) => { changeDistribution(selectedCacheKey, ids); }} + /> + {deleteConfirmDialog}
@@ -2204,7 +2169,6 @@ const CacheView = memo((props) => { height: 35, textTransform: 'none', - //border: isAutomating ? `1px solid ${theme.palette.primary.main}` : null, }} variant="outlined" color="secondary" @@ -2272,12 +2236,12 @@ const CacheView = memo((props) => { datastoreCategories !== null && datastoreCategories.length > 1 ? ( - + { marginLeft: 3, }} variant="outlined" - color="secondary" + color={isAutomatingAccess ? "primary" : "secondary"} disabled={selectedCategory === undefined || selectedCategory === "" || selectedCategory === "default"} onClick={() => { setShowSettingsMenu(true) }} > - + @@ -2877,27 +2841,8 @@ const CacheView = memo((props) => { - {formMessage} -
- : null + + : null }
) diff --git a/frontend/src/components/DeleteConfirmDialog.jsx b/frontend/src/components/DeleteConfirmDialog.jsx new file mode 100644 index 00000000..459f077b --- /dev/null +++ b/frontend/src/components/DeleteConfirmDialog.jsx @@ -0,0 +1,61 @@ +import React, { memo, useContext } from 'react'; +import { + Button, + Dialog, + DialogTitle, + DialogContent, + DialogActions, + Typography, +} from '@mui/material'; +import { getTheme } from '../theme.jsx'; +import { Context } from '../context/ContextApi.jsx'; + +const DeleteConfirmDialog = memo(({ open, onClose, onConfirm, title, description, warningText }) => { + const { themeMode, brandColor } = useContext(Context); + const theme = getTheme(themeMode, brandColor); + + return ( + + + {title} + + + {description} + {warningText && ( + + {warningText} + + )} + + + + + + + ); +}); + +export default DeleteConfirmDialog; diff --git a/frontend/src/components/DiscordChat.jsx b/frontend/src/components/DiscordChat.jsx index 2bbc3b2c..c220d36a 100644 --- a/frontend/src/components/DiscordChat.jsx +++ b/frontend/src/components/DiscordChat.jsx @@ -3,11 +3,8 @@ import algoliasearch from 'algoliasearch'; import theme from '../theme.jsx'; import { InstantSearch, connectSearchBox, connectHits } from 'react-instantsearch-dom'; import { - Grid, - Paper, TextField, Typography, - Button, InputAdornment, Avatar, List, @@ -16,6 +13,7 @@ import { ListItemText, } from '@mui/material'; import { Search as SearchIcon } from '@mui/icons-material'; +import SearchContactForm from '../components/SearchContactForm.jsx'; import useDebouncedCallback from '../utils/useDebouncedCallback.jsx'; @@ -23,52 +21,9 @@ const searchClient = algoliasearch("JNSS5CFDZZ", "1e5f29b1550939855de5915eac3bf5 const DiscordChat = props => { const { isMobile, globalUrl } = props - const [value, setValue] = useState(""); - const [formMail, setFormMail] = React.useState(""); - const [message, setMessage] = React.useState(""); - const [formMessage, setFormMessage] = React.useState(""); - const buttonStyle = {borderRadius: 30, height: 50, width: 220, margin: isMobile ? "15px auto 15px auto" : 20, fontSize: 18,} const borderRadius = 3 - const submitContact = (email, message) => { - const data = { - "firstname": "", - "lastname": "", - "title": "", - "companyname": "", - "email": email, - "phone": "", - "message": message, - } - - const errorMessage = "Something went wrong. Please contact frikky@shuffler.io directly." - - fetch(globalUrl+"/api/v1/contact", { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(data), - }) - .then(response => response.json()) - .then(response => { - if (response.success === true) { - setFormMessage(response.reason) - //toast("Thanks for submitting!") - } else { - setFormMessage(errorMessage) - } - - setFormMail("") - setMessage("") - }) - .catch(error => { - setFormMessage(errorMessage) - console.log(error) - }); - } - const SearchBox = ({ currentRefinement, refine }) => { const [inputValue, setInputValue] = useState(""); const debouncedRefine = useDebouncedCallback((value) => refine(value), 300); @@ -168,61 +123,7 @@ const DiscordChat = props => {
-
- - Can't find what you're looking for? - -
- setFormMail(e.target.value)} - /> - setMessage(e.target.value)} - /> -
- - {formMessage} -
+ {/* Search by diff --git a/frontend/src/components/DocsGrid.jsx b/frontend/src/components/DocsGrid.jsx index 57616bd2..64cbd8c4 100644 --- a/frontend/src/components/DocsGrid.jsx +++ b/frontend/src/components/DocsGrid.jsx @@ -4,6 +4,7 @@ import theme from '../theme.jsx'; import ReactGA from 'react-ga4'; import {Link} from 'react-router-dom'; import { removeQuery } from '../components/ScrollToTop.jsx'; +import SearchContactForm from '../components/SearchContactForm.jsx'; import { Search as SearchIcon, CloudQueue as CloudQueueIcon, Code as CodeIcon, Close as CloseIcon, Folder as FolderIcon, LibraryBooks as LibraryBooksIcon } from '@mui/icons-material'; import aa from 'search-insights' @@ -30,61 +31,19 @@ import { useDebouncedCallback } from "../utils/useDebouncedCallback.jsx"; -const searchClient = algoliasearch("JNSS5CFDZZ", "c8f882473ff42d41158430be09ec2b4e") +const searchClient = algoliasearch("JNSS5CFDZZ", "eb5fd80aa6ed5ab4730d836cff3ea283") const DocsGrid = props => { const { maxRows, showName, showSuggestion, isMobile, globalUrl, parsedXs, userdata, } = props const rowHandler = maxRows === undefined || maxRows === null ? 50 : maxRows const xs = parsedXs === undefined || parsedXs === null ? isMobile ? 6 : 2 : parsedXs //const [apps, setApps] = React.useState([]); //const [filteredApps, setFilteredApps] = React.useState([]); - const [formMail, setFormMail] = React.useState(""); - const [message, setMessage] = React.useState(""); - const [formMessage, setFormMessage] = React.useState(""); - const buttonStyle = {borderRadius: 30, height: 50, width: 220, margin: isMobile ? "15px auto 15px auto" : 20, fontSize: 18,} const innerColor = "rgba(255,255,255,0.65)" const borderRadius = 3 window.title = "Shuffle | Apps | Find and integrate any app" - const submitContact = (email, message) => { - const data = { - "firstname": "", - "lastname": "", - "title": "", - "companyname": "", - "email": email, - "phone": "", - "message": message, - } - - const errorMessage = "Something went wrong. Please contact frikky@shuffler.io directly." - - fetch(globalUrl+"/api/v1/contact", { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(data), - }) - .then(response => response.json()) - .then(response => { - if (response.success === true) { - setFormMessage(response.reason) - //toast("Thanks for submitting!") - } else { - setFormMessage(errorMessage) - } - - setFormMail("") - setMessage("") - }) - .catch(error => { - setFormMessage(errorMessage) - console.log(error) - }); - } - const SearchBox = ({currentRefinement, refine, isSearchStalled} ) => { var defaultSearch = "" if (window !== undefined && window.location !== undefined && window.location.search !== undefined && window.location.search !== null) { @@ -124,8 +83,15 @@ const DocsGrid = props => { placeholder="Search our Documentation..." id="shuffle_search_field" onChange={(event) => { - removeQuery("q") - debouncedRefine(event.currentTarget.value) + const value = event.currentTarget.value + debouncedRefine(value) + const urlSearchParams = new URLSearchParams(window.location.search) + if (value) { + urlSearchParams.set("q", value) + } else { + urlSearchParams.delete("q") + } + window.history.replaceState(null, "", value ? `?${urlSearchParams.toString()}` : window.location.pathname) }} onKeyDown={(event) => { if(event.key === "Enter") { @@ -291,62 +257,8 @@ const DocsGrid = props => { {showSuggestion === true ? -
- - Can't find what you're looking for? - -
- setFormMail(e.target.value)} - /> - setMessage(e.target.value)} - /> -
- - {formMessage} -
- : null + + : null } {/* diff --git a/frontend/src/components/EnvironmentTab.jsx b/frontend/src/components/EnvironmentTab.jsx index 58dd47d5..6d17f383 100644 --- a/frontend/src/components/EnvironmentTab.jsx +++ b/frontend/src/components/EnvironmentTab.jsx @@ -36,6 +36,7 @@ import { ExpandLess as ExpandLessIcon, ExpandMore as ExpandMoreIcon, Delete as DeleteIcon, + Computer as ComputerIcon, } from "@mui/icons-material"; import { toast } from 'react-toastify'; import { Context } from '../context/ContextApi.jsx'; @@ -49,6 +50,7 @@ const EnvironmentTab = memo((props) => { const [modalUser, setModalUser] = React.useState({}); const [loginInfo, setLoginInfo] = React.useState(""); const [modalOpen, setModalOpen] = React.useState(false); + const [sensorGroup, setSensorGroup] = React.useState(false); const [showLoader, setShowLoader] = useState(true) const [commandController, setCommandController] = React.useState({ pipelines: false, @@ -535,13 +537,36 @@ const EnvironmentTab = memo((props) => { }, }, }} + style={{ + }} > - - Add Location + + Add Location + -
- Location Name + + {sensorGroup ? + 'With "Sensor Groups" enabled, Runtime Locations allows you to run a lightweight log-collector and response agent onprem.' + : + 'Runtime Locations are a way to run automation in Shuffle. By default, it runs in Docker/Kubernetes and allows you to run AI Agents and Workflows in your designated datacenter.' + } + +
+ + {sensorGroup ? + "Sensor Group name" + : + "Location Name" + } + { }} required fullWidth={true} - placeholder="datacenter froglantern" + placeholder="automation location 3" id="environment_name" margin="normal" variant="outlined" - onChange={(event) => + onChange={(event) => { changeModalData("environment", event.target.value) - } + setUpdate(Math.random()) + }} />
{loginInfo} @@ -576,12 +602,13 @@ const EnvironmentTab = memo((props) => { @@ -956,7 +983,7 @@ const EnvironmentTab = memo((props) => { { > + + + : environment.run_type === "cloud" || environment.name === "Cloud" ? ( @@ -1275,6 +1307,9 @@ const EnvironmentTab = memo((props) => { + : + environment?.sensor_group === true ? + "N/A" : environment?.data_lake?.enabled && environment?.archived !== true ? ( { /> {
+ {environment?.sensor_group === true ? +
+ + Sensor Group - Host controls available in Shuffle Security + + + Total registered hosts: {environment?.sensor_hosts?.length || 0}.
Host management and response actions is done in Shuffle Security. Click the link above to manage. + +
+ :
Self-Hosted Orborus instance @@ -1703,6 +1748,7 @@ const EnvironmentTab = memo((props) => { }
+ }
{currentEnvQueue.length === 0 ? null : diff --git a/frontend/src/components/Files.jsx b/frontend/src/components/Files.jsx index f98e8e22..5428a881 100644 --- a/frontend/src/components/Files.jsx +++ b/frontend/src/components/Files.jsx @@ -29,6 +29,8 @@ import { Menu, Pagination, PaginationItem, + Box, + InputAdornment, } from "@mui/material"; import { DataGrid } from "@mui/x-data-grid"; @@ -45,9 +47,12 @@ import { Clear as ClearIcon, Add as AddIcon, SelectAll, + Search as SearchIcon, } from "@mui/icons-material"; import Dropzone from "../components/Dropzone.jsx"; +import SubOrgDistributionDialog from "./SubOrgDistributionDialog.jsx"; +import DeleteConfirmDialog from "./DeleteConfirmDialog.jsx"; import ShuffleCodeEditor from "../components/ShuffleCodeEditor1.jsx"; import {getTheme} from "../theme.jsx"; import { Context } from "../context/ContextApi.jsx"; @@ -82,6 +87,10 @@ const Files = memo((props) => { const [showDistributionPopup, setShowDistributionPopup] = useState(false) const [selectedSubOrg, setSelectedSubOrg] = useState([]) const [fileIdSelectedForDistribution, setFileIdSelectedForDistribution] = useState("") + const [fileNameSelectedForDistribution, setFileNameSelectedForDistribution] = useState("") + const [distribOrgOrder, setDistribOrgOrder] = useState([]) + const [deleteConfirmOpen, setDeleteConfirmOpen] = useState(false) + const [deleteConfirmTarget, setDeleteConfirmTarget] = useState(null) const [totalAmount, setTotalAmount] = useState(0); const [page, setPage] = useState(0); const [pageSize, setPageSize] = useState(50) @@ -326,7 +335,7 @@ const [filesLoaded, setFilesLoaded] = useState(false); navigator.clipboard.writeText(file.id); document.execCommand("copy"); - toast(file.id + " copied to clipboard"); + toast.info(file.id + " copied to clipboard"); }} > { e.stopPropagation(); e.preventDefault(); - deleteFile(file.id, true); + setDeleteConfirmTarget({ id: file.id, filename: file.filename }); + setDeleteConfirmOpen(true); }} > o.creator_org === userdata.active_org.id) + .map(o => o.id) + ) }} /> @@ -534,7 +550,7 @@ const [filesLoaded, setFilesLoaded] = useState(false); }) .then((responseJson) => { if (responseJson.success === true) { - toast("Successfully updated file"); + toast.success("Successfully updated file"); } }) .catch((error) => { @@ -848,125 +864,39 @@ const [filesLoaded, setFilesLoaded] = useState(false); : null - const handleSelectSubOrg = (id, action) => { - if (action === "all") { - const childOrgs = userdata.orgs.filter( - (data) => data.creator_org === userdata.active_org.id - ); - setSelectedSubOrg((prev) => { - if (prev.length === childOrgs.length) { - // If all child orgs are already selected, clear the selection - return []; - } else { - // Otherwise, select all child org IDs - return childOrgs.map((data) => data.id); - } - }); - } else if (action === "none") { - setSelectedSubOrg([]); - } else { - setSelectedSubOrg((prev) => { - if (prev.includes(id)) { - return prev.filter((data) => data !== id); - } else { - return [...prev, id]; - } - }); - } - }; - - const fileDistributionModal = showDistributionPopup ? ( - setShowDistributionPopup(false)} - PaperProps={{ - sx: { - borderRadius: theme?.palette?.DialogStyle?.borderRadius, - border: theme?.palette?.DialogStyle?.border, - fontFamily: theme?.typography?.fontFamily, - backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, - zIndex: 1000, - minWidth: "600px", - minHeight: "320px", - overflow: "auto", - '& .MuiDialogContent-root': { - backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, - }, - '& .MuiDialogTitle-root': { - backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, - }, - '& .MuiDialogActions-root': { - backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, - }, - }, - }} - > - - - Select sub-org to distribute files - - - - {handleSelectSubOrg(null, "none")}}>None - {handleSelectSubOrg(null, "all")}}>All - {userdata.orgs.map((data, index) => { - if (data.creator_org !== userdata.active_org.id) { - return null; - } - - const imagesize = 22; - const imageStyle = { - width: imagesize, - height: imagesize, - pointerEvents: "none", - marginRight: 10, - marginLeft: data.id === userdata.active_org.id ? 0 : 20, - }; - - const image = data.image === "" ? ( - {data.name} - ) : ( - {data.name} - ); - - return ( - handleSelectSubOrg(data.id)} - style={{ display: "flex", alignItems: "center" }} - > - - {image} - {data.name} - - ); - })} - -
- - -
- - - - ): null + const deleteConfirmDialog = ( + { setDeleteConfirmOpen(false); setDeleteConfirmTarget(null); }} + onConfirm={() => { + if (deleteConfirmTarget?.bulk) { + const toDelete = [...selectedRows]; + const count = toDelete.length; + setSelectedRows([]); + toDelete.forEach(fileId => deleteFile(fileId, false)); + setTimeout(() => { + getFiles(selectedCategory); + toast.success('Deleted ' + count + ' file' + (count > 1 ? 's' : '')); + }, 3000); + } else { + deleteFile(deleteConfirmTarget.id, true); + } + setDeleteConfirmOpen(false); + setDeleteConfirmTarget(null); + }} + title={ + deleteConfirmTarget?.bulk + ? 'Delete ' + selectedRows?.length + ' File' + (selectedRows?.length > 1 ? 's' : '') + '?' + : 'Delete File?' + } + description={ + deleteConfirmTarget?.bulk + ? <>Are you sure you want to delete {selectedRows?.length} file{selectedRows?.length > 1 ? 's' : ''}? + : <>Are you sure you want to delete {deleteConfirmTarget?.filename}? + } + warningText="This cannot be undone." + /> + ); const deleteFile = (fileId, showSinglDeleteToast) => { @@ -1295,7 +1225,17 @@ const [filesLoaded, setFilesLoaded] = useState(false); }} onDrop={uploadFile} > - {fileDistributionModal} + { setShowDistributionPopup(false); }} + title="Distribute File to Sub-Organizations" + extraInfo={fileNameSelectedForDistribution ? `Selected File: ${fileNameSelectedForDistribution}` : null} + orgs={distribOrgOrder.map(id => (userdata?.orgs || []).find(o => o.id === id)).filter(Boolean)} + selectedOrgIds={selectedSubOrg} + onSelectionChange={setSelectedSubOrg} + onSave={(ids) => { changeDistribution(fileIdSelectedForDistribution, ids); setShowDistributionPopup(false); }} + /> + {deleteConfirmDialog}
@@ -1559,17 +1499,20 @@ const [filesLoaded, setFilesLoaded] = useState(false); autoFocus />} - + {openEditor === true && fileContent !== undefined && fileContent !== null && fileContent.length > 0 ? + + : null} + {isSelectedFiles?null: { - setSelectedRows([]); - getFiles(selectedCategory); - toast.success( - `Deleted ${selectedRows.length} file${selectedRows.length === 1 ? "" : "s"}` - ); - }, 2500); - } - } - }} + setDeleteConfirmTarget({ bulk: true }); + setDeleteConfirmOpen(true); + }} variant={"outlined"} color="secondary" startIcon={ diff --git a/frontend/src/components/HealthPage.jsx b/frontend/src/components/HealthPage.jsx index b60a46be..8018237f 100644 --- a/frontend/src/components/HealthPage.jsx +++ b/frontend/src/components/HealthPage.jsx @@ -1,7 +1,17 @@ -import React, { useEffect, useState, useCallback } from 'react'; +import React, { useEffect, useState, useCallback, useMemo, useContext } from 'react'; +import { Context } from '../context/ContextApi'; +import { useNavigate } from 'react-router-dom'; import { toast } from "react-toastify"; import { - CheckOutlined as CheckOutlinedIcon, + CheckCircle as CheckCircleIcon, + Error as ErrorIcon, + Warning as WarningIcon, + AccountTree as WorkflowIcon, + Apps as AppsIcon, + Storage as StorageIcon, + FolderOpen as FileIcon, + FindInPage as SearchIcon, + ContentCopy as CopyIcon, } from '@mui/icons-material'; import { @@ -9,52 +19,274 @@ import { ButtonGroup, Typography, LinearProgress, + Chip, + Tooltip, + Select, + MenuItem, + FormControl, + TextField, + Popover, } from "@mui/material"; +import { CalendarMonth as CalendarIcon } from '@mui/icons-material'; + import HealthBarChart from '../components/HealthBarChart.jsx'; -import LiveExecutionsChart from '../components/LiveExecutionsGraph.jsx'; +import LiveExecutionsChart from './LiveExecutionsGraph.jsx'; + + +const STATUS_STYLE = { + operational: { color: '#00F670', label: 'Operational', bg: 'rgba(0, 246, 112, 0.07)' }, + degraded: { color: '#FFD700', label: 'Degraded', bg: 'rgba(255, 215, 0, 0.07)' }, + outage: { color: '#FF354C', label: 'Outage', bg: 'rgba(255, 53, 76, 0.07)' }, +}; + +const REGION_DOMAIN = { + 'london': 'https://shuffler.io', + 'california': 'https://california.shuffler.io', + 'EU': 'https://frankfurt.shuffler.io', + 'canada': 'https://ca.shuffler.io', + 'australia': 'https://au.shuffler.io', +}; + +const RANGE_MILLIS = { + '24hr': 24 * 60 * 60 * 1000, + '7day': 7 * 24 * 60 * 60 * 1000, + '30d': 30 * 24 * 60 * 60 * 1000, + '90d': 90 * 24 * 60 * 60 * 1000, + '180d': 180 * 24 * 60 * 60 * 1000, + '365d': 365 * 24 * 60 * 60 * 1000, +}; + +const SERVICE_CONFIG = [ + { + key: 'workflows', + label: 'Workflows', + Icon: WorkflowIcon, + sloTarget: 99.95, + isHealthy: (item) => item.workflows?.run_finished === true, + getOperations: (item) => [ + { key: 'create', label: 'Create', value: item.workflows?.create }, + { key: 'run', label: 'Execute', value: item.workflows?.run }, + { key: 'run_finished', label: 'Completed', value: item.workflows?.run_finished }, + { key: 'delete', label: 'Delete', value: item.workflows?.delete }, + ], + getExtra: (item) => { + const took = item.workflows?.execution_took; + return (took != null && took > 0) ? `Exec time: ${Number(took).toFixed(2)}s` : null; + }, + getIds: (item) => [ + { label: 'Execution ID', value: item.workflows?.execution_id }, + { label: 'Workflow ID', value: item.workflows?.workflow_id }, + ].filter(id => !!id.value), + getErrors: (item) => { + const e = item.workflows?.error; + if (!e) return []; + return [ + { key: 'create', label: 'Create', msg: e.create }, + { key: 'run', label: 'Execute', msg: e.run }, + { key: 'run_finished', label: 'Completed', msg: e.run_finished }, + { key: 'workflow_validation', label: 'Validation', msg: e.workflow_validation }, + { key: 'delete', label: 'Delete', msg: e.delete }, + ].filter(err => !!err.msg); + }, + }, + { + key: 'apps', + label: 'Apps', + Icon: AppsIcon, + sloTarget: 99.95, + isHealthy: (item) => { const a = item.apps; return !!(a && a.create && a.run && a.delete); }, + getOperations: (item) => [ + { key: 'create', label: 'Create', value: item.apps?.create }, + { key: 'validate', label: 'Validate', value: item.apps?.validate }, + { key: 'run', label: 'Execute', value: item.apps?.run }, + { key: 'read', label: 'Read', value: item.apps?.read }, + { key: 'delete', label: 'Delete', value: item.apps?.delete }, + ], + getExtra: () => null, + getIds: (item) => [ + { label: 'App ID', value: item.apps?.app_id }, + { label: 'Execution ID', value: item.apps?.execution_id }, + ].filter(id => !!id.value), + getErrors: (item) => { + const e = item.apps?.error; + if (!e) return []; + return [ + { key: 'create', label: 'Create', msg: e.create }, + { key: 'validate', label: 'Validate', msg: e.validate }, + { key: 'run', label: 'Execute', msg: e.run }, + { key: 'read', label: 'Read', msg: e.read }, + { key: 'delete', label: 'Delete', msg: e.delete }, + ].filter(err => !!err.msg); + }, + }, + { + key: 'datastore', + label: 'Datastore', + Icon: StorageIcon, + sloTarget: 99.95, + isHealthy: (item) => { const d = item.datastore; return !!(d && d.create && d.read && d.delete); }, + getOperations: (item) => [ + { key: 'create', label: 'Create', value: item.datastore?.create }, + { key: 'read', label: 'Read', value: item.datastore?.read }, + { key: 'delete', label: 'Delete', value: item.datastore?.delete }, + ], + getExtra: () => null, + getIds: () => [], + getErrors: (item) => { + const e = item.datastore?.error; + if (!e) return []; + return [ + { key: 'create', label: 'Create', msg: e.create }, + { key: 'read', label: 'Read', msg: e.read }, + { key: 'delete', label: 'Delete', msg: e.delete }, + ].filter(err => !!err.msg); + }, + }, + { + key: 'fileops', + label: 'File Storage', + Icon: FileIcon, + sloTarget: 99.95, + isHealthy: (item) => { const f = item.fileops; return !!(f && f.create && f.get_file && f.delete); }, + getOperations: (item) => [ + { key: 'create', label: 'Create', value: item.fileops?.create }, + { key: 'get_file', label: 'Upload/Fetch', value: item.fileops?.get_file }, + { key: 'delete', label: 'Delete', value: item.fileops?.delete }, + ], + getExtra: () => null, + getIds: (item) => [ + { label: 'File ID', value: item.fileops?.fileId }, + ].filter(id => !!id.value), + getErrors: (item) => { + const e = item.fileops?.error; + if (!e) return []; + return [ + { key: 'create', label: 'Create', msg: e.create }, + { key: 'upload', label: 'Upload/Fetch', msg: e.upload }, + { key: 'delete', label: 'Delete', msg: e.delete }, + ].filter(err => !!err.msg); + }, + }, +]; + +const OPENSEARCH_CONFIG = { + key: 'opensearch', + label: 'OpenSearch', + Icon: SearchIcon, + sloTarget: 99.95, + isHealthy: (item) => item.opnsearch?.status === 'green', + getOperations: (item) => { + const s = item.opnsearch?.status; + return [{ key: 'cluster', label: 'Cluster', value: s === 'green' ? true : s === 'yellow' ? 'warn' : false }]; + }, + getExtra: (item) => item.opnsearch?.status ? `Cluster: ${item.opnsearch.status}` : null, + getIds: () => [], + getErrors: () => [], +}; + +const computeAvgUptime = (chartData) => { + if (!chartData || chartData.length === 0) return 100; + const withData = chartData.filter(d => d.avgRunFinished !== null); + if (withData.length === 0) return 100; + return parseFloat((withData.reduce((acc, d) => acc + d.avgRunFinished, 0) / withData.length).toFixed(2)); +}; + +const getStatusKey = (uptime, sloTarget) => { + if (uptime >= sloTarget) return 'operational'; + if (uptime >= 95) return 'degraded'; + return 'outage'; +}; + +// --- const HealthPage = (props) => { - const { userdata, globalUrl } = props; + const { userdata, isLoaded } = props; + const navigate = useNavigate(); + const { leftSideBarOpenByClick } = useContext(Context); const [healthData, setHealthData] = useState(null); - const [selectedRange, setSelectedRange] = useState('30d'); + const [selectedRange, setSelectedRange] = useState('24hr'); + const [selectedRegion, setSelectedRegion] = useState('london'); const [liveExecutionsData, setLiveExecutionsData] = useState([]); - const [filteredData, setFilteredData] = useState([]); - const [averageUptime, setAverageUptime] = useState(0); - const [liveExecutionsRange, setLiveExecutionsRange] = useState('1h'); // Default to 1h - const [isHealthLoading, setIsHealthLoading] = useState(false); // Loading state for HealthBarChart - const [isLiveExecutionsLoading, setIsLiveExecutionsLoading] = useState(false); // Loading state for LiveExecutionsChart + const [liveExecutionsRange, setLiveExecutionsRange] = useState('1h'); + const [isHealthLoading, setIsHealthLoading] = useState(false); + const [isLiveExecutionsLoading, setIsLiveExecutionsLoading] = useState(false); const [isFixingOpensearchPrefix, setIsFixingOpensearchPrefix] = useState(false); + const [selectedFailureDetails, setSelectedFailureDetails] = useState(null); + const [calendarAnchor, setCalendarAnchor] = useState(null); + const [customStart, setCustomStart] = useState(''); + const [customEnd, setCustomEnd] = useState(''); + const [customRange, setCustomRange] = useState(null); // { after: unix, before: unix } or null - const isCloud = (window.location.host === "localhost:3002" || window.location.host === "shuffler.io") ? true : (import.meta.env.VITE_IS_SSR === "true"); + const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; const fetchHealthStats = useCallback(async () => { - setIsHealthLoading(true); // Start loading for HealthBarChart + setIsHealthLoading(true); try { - const response = await fetch(`${globalUrl}/api/v1/health/stats`, { - method: "GET", - headers: { - "Content-Type": "application/json", - Accept: "application/json", - }, - credentials: "include", - }); - if (!response.ok) { - throw new Error("Failed to fetch health stats"); + const activeDomain = isCloud ? REGION_DOMAIN[selectedRegion] : window.location.origin; + const nowMs = Date.now(); + const CHUNK_MS = 90 * 24 * 60 * 60 * 1000; // 90-day chunks + + let rangeStartSec, rangeEndSec; + if (customRange) { + rangeStartSec = customRange.after; + rangeEndSec = customRange.before; + } else { + const rangeMs = RANGE_MILLIS[selectedRange] || RANGE_MILLIS['30d']; + rangeStartSec = Math.floor((nowMs - rangeMs) / 1000); + rangeEndSec = Math.floor(nowMs / 1000); } - const data = await response.json(); - setHealthData(data); + + // Split large ranges into 90-day chunks to avoid oversized responses + const chunks = []; + let chunkEndSec = rangeEndSec; + const CHUNK_SEC = Math.floor(CHUNK_MS / 1000); + while (chunkEndSec > rangeStartSec) { + const chunkStartSec = Math.max(chunkEndSec - CHUNK_SEC, rangeStartSec); + chunks.push({ after: chunkStartSec, before: chunkEndSec }); + chunkEndSec = chunkStartSec; + } + + const fetchChunk = async (after, before) => { + const resp = await fetch(`${activeDomain}/api/v1/health/stats?after=${after}&before=${before}`, { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }); + if (!resp.ok) throw new Error("Failed to fetch health stats"); + return resp.json(); + }; + + let allData = []; + // Fetch chunks sequentially to avoid overwhelming the server + for (const chunk of chunks) { + const chunkData = await fetchChunk(chunk.after, chunk.before); + if (Array.isArray(chunkData)) { + // Filter to only items within this chunk's window to avoid duplicates + const filtered = chunkData.filter( + item => item.updated >= chunk.after && item.updated < chunk.before + ); + allData = allData.concat(filtered); + } + } + + setHealthData(allData); } catch (error) { console.error("Error fetching health stats:", error); toast.error("Failed loading health stats"); } finally { - setIsHealthLoading(false); // Stop loading for HealthBarChart + setIsHealthLoading(false); } - }, [globalUrl]); + }, [selectedRegion, selectedRange, customRange]); const fetchLiveExecutions = useCallback(async (range = '1h') => { - setIsLiveExecutionsLoading(true); // Start loading for LiveExecutionsChart + if (!userdata.support_access) return; + setIsLiveExecutionsLoading(true); try { + const activeDomain = isCloud ? REGION_DOMAIN[selectedRegion] : window.location.origin; const fetchOptions = { method: "GET", credentials: "include", @@ -67,326 +299,217 @@ const HealthPage = (props) => { }; } - const now = Math.floor(Date.now() / 1000); - // let after = now - 3600; // Default to 1h - let mode = "" - - switch (range) { - case '1h': - mode = "1h" - break; - case '7h': - mode = "7h" - break; - case '1d': - mode = "1d" - break; - case '7d': - mode = "7d" - break; - case 'month': - mode = "month" - break; - default: - mode = "1h" - } - - if (!userdata.support_access) { - return - } - const response = await fetch( - `${globalUrl}/api/v1/health/executions/live?mode=${mode}`, + `${activeDomain}/api/v1/health/executions/live?mode=${range}`, fetchOptions ); - if (!response.ok) { - throw new Error("Failed to fetch live executions"); - } + if (!response.ok) throw new Error("Failed to fetch live executions"); const data = await response.json(); - console.log("Raw live executions data:", data); - if (Array.isArray(data)) { const formattedData = data .map(item => ({ ...item, - // failed: Number(item.failed) || 0, executing: Number(item.executing) || 0, finished: Number(item.finished) || 0, aborted: Number(item.aborted) || 0, - created_at: Number(item.created_at) || 0 + created_at: Number(item.created_at) || 0, })) .sort((a, b) => a.created_at - b.created_at); - - console.log("Formatted live executions data:", formattedData); setLiveExecutionsData(formattedData); } else { - console.error("Received invalid data format:", data); setLiveExecutionsData([]); } } catch (error) { console.error("Error fetching live executions:", error); toast.error("Failed loading live executions data"); } finally { - setIsLiveExecutionsLoading(false); // Stop loading for LiveExecutionsChart + setIsLiveExecutionsLoading(false); } - }, [globalUrl]); + }, [userdata.support_access, selectedRegion]); useEffect(() => { fetchHealthStats(); fetchLiveExecutions(liveExecutionsRange); - const interval = setInterval(() => fetchLiveExecutions(liveExecutionsRange), 60000); return () => clearInterval(interval); }, [fetchHealthStats, fetchLiveExecutions, liveExecutionsRange]); - const extractRunFinished = (data, range) => { + // --- Derived data (memoized) --- + + const filteredHealthData = useMemo(() => { + if (!healthData || !Array.isArray(healthData)) return []; + if (customRange) { + return healthData.filter(item => item.updated >= customRange.after && item.updated <= customRange.before); + } + const now = Date.now(); + const ms = RANGE_MILLIS[selectedRange] || RANGE_MILLIS['30d']; + return healthData.filter(item => now - item.updated * 1000 <= ms); + }, [healthData, selectedRange, customRange]); + + const hasOpensearch = useMemo(() => + Array.isArray(healthData) && healthData.some(item => item.opnsearch && item.opnsearch.status), + [healthData]); + + const latestEntry = useMemo(() => { + if (!filteredHealthData.length) return null; + return filteredHealthData.reduce((best, item) => item.updated > (best?.updated || 0) ? item : best, null); + }, [filteredHealthData]); + + const extractServiceChartData = useCallback((config, data, range) => { if (!data || !Array.isArray(data)) return []; + const agg = new Map(); - const currentDate = new Date().getTime(); - const rangeInMillis = { - '24hr': 24 * 60 * 60 * 1000, - '7day': 7 * 24 * 60 * 60 * 1000, - '30d': 30 * 24 * 60 * 60 * 1000, - '90d': 90 * 24 * 60 * 60 * 1000 + // Anchor pre-fill to the most recent data point so the newest bar always + // reflects actual data instead of an empty "today/current-hour" slot. + const mostRecentMs = data.length > 0 + ? Math.max(...data.map(item => item.updated)) * 1000 + : Date.now(); + // Custom range uses UTC keys; all preset ranges use local timezone keys + const isCustom = range === 'custom'; + const getKey = (timestamp) => { + const d = new Date(timestamp); + if (isCustom) { + // UTC bucketing for custom date range (user picks UTC dates) + if (range === '24hr') { + return `${d.toISOString().split('T')[0]} ${String(d.getUTCHours()).padStart(2, '0')}:00`; + } + return d.toISOString().split('T')[0]; + } + // Local timezone bucketing for preset ranges + if (range === '24hr') { + return `${d.toLocaleDateString()} ${String(d.getHours()).padStart(2, '0')}:00`; + } + return d.toLocaleDateString(); }; - const filteredData = data.filter(item => currentDate - item.updated * 1000 <= rangeInMillis[range]); - const aggregatedData = new Map(); - - filteredData.forEach(item => { - const timestamp = item.updated * 1000; // Convert Unix timestamp to milliseconds - let key; - - switch (range) { - case '24hr': - const date = new Date(timestamp); - const hour = date.getHours(); - const formattedDate = `${date.toLocaleDateString()} ${hour}:00`; - key = formattedDate; - break; - case '7day': - const date1 = new Date(timestamp); - const hour1 = date1.getHours(); - const formattedDate1 = `${date1.toLocaleDateString()} ${hour1}:00`; - key = formattedDate1; - break; - default: - key = new Date(timestamp).toLocaleDateString(); + // Pre-fill all slots anchored to the most recent data point (newest -> oldest) + if (range === '24hr') { + for (let h = 0; h <= 23; h++) { + const d = new Date(mostRecentMs - h * 60 * 60 * 1000); + d.setMinutes(0, 0, 0); + agg.set(getKey(d.getTime()), { total: 0, healthyCount: 0, failures: [] }); } - - // Check if date already exists in the map - if (aggregatedData.has(key)) { - // Update aggregated values - const existingData = aggregatedData.get(key); - existingData.totalEntries++; - existingData.totalRunFinished += item.workflows.run_finished ? 1 : 0; - existingData.executionIds.push(item.workflows.execution_id); - } else { - // Add new entry to the map - aggregatedData.set(key, { - totalEntries: 1, - totalRunFinished: item.workflows.run_finished ? 1 : 0, - executionIds: [item.workflows.execution_id] - }); + } else if (range === '7day') { + for (let day = 0; day <= 6; day++) { + const d = new Date(mostRecentMs - day * 86400000); + agg.set(d.toLocaleDateString(), { total: 0, healthyCount: 0, failures: [] }); } + } else if (range === '30d') { + for (let day = 0; day <= 29; day++) { + const d = new Date(mostRecentMs - day * 86400000); + agg.set(d.toLocaleDateString(), { total: 0, healthyCount: 0, failures: [] }); + } + } else if (range === '90d') { + for (let day = 0; day <= 89; day++) { + const d = new Date(mostRecentMs - day * 86400000); + agg.set(d.toLocaleDateString(), { total: 0, healthyCount: 0, failures: [] }); + } + } else if (range === '180d') { + for (let day = 0; day <= 179; day++) { + const d = new Date(mostRecentMs - day * 86400000); + agg.set(d.toLocaleDateString(), { total: 0, healthyCount: 0, failures: [] }); + } + } else if (range === '365d') { + for (let day = 0; day <= 364; day++) { + const d = new Date(mostRecentMs - day * 86400000); + agg.set(d.toLocaleDateString(), { total: 0, healthyCount: 0, failures: [] }); + } + } else if (range === 'custom' && customRange) { + // Pre-fill using original UTC date strings to avoid local TZ rollover + const startDay = new Date(customRange.startLabel + 'T00:00:00Z'); + const endDay = new Date(customRange.endLabel + 'T00:00:00Z'); + const totalDays = Math.round((endDay - startDay) / 86400000) + 1; + for (let day = 0; day < totalDays; day++) { + agg.set(new Date(startDay.getTime() + day * 86400000).toISOString().split('T')[0], { total: 0, healthyCount: 0, failures: [] }); + } + } + + data.forEach(item => { + const key = getKey(item.updated * 1000); + const healthy = config.isHealthy(item); + if (agg.has(key)) { + const ex = agg.get(key); + ex.total++; + if (healthy) ex.healthyCount++; + else ex.failures.push(item); + } + // skip items that fall outside the pre-filled window }); - // Calculate averages and assign colors - const result = Array.from(aggregatedData.entries()).map(([key, { totalEntries, totalRunFinished, executionIds }]) => { - const avg = totalEntries > 0 ? totalRunFinished / totalEntries : 0; - const FinalAvg = avg * 100; - let color; - - if (FinalAvg >= 100) { - color = '#00F670'; - } else if (FinalAvg >= 98.50 && FinalAvg <= 99.99) { - color = '#FFD700'; - } else if (FinalAvg <= 98.49) { - color = '#FF354C'; - } - + return Array.from(agg.entries()) + .map(([date, { total, healthyCount, failures }]) => { + const pct = total === 0 ? null : (healthyCount / total) * 100; + const color = pct === null ? '#2a2a2a' : pct >= 99.5 ? '#00F670' : pct >= 95 ? '#FFD700' : '#FF354C'; return { - date: range === '24hr' ? `${key}:00` : key, - avgRunFinished: FinalAvg, + date, + avgRunFinished: pct === null ? null : parseFloat(pct.toFixed(2)), + total, color, - executionIds + executionIds: failures.map(f => f.workflows?.execution_id || f.id), + failures, }; }); + }, [customRange]); + const serviceCharts = useMemo(() => { + const configs = hasOpensearch ? [...SERVICE_CONFIG, OPENSEARCH_CONFIG] : SERVICE_CONFIG; + const result = {}; + const activeRange = customRange ? 'custom' : selectedRange; + for (const cfg of configs) { + result[cfg.key] = extractServiceChartData(cfg, filteredHealthData, activeRange); + } return result; - }; + }, [filteredHealthData, hasOpensearch, extractServiceChartData, selectedRange, customRange]); + + const systemStatus = useMemo(() => { + const configs = hasOpensearch ? [...SERVICE_CONFIG, OPENSEARCH_CONFIG] : SERVICE_CONFIG; + const statuses = configs.map(cfg => getStatusKey(computeAvgUptime(serviceCharts[cfg.key]), cfg.sloTarget)); + if (statuses.every(s => s === 'operational')) return 'operational'; + if (statuses.some(s => s === 'outage')) return 'outage'; + return 'degraded'; + }, [serviceCharts, hasOpensearch]); + + // Access guard — only redirect after auth state is confirmed loaded useEffect(() => { - if (healthData) { - const newData = extractRunFinished(healthData, selectedRange); - setFilteredData(newData); - - const totalUptime = newData.reduce((acc, curr) => acc + curr.avgRunFinished, 0); - const avgUptime = totalUptime / newData.length; - setAverageUptime(avgUptime); + if (!isLoaded) return; + if (!userdata || !userdata.id) { + navigate('/login?view=health&message=You must be logged in to view this page', { replace: true }); + } else if (userdata.support_access === false) { + navigate('/', { replace: true }); } - }, [selectedRange, healthData]); + }, [userdata, isLoaded, navigate]); - const filterDataByRange = (range) => { - setSelectedRange(range); - }; + // Render nothing only when definitively not authenticated/authorized + if (!isLoaded || !userdata?.id || userdata?.support_access === false) return null; - const updateChartData = () => { - if (!filteredData) { - return { - labels: [], - datasets: [{ - label: "", - data: [], - backgroundColor: [], - borderWidth: 1, - barThickness: 7, // Default bar thickness - }], - }; - } - let barThickness = 7; + const activeServices = hasOpensearch ? [...SERVICE_CONFIG, OPENSEARCH_CONFIG] : SERVICE_CONFIG; + const lastUpdated = latestEntry ? new Date(latestEntry.updated * 1000).toLocaleString() : null; + const backendVersion = latestEntry?.workflows?.backend_version || null; - const labels = filteredData.map((value, i) => { - if (selectedRange === '24hr') { - const [datePart, hourPart] = value.date.split(' '); - const [month, day, year] = datePart.split('/'); - const monthIndex = parseInt(month, 10) - 1; - const date = new Date(year, monthIndex, day); - let formattedDate = `${date.toLocaleString('en-US', { month: 'short', day: '2-digit' })}`; - // Add hour part if available - if (hourPart) { - formattedDate += `, ${hourPart.split(':').slice(0, 2).join(':')}`; - } + // --- Handlers --- - return `${formattedDate} \nUptime: ${value.avgRunFinished.toFixed(2)}%`; - } else if (selectedRange === '7day') { - const [datePart, hourPart] = value.date.split(' '); - const [month, day, year] = datePart.split('/'); - const monthIndex = parseInt(month, 10) - 1; - const date = new Date(year, monthIndex, day); - let formattedDate = `${date.toLocaleString('en-US', { month: 'short', day: '2-digit' })}`; - - // Add hour part if available - if (hourPart) { - formattedDate += `, ${hourPart.split(':').slice(0, 2).join(':')}`; - } - return `${formattedDate} \nUptime: ${value.avgRunFinished.toFixed(2)}%`; - } - else { - const dateParts = value.date.split('/'); // Assuming the date format is "DD/MM/YYYY" - const date = new Date(`${dateParts[2]}-${dateParts[0]}-${dateParts[1]}`); // Reformat the date string to "YYYY-MM-DD" - - return `${date.toLocaleDateString('en-US', { month: 'short', day: '2-digit', year: 'numeric' })} \nUptime: ${value.avgRunFinished.toFixed(2)}%`; - } - }); - - if (selectedRange === '24hr') { - barThickness = 35; - } - else if (selectedRange === '7day') { - barThickness = 5; - } - else if (selectedRange === '30d') { - barThickness = 25; - } - - const datasets = [{ - label: "", - data: filteredData.map(item => 1), - backgroundColor: filteredData.map(item => item.color), - borderWidth: 1, - barThickness: barThickness, - }]; - - return { labels, datasets }; - }; - - const options = { - legend: { - display: false - }, - layout: { - padding: { - top: 0, // Adjust the top padding as needed - bottom: 20, // Adjust the bottom padding as needed - left: 20, // Adjust the left padding as needed - right: 20 // Adjust the right padding as needed - } - }, - scales: { - yAxes: [{ - ticks: { - display: false - } - }], - xAxes: [{ - ticks: { - display: false - } - }] - }, - tooltips: { - callbacks: { - label: function (tooltipItem, data) { - const label = data.labels[tooltipItem.index]; - return label.split('\n')[0]; // Return only the date part - }, - afterLabel: function (tooltipItem, data) { - const label = data.labels[tooltipItem.index]; - const uptime = label.match(/Uptime:\s*(\d+(?:\.\d+)?)/)[1]; // Extract uptime value using regex - return `Test-Workflow Health: ${uptime}%`; // Customize the uptime display - }, - title: function () { - return 'Fully Operational'; // Hide the tooltip title - } - } - } - }; - - const handleBarClick = (event, elements) => { - if (event && event.length > 0) { - const clickedIndex = event[0]._index - const clickedData = filteredData[clickedIndex] - const executionIds = clickedData.executionIds - .filter(executionId => { - const item = healthData.find(dataItem => dataItem.workflows.execution_id === executionId); - return item && item.workflows.run_finished === false; - }); - - // console.log("Filtered Execution IDs:", executionIds); - if (executionIds.length > 0) { - const url = `${globalUrl}/api/v1/health/stats?execution_id=${executionIds.join(',')}`; - window.open(url, '_blank'); - } else { - toast.success("All executions in selected period succeeded"); - } - } + const copyToClipboard = (text) => { + navigator.clipboard.writeText(text) + .then(() => toast.success('Copied to clipboard')) + .catch(() => toast.error('Failed to copy')); }; const handleFixOpensearchPrefix = async () => { - if (isFixingOpensearchPrefix) { - return; - } - + if (isFixingOpensearchPrefix) return; setIsFixingOpensearchPrefix(true); try { - const response = await fetch(`${globalUrl}/api/v1/health/opensearch-prefix`, { + const response = await fetch(`${window.location.origin}/api/v1/health/opensearch-prefix`, { method: "POST", - headers: { - "Content-Type": "application/json", - Accept: "application/json", - }, + headers: { "Content-Type": "application/json", Accept: "application/json" }, credentials: "include", }); - const data = await response.json(); if (!response.ok || !data.success) { - const reason = data && data.reason ? data.reason : "Failed to fix opensearch prefix"; - throw new Error(reason); + throw new Error(data?.reason || "Failed to fix opensearch prefix"); } - const reindexed = data.reindexed ? data.reindexed.length : 0; const aliasUpdates = data.alias_updates ? data.alias_updates.length : 0; const deleted = data.deleted_indices ? data.deleted_indices.length : 0; @@ -399,130 +522,513 @@ const HealthPage = (props) => { } }; - const healthBarData = updateChartData() + const handleBarClick = (serviceKey, barData) => { + if (!barData.failures || barData.failures.length === 0) { + toast.success('All checks passed in this period'); + setSelectedFailureDetails(null); + return; + } + setSelectedFailureDetails({ serviceKey, barData }); + }; + // --- Render helpers --- + + const renderOpBadge = (op) => { + const ok = op.value === true; + const warn = op.value === 'warn'; + const dotColor = ok ? '#00F670' : warn ? '#FFD700' : '#FF354C'; + const bg = ok ? 'rgba(0,246,112,0.07)' : warn ? 'rgba(255,215,0,0.07)' : 'rgba(255,53,76,0.07)'; + const border = ok ? 'rgba(0,246,112,0.18)' : warn ? 'rgba(255,215,0,0.18)' : 'rgba(255,53,76,0.18)'; + return ( +
+
+ {op.label} +
+ ); + }; + + const renderServiceCard = (cfg) => { + const chartData = serviceCharts[cfg.key] || []; + const avgUptime = computeAvgUptime(chartData); + const statusKey = getStatusKey(avgUptime, cfg.sloTarget); + const status = STATUS_STYLE[statusKey]; + const sloMet = statusKey === 'operational'; + const latestOps = latestEntry ? cfg.getOperations(latestEntry) : []; + const extra = latestEntry ? cfg.getExtra(latestEntry) : null; + const totalFails = chartData.reduce((acc, d) => acc + (d.failures?.length || 0), 0); + const IconComp = cfg.Icon; + + return ( +
+ {/* LEFT — identity + uptime number */} +
+
+
+ +
+ {cfg.label} + +
+ +
+ + {avgUptime.toFixed(2)} + + % + uptime +
+ +
+ SLO {cfg.sloTarget}% + + {totalFails > 0 && ( + {totalFails} incident{totalFails !== 1 ? 's' : ''} + )} +
+
+ + {/* CENTRE — full-width bar chart + SLO rail */} +
+ {/* SLO progress rail */} +
+
+ +
+ +
+ + {/* History spark bars */} + handleBarClick(cfg.key, barData)} + /> + + + ← newest · oldest → · click bar for details + +
+ + {/* RIGHT — last check operations */} +
+ Last Check +
+ {latestOps.map(renderOpBadge)} +
+ {extra && ( + {extra} + )} +
+
+ ); + }; + + const renderFailureDetails = () => { + if (!selectedFailureDetails) return null; + const { serviceKey, barData } = selectedFailureDetails; + const cfg = activeServices.find(s => s.key === serviceKey); + if (!cfg) return null; + + console.log("bar is: ", barData) + + return ( +
+ {/* Panel header */} +
+
+
+ + {cfg.label} — {barData.date} + + + {barData.value != null ? barData.value.toFixed(2) : '—'}% uptime in this period +
+ +
+ + {/* Failure rows */} +
+ {barData.failures.slice(0, 20).map((item, idx) => { + const ops = cfg.getOperations(item); + const extra = cfg.getExtra(item); + const ids = cfg.getIds ? cfg.getIds(item) : []; + const errors = cfg.getErrors ? cfg.getErrors(item) : []; + return ( +
+ + {/* Row header: index + timestamp + op-status badges */} +
+ #{idx + 1} + + {new Date(item.updated * 1000).toLocaleString()} + +
+ {ops.map(op => { + const ok = op.value === true; + const warn = op.value === 'warn'; + return ( + + ); + })} +
+ {extra && {extra}} +
+ + {/* Error reasons */} + {errors.length > 0 && ( +
+ Details + {errors.map(err => ( +
+ {err.label}: + {err.msg} +
+ ))} +
+ )} + + {/* IDs with copy buttons */} + {ids.length > 0 && ( +
+ {ids.map(id => ( +
+ {id.label} + + {id.value.length > 20 ? id.value.substring(0, 20) + '…' : id.value} + + + + +
+ ))} +
+ )} + +
+ ); + })} + {barData.failures.length > 20 && ( +
+ + +{barData.failures.length - 20} more records not shown + +
+ )} +
+
+ ); + }; + + // --- Render --- + + const rangeLabel = { '24hr': 'last 24h', '7day': 'last 7 days', '30d': 'last 30 days', '90d': 'last 90 days', '180d': 'last 180 days', '365d': 'last 365 days' }; + const activRangeLabel = customRange + ? `${customRange.startLabel} – ${customRange.endLabel}` + : (rangeLabel[selectedRange] || selectedRange); return ( -
- {/* Health Bar Chart Section */} -
- - - - - - - - -
+
+
- {/* Loading Bar for HealthBarChart */} - {isHealthLoading && ( - - )} - -
-
-
- -
- Workflow Health - Operational -
+ {/* === Page header === */} +
+
+
+ Platform Health + {isCloud && ( + + + + )} + {!isCloud && ( + + )}
-
- {averageUptime.toFixed(2)}% - Success Rate + +
+ {backendVersion && Backend v{backendVersion}} + {lastUpdated && Last check: {lastUpdated}} + {isCloud ? 'Cloud' : 'On-Premises'} +
+ + {/* === Legend (Moved to top) === */} +
+ {[{ color: '#00F670', label: '≥ SLO Healthy' }, { color: '#FFD700', label: '95–SLO Degraded' }, { color: '#FF354C', label: '< 95% Outage' }].map(({ color, label }) => ( +
+
+ {label} +
+ ))} + Click any bar to view failure details · White marker = SLO target
- -
- {userdata.support_access && ( -
-
- Live Executions - - - - - - {/* */} +
+
+ + {[['24hr', '24h'], ['7day', '7d'], ['30d', '30d'], ['90d', '90d'], ['180d', '180d'], ['365d', '365d']].map(([key, label]) => { + const isDisabled = isHealthLoading || ['90d', '180d', '365d'].includes(key); + const isActive = !customRange && selectedRange === key; + return ( + + ); + })} + + {/* Calendar range picker button */} + + +
- {/* Loading Bar for LiveExecutionsChart */} - {isLiveExecutionsLoading && ( - + + {/* Show active custom range label */} + {customRange && ( +
+ + {customRange.startLabel} – {customRange.endLabel} + + +
)} - +
+ + {/* Calendar popover */} + setCalendarAnchor(null)} + anchorOrigin={{ vertical: 'bottom', horizontal: 'right' }} + transformOrigin={{ vertical: 'top', horizontal: 'right' }} + PaperProps={{ style: { backgroundColor: '#111', border: '1px solid #252525', borderRadius: 12, minWidth: 310, overflow: 'hidden', boxShadow: '0 12px 40px rgba(0,0,0,0.7)', display: 'flex', flexDirection: 'column' } }} + > + {/* Header */} +
+ + Custom Date Range + Max 60 days +
+ + {/* Fields */} +
+ setCustomStart(e.target.value)} + InputLabelProps={{ shrink: true, style: { color: '#777', fontSize: 12 } }} + inputProps={{ max: customEnd || undefined, style: { color: '#e0e0e0', fontSize: 13, backgroundColor: '#1a1a1a', borderRadius: 6 } }} + sx={{ + '& .MuiOutlinedInput-root': { backgroundColor: '#1a1a1a', borderRadius: '8px' }, + '& .MuiOutlinedInput-notchedOutline': { borderColor: '#2e2e2e' }, + '& .MuiOutlinedInput-root:hover .MuiOutlinedInput-notchedOutline': { borderColor: '#FF844466' }, + '& .MuiOutlinedInput-root.Mui-focused .MuiOutlinedInput-notchedOutline': { borderColor: '#FF8444', borderWidth: 1 }, + '& .MuiInputLabel-root.Mui-focused': { color: '#FF8444' }, + '& input::-webkit-calendar-picker-indicator': { filter: 'invert(0.4)' }, + }} + /> + setCustomEnd(e.target.value)} + InputLabelProps={{ shrink: true, style: { color: '#777', fontSize: 12 } }} + inputProps={{ min: customStart || undefined, style: { color: '#e0e0e0', fontSize: 13, backgroundColor: '#1a1a1a', borderRadius: 6 } }} + sx={{ + '& .MuiOutlinedInput-root': { backgroundColor: '#1a1a1a', borderRadius: '8px' }, + '& .MuiOutlinedInput-notchedOutline': { borderColor: '#2e2e2e' }, + '& .MuiOutlinedInput-root:hover .MuiOutlinedInput-notchedOutline': { borderColor: '#FF844466' }, + '& .MuiOutlinedInput-root.Mui-focused .MuiOutlinedInput-notchedOutline': { borderColor: '#FF8444', borderWidth: 1 }, + '& .MuiInputLabel-root.Mui-focused': { color: '#FF8444' }, + '& input::-webkit-calendar-picker-indicator': { filter: 'invert(0.4)' }, + }} + /> + + {/* Live duration indicator */} + {customStart && customEnd && (() => { + const days = Math.round((new Date(customEnd) - new Date(customStart)) / (1000 * 60 * 60 * 24)) + 1; + const tooLong = days > 60; + const invalid = days <= 0; + const color = invalid ? '#666' : tooLong ? '#FF354C' : '#FF8444'; + const bg = invalid ? 'rgba(255,255,255,0.03)' : tooLong ? 'rgba(255,53,76,0.08)' : 'rgba(255,132,68,0.08)'; + const border = invalid ? '#2a2a2a' : tooLong ? 'rgba(255,53,76,0.25)' : 'rgba(255,132,68,0.25)'; + return ( +
+
+ + {invalid ? 'End date must be after start date' : tooLong ? `${days} days — exceeds 60-day limit` : `${days} day${days !== 1 ? 's' : ''} selected`} + +
+ ); + })()} +
+ + {/* Footer */} +
+ + +
+ +
+ + {/* Loading */} + {isHealthLoading && } + + {/* === System status banner === */} + {!isHealthLoading && filteredHealthData.length > 0 && ( +
+ {systemStatus === 'operational' + ? + : systemStatus === 'degraded' + ? + : } +
+ + {systemStatus === 'operational' ? 'All Systems Operational' : systemStatus === 'degraded' ? 'Degraded Performance Detected' : 'Some Services Affected'} + + + {filteredHealthData.length} health check{filteredHealthData.length !== 1 ? 's' : ''} · {activRangeLabel} + +
+ + {/* SLO summary dots */} +
+ {activeServices.map(cfg => { + const uptime = computeAvgUptime(serviceCharts[cfg.key]); + const dotColor = uptime >= cfg.sloTarget ? '#00F670' : uptime >= 95 ? '#FFD700' : '#FF354C'; + return ( + +
+
+ {cfg.label} + {uptime.toFixed(2)}% +
+ + ); + })} +
)} + {/* === Service health rows (vertical) === */} +
+ {activeServices.map(cfg => renderServiceCard(cfg))} +
+ {/* === Failure details panel === */} + {renderFailureDetails()} + + {/* Legend removed and moved to the top */} + + {/* === Live Executions === */} + {userdata.support_access && ( + <> +
+
+
+ Live Executions + + {[['1h', '1h'], ['7h', '7h'], ['1d', '1d'], ['7d', '7d']].map(([key, label]) => ( + + ))} + +
+ {isLiveExecutionsLoading && } + +
+ + )} +
); }; -export default HealthPage; +export default HealthPage; \ No newline at end of file diff --git a/frontend/src/components/LeftSideBar.jsx b/frontend/src/components/LeftSideBar.jsx index 056867ae..6050f092 100644 --- a/frontend/src/components/LeftSideBar.jsx +++ b/frontend/src/components/LeftSideBar.jsx @@ -52,6 +52,12 @@ const ShuffleLogo = "/images/Shuffle_logo.png"; const detectionIcon = "/icons/detection.svg"; const documentationIcon = "/icons/documentation.svg"; const ExpandMoreAndLessIcon = "/icons/expandMoreIcon.svg"; +const shuffleSecurityLogo = ( + + + +); + const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, SHUFFLE_VERSION }) => { @@ -87,6 +93,17 @@ const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, SHUFFLE_V ); const [activeOrgData, setActiveOrgData] = useState(null); const [isProdStatusOn, setIsProdStatusOn] = useState(false); + const [productAnchorEl, setProductAnchorEl] = useState(null); + + const handleProductClick = (event) => { + event.preventDefault(); + setProductAnchorEl((prev) => (prev ? null : event.currentTarget)); + }; + + const handleProductClose = () => { + setProductAnchorEl(null); + }; + const userOrgs = React.useMemo(() => { return orgOptions.find((option) => option.name === selectedOrg); }, [selectedOrg, orgOptions]); @@ -120,7 +137,7 @@ const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, SHUFFLE_V setCurrentSelectedTheme(userdata?.theme); } }, [userdata]); - + const CustomPopper = (props) => { @@ -839,6 +856,8 @@ const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, SHUFFLE_V regiontag = "EU-2"; } else if (regiontag === "ca"){ regiontag = "CA"; + } else if (regiontag === "uk"){ + regiontag = "UK"; } } } @@ -1022,7 +1041,7 @@ const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, SHUFFLE_V }} > - - Shuffle Logo - - + + { + !showPartnerLogo && e.preventDefault(); + }} + > + Shuffle Logo + {!showPartnerLogo && expandLeftNav && ( + + )} + + + + + + { + if (isCloud) { + //ReactGA.event({ + // category: "sidebar", + // action: "click_shuffle_security", + // label: "", + //}) + + window.location.href = "https://security.shuffler.io/incidents?utm_source=shuffler_sidebar"; + } else { + const { protocol, hostname } = window.location; + + var newPort = 3002; + if (protocol === "https") { + newPort = 3444 + } + + const newUrl = `${protocol}//${hostname}:${newPort}/incidents`; + window.location.href = newUrl; + } + }} + sx={{ + borderRadius: "8px", + padding: "10px 12px", + border: "1px solid transparent", + "&:hover": { + backgroundColor: themeMode === "dark" ? "#2C2C2C" : "#F5F5F5", + }, + display: "flex", + gap: "12px", + alignItems: "center", + }} + > + + {shuffleSecurityLogo} + + + Shuffle{" "} + Security + + + { + handleProductClose(); + window.location.href = + isCloud && !showPartnerLogo ? "/" : "/workflows"; + }} + sx={{ + borderRadius: "8px", + padding: "10px 12px", + border: + themeMode === "dark" + ? "1px solid rgba(242, 100, 2, 0.3)" + : "1px solid rgba(242, 100, 2, 0.2)", + backgroundColor: + themeMode === "dark" + ? "rgba(242, 100, 2, 0.05)" + : "rgba(242, 100, 2, 0.02)", + "&:hover": { + backgroundColor: + themeMode === "dark" + ? "rgba(242, 100, 2, 0.1)" + : "rgba(242, 100, 2, 0.06)", + }, + display: "flex", + gap: "12px", + alignItems: "center", + }} + > + + Shuffle + + + Shuffle{" "} + Core + + + { !isCloud && expandLeftNav && ( @@ -1747,7 +1944,8 @@ const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, SHUFFLE_V
- {activeMainTab === "setup" && ( - <> - {(isIntegration || isAgent) && selectedAction && Object.getOwnPropertyNames(selectedAction).length > 0 && selectedActionParameters?.length > 0 ? - apps !== undefined && apps !== null && apps.length > 0 && wrapperapp !== undefined && newimage !== undefined ? -
-
{ - selectedAction.example = "noapp" - selectedAction.large_image = newimage - if (cy !== undefined && cy !== null) { - const foundnode = cy.getElementById(selectedAction.id) - if (foundnode !== undefined && foundnode !== null) { - foundnode.data("large_image", newimage) - } - } - - /* - const iconInfo = GetIconInfo(selectedAction) - if (iconInfo !== undefined && iconInfo !== null) { - selectedAction.fillGradient = iconInfo.fillGradient - - selectedAction.iconBackground = iconInfo.iconBackgroundColor - selectedAction.fillstyle = "linear-gradient" - } - */ - - const paramIndex = selectedAction.parameters !== undefined && selectedAction.parameters !== null ? selectedAction.parameters.findIndex((param) => param.name === "app_name") : -1 - if (paramIndex === -1) { - console.log("Couldn't find app_name parameter") - selectedAction.parameters.push({ - name: "app_name", - value: wrapperapp.name, - autocompleted: false, - }) - } else { - selectedAction.parameters[paramIndex].value = wrapperapp.name - } - - setSelectedAction(selectedAction) - setUpdate(Math.random()) - - }}> - -
- -
- -
- - {apps.map((app, appIndex) => { - // Forces it into every category (for now) - // This is to make it possible to "use" shuffle for Singul natively - if (app.name === "Shuffle Tools") { - if (actionname == "Intel" || actionname == "Intel") { - app.categories = [actionname] - } - } - - if (app.categories === undefined || app.categories === null || app.categories.length === 0) { - return null - } - - var newactionname = actionname.toLowerCase() - if (isAgent === true) { - newactionname = "ai" - } - - var found = false - for (var key in app.categories) { - - var localnewactionname = newactionname - if (newactionname == "comms") { - localnewactionname = "communication" - } - - if (app.categories[key].toLowerCase() !== localnewactionname) { - continue - } - - found = true - break - } - - if (!found) { - return null - } - - var isAppSelected = false - const paramIndex = selectedAction.parameters.findIndex((param) => param.name === "app_name") - if (paramIndex > -1) { - // Check the actual value and if it's the same - if (selectedAction.parameters[paramIndex].value === app.name) { - isAppSelected = true - } - } - - return ( -
{ - selectedAction.example = "" - selectedAction.large_image = app.large_image - if (cy !== undefined && cy !== null) { - const foundnode = cy.getElementById(selectedAction.id) - if (foundnode !== undefined && foundnode !== null) { - foundnode.data("large_image", app.large_image) - } - } - - if (paramIndex === -1) { - console.log("Couldn't find app_name parameter") - selectedAction.parameters.push({ - name: "app_name", - value: app.name, - autocompleted: false, - }) - } else { - selectedAction.parameters[paramIndex].value = app.name - } - - setSelectedAction(selectedAction) - setUpdate(Math.random()) - - - var requiresAuth = app?.authentication?.required - if (requiresAuth && appAuthentication?.length > 0) { - for (var key in appAuthentication) { - if (appAuthentication[key]?.app?.name === app?.name) { - requiresAuth = false - break - } - } - } - - setRequiresAuthentication(requiresAuth); - }}> - - - -
- ) - })} -
- : null - : - null - } + + {activeMainTab === "setup" && ( + <>
Name @@ -2657,7 +2652,21 @@ const ParsedAction = (props) => { placeholder={selectedAction.execution_delay} value={delay} onChange={(event) => { + // Check if positive number + if (isNaN(event.target.value) || Number(event.target.value) < 0) { + toast.error("Please enter a valid positive number for delay.") + return + } + + // Check if first number is 0 + if (event.target.value.length > 1 && event.target.value.charAt(0) === "0") { + event.target.value = event.target.value.substring(1) + } + setDelay(event.target.value) + selectedAction.execution_delay = event.target.value + setSelectedAction(selectedAction) + setUpdate(Math.random()) }} /> @@ -2702,9 +2711,13 @@ const ParsedAction = (props) => { fullWidth variant="contained" onClick={() => { - //if (authenticationType.type === "oauth2" && authenticationType.redirect_uri !== undefined && authenticationType.redirect_uri !== null) { - // return null - //} + if (isCloud) { + ReactGA.event({ + category: "Integration", + action: "Authenticate", + label: `${selectedApp?.name} - Open 1`, + }) + } setAuthenticationModalOpen(true); }} @@ -2718,19 +2731,7 @@ const ParsedAction = (props) => { ) : null} {/* Change made in new release when we added Tabs system in it */} - {( - (selectedAction.authentication !== undefined && - selectedAction.authentication !== null && - selectedAction.authentication.length > 0) || - (selectedApp.name !== undefined && - (((selectedAction.authentication === undefined || - selectedAction.authentication === null || - selectedAction.authentication.length === 0)) || - isAgent || - isIntegration) && - requiresAuthentication) - ) ? ( - + {appMayNeedAuth ? (
{ 0 && selectedAction?.selectedAuthentication && Object.getOwnPropertyNames(selectedAction.selectedAuthentication).length !== 0 ? ( + workflow?.suborg_distribution?.length > 0 && selectedAction?.selectedAuthentication && typeof selectedAction.selectedAuthentication === 'object' && Object.getOwnPropertyNames(selectedAction.selectedAuthentication).length !== 0 ? (
{ labelId="select-app-auth" value={ selectedAction?.authentication_id === "authgroups" ? "authgroups" : - !selectedAction?.selectedAuthentication || Object.getOwnPropertyNames(selectedAction.selectedAuthentication).length === 0 + (selectedAction?.selectedAuthentication === null || !selectedAction?.selectedAuthentication || typeof selectedAction.selectedAuthentication !== 'object' || Object.getOwnPropertyNames(selectedAction.selectedAuthentication).length === 0) ? "No selection" : selectedAction?.selectedAuthentication } @@ -2997,6 +2998,14 @@ const ParsedAction = (props) => { variant="outlined" style={{}} onClick={() => { + if (isCloud) { + ReactGA.event({ + category: "Integration", + action: "Authenticate", + label: `${selectedApp?.name} - Open 2`, + }) + } + setAuthenticationModalOpen(true); }} > @@ -3005,68 +3014,18 @@ const ParsedAction = (props) => {
- {requiresAuthentication && (!selectedAction.authentication_id || selectedAction.authentication_id === "") ? ( -
- - Authentication needed. - - - Some steps in this workflow won’t run until you connect your account. - - -
- ) : null} +
) : null} - {selectedAction.authentication_id === "authgroups" && (authGroups === undefined || authGroups === null || authGroups.length === 0) ? - - - Create your first Authentication group - - - : null} + {selectedAction.authentication_id === "authgroups" && (authGroups === undefined || authGroups === null || authGroups.length === 0) ? + + + Create your first Authentication group + + + : null} {/*showEnvironment !== undefined && showEnvironment && environments.length > 1 && !isIntegration ? ( @@ -3159,6 +3118,8 @@ const ParsedAction = (props) => {
) : null*/} + + {workflow.execution_variables !== undefined && workflow.execution_variables !== null && workflow.execution_variables.length > 0 ? (
Runtime variable (optional) @@ -3470,11 +3431,80 @@ const ParsedAction = (props) => { ) : null}
+ {appMayNeedAuth && !hasAuth && !isAgent && !isIntegration ? ( +
+ + Authentication needed {isIntegration || isAgent ? `` : "."} + + + This step may not work until you authenticate it. + + +
+ ) : null} + {activeMainTab === "setup" && ( + + + + setAnchorEl(null)} + + anchorOrigin={{ + vertical: 'bottom', + horizontal: 'left', + }} + transformOrigin={{ + vertical: 'top', + horizontal: 'left', + }} + + style={{ + zIndex: 20000, + marginTop: 2, + border: "1px solid rgba(255,255,255,0.3)", + }} + + PaperProps={{ + style: { + maxHeight: 600, + maxWidth: 450, + } + }} + > + { + e.preventDefault() + e.stopPropagation() + + const paramIndex = selectedAction.parameters.findIndex((param) => param.name === "app_name") + if (paramIndex === -1) { + selectedAction.parameters.push({ + "description": "The name of the app to run the LLM query against", + "id": "", + "name": "app_name", + "example": "", + "value": "", + "multiline": false, + "multiselect": false, + "options": null, + "action_field": "", + "variant": "STATIC_VALUE", + "required": true, + "configuration": false, + "tags": null, + "schema": { + "type": "" + }, + "skip_multicheck": false, + "value_replace": null, + "unique_toggled": false, + "error": "", + "hidden": false, + + "custom_value": true, + }) + } else { + selectedAction.parameters[paramIndex].custom_value = true + } + + // Overwrite params for custom value handling + const newSelectedActionParameters = JSON.parse(JSON.stringify(selectedAction?.parameters)) + setSelectedActionParameters(newSelectedActionParameters) + setSelectedAction(selectedAction) + setAnchorEl(null) + }} + selected={false} + style={{ + margin: 7, + display: "flex", + minWidth: 400, + maxWidth: 400, + cursor: "pointer", + }} + > + + Custom Value + + + + + {apps.map((item, index) => { + const parsedName = (item.name?.charAt(0).toUpperCase() + item.name?.substring(1)).replace(/_/g, " ") + return ( + { + //handleSelect(item) + setSelectedActionLocal(selectedAction, item) + setAnchorEl(null) + }} + selected={false} + style={{ + margin: 7, + display: "flex", + minWidth: 400, + maxWidth: 400, + cursor: "pointer", + }} + > + + + {parsedName} + + + ) + })} + + + + {apps.map((app, appIndex) => { + // Forces it into every category (for now) + // This is to make it possible to "use" shuffle for Singul natively + if (app.name === "Shuffle Tools") { + if (actionname == "Intel" || actionname == "Intel") { + app.categories = [actionname] + } + } + + var isAppSelected = false + const paramIndex = selectedAction.parameters.findIndex((param) => param.name === "app_name") + if (paramIndex > -1) { + // Check the actual value and if it's the same + //if (selectedAction.parameters[paramIndex].value === app.name) { + if (selectedAction.parameters[paramIndex].value.includes(app.name)) { + isAppSelected = true + } + } + + if (app.categories === undefined || app.categories === null || app.categories.length === 0) { + if (!isAppSelected) { + return null + } + } + + var newactionname = actionname.toLowerCase() + if (isAgent === true) { + //newactionname = "ai" + } else { + var found = false + for (var key in app.categories) { + + var localnewactionname = newactionname + if (newactionname == "comms") { + localnewactionname = "communication" + } + + if (app.categories[key].toLowerCase() !== localnewactionname) { + continue + } + + found = true + break + } + + if (!found && !isAppSelected) { + return null + } + } + + + return ( +
{ + setSelectedActionLocal(selectedAction, app) + }}> + + + +
+ ) + })} + +
+
+ : null + : null + }
@@ -3524,7 +3824,7 @@ const ParsedAction = (props) => { marginBottom: hideExtraTypes ? 50 : 200, }} > { - selectedActionParameters !== undefined && selectedActionParameters !== null && selectedAction && Object.getOwnPropertyNames(selectedAction).length > 0 && selectedActionParameters.length > 0 ? + selectedActionParameters !== undefined && selectedActionParameters !== null && selectedAction && selectedAction !== null && typeof selectedAction === 'object' && Object.getOwnPropertyNames(selectedAction).length > 0 && selectedActionParameters.length > 0 ?
{/* { } if ((isIntegration || isAgent) && data.name === "app_name") { - return null - } + if (data?.custom_value === true) { + //data.label = "Allowed MCPs" + } else { + return null + } + } /* // Somehow autogenerate from the app itself @@ -3827,13 +4131,9 @@ const ParsedAction = (props) => { } if (selectedAction.name === "custom_action" && data.name === "body") { - for (var key in selectedActionParameters) { - const param = selectedActionParameters[key] - if (param.name === "method") { - if (param.value === "GET") { - return null - } - } + const methodParam = selectedActionParameters.find(p => p.name === "method") || selectedAction.parameters?.find(p => p.name === "method"); + if (methodParam?.value?.toUpperCase() === "GET") { + return null } } @@ -3897,6 +4197,7 @@ const ParsedAction = (props) => { backgroundColor: themeMode === "dark" ? "#161616" : "#CCCCCC", color: theme.palette.text.primary, fontWeight: 600, + borderRadius: "6px !important", "&:hover": { backgroundColor: themeMode === "dark" ? "rgba(0,0,0,0.3)" : "rgba(0,0,0,0.1)", }, @@ -4322,7 +4623,7 @@ const ParsedAction = (props) => { } - if ((multiline === undefined || multiline === false) && ((data?.autocompleted === true || data?.field_active === true) || data.name.startsWith("${") && data.name.endsWith("}"))) { + if ((multiline === undefined || multiline === false) && (data.name.startsWith("${") && data.name.endsWith("}"))) { multiline = true } @@ -4827,6 +5128,16 @@ const ParsedAction = (props) => { fullWidth id={"rightside_field_" + count} onChange={(e) => { + if (e.target.value.includes("custom_shuffle_action")) { + data.options = [] + selectedActionParameters[count].options = [] + setSelectedActionParameters(selectedActionParameters) + selectedAction.parameters = selectedActionParameters + setSelectedAction(selectedAction) + setUpdate(Math.random()) + return + } + changeActionParameter(e, count, data); setUpdate(Math.random()); }} @@ -4871,6 +5182,22 @@ const ParsedAction = (props) => { ); } )} + + + {isAgent || isIntegration ? + + Custom Value + + : null} ); } else if (data.variant === "STATIC_VALUE") { @@ -5263,12 +5590,14 @@ const ParsedAction = (props) => { const buttonTitle = `Authenticate the ${selectedApp?.name?.replaceAll("_", " ")} API` const hasAutocomplete = data?.autocompleted === true + const isPathField = selectedAction?.name === "custom_action" && data?.name === "path" + if (data.variant === undefined || data.variant === null) { data.variant = "STATIC_VALUE" } - var isFirstOptional = optionalFound === false && data.configuration === false && data.required === false ? true : false - if (optionalFound === false && data.configuration === false && data.required === false) { + var isFirstOptional = optionalFound === false && data.configuration === false && data.required === false && !isPathField ? true : false + if (optionalFound === false && data.configuration === false && data.required === false && !isPathField) { optionalFound = true } @@ -5295,7 +5624,7 @@ const ParsedAction = (props) => { } } - const isOptional = data.configuration === false && data.required === false + const isOptional = (data.configuration === false && data.required === false) && !isPathField return (
@@ -5352,6 +5681,13 @@ const ParsedAction = (props) => { color: theme.palette.textPrimary, }} onClick={() => { + if (isCloud) { + ReactGA.event({ + category: "Integration", + action: "Authenticate", + label: `${selectedApp?.name} - Open 4`, + }) + } setAuthenticationModalOpen(true); }} /> diff --git a/frontend/src/components/PartnersUsecasesTab.jsx b/frontend/src/components/PartnersUsecasesTab.jsx index 9cbc31a1..85c40d56 100644 --- a/frontend/src/components/PartnersUsecasesTab.jsx +++ b/frontend/src/components/PartnersUsecasesTab.jsx @@ -846,6 +846,7 @@ const PartnersUsecasesTab = ({ isCloud, globalUrl, userdata, partnerData, setPar setIsLoading(true); if(!isCloud || !userdata?.active_org?.is_partner) { // If the user is not a partner or if it's not a cloud environment do not make api call :) + setIsLoading(false); return; } // Load usecase data from API diff --git a/frontend/src/components/RuntimeDebugger.jsx b/frontend/src/components/RuntimeDebugger.jsx index ba531c5b..51a3aaaf 100644 --- a/frontend/src/components/RuntimeDebugger.jsx +++ b/frontend/src/components/RuntimeDebugger.jsx @@ -1087,6 +1087,9 @@ const RuntimeDebugger = (props) => { options={[{ "name": "Agent Runs", "id": "AGENT", + },{ + "name": "Sensor Actions", + "id": "SENSOR_ACTION", }].concat(workflows)} fullWidth style={{ diff --git a/frontend/src/components/SearchContactForm.jsx b/frontend/src/components/SearchContactForm.jsx new file mode 100644 index 00000000..ff5747a5 --- /dev/null +++ b/frontend/src/components/SearchContactForm.jsx @@ -0,0 +1,121 @@ +import React, { useState } from "react"; +import theme from "../theme.jsx"; +import { TextField, Typography, Button } from "@mui/material"; + +const SearchContactForm = ({ globalUrl, isMobile, tabName }) => { + const [formMail, setFormMail] = useState(""); + const [message, setMessage] = useState(""); + const [formMessage, setFormMessage] = useState(""); + + const submitContact = (email, message) => { + const data = { + firstname: "", + lastname: "", + title: "", + companyname: "", + email: email, + phone: "", + message: message, + }; + + const errorMessage = + "Something went wrong. Please contact frikky@shuffler.io directly."; + + fetch(globalUrl + "/api/v1/contact", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(data), + }) + .then((response) => response.json()) + .then((response) => { + setFormMessage( + response?.success === true ? response.reason : errorMessage + ); + setFormMail(""); + setMessage(""); + }) + .catch(() => { + setFormMessage(errorMessage); + }); + }; + + return ( +
+ + Can't find what you're looking for? + +
+ setFormMail(e.target.value)} + /> + setMessage(e.target.value)} + /> +
+ + + {formMessage} + +
+ ); +}; + +export default SearchContactForm; diff --git a/frontend/src/components/SearchData.jsx b/frontend/src/components/SearchData.jsx index de209b4b..f7a7dd33 100644 --- a/frontend/src/components/SearchData.jsx +++ b/frontend/src/components/SearchData.jsx @@ -47,7 +47,7 @@ const chipStyle = { backgroundColor: "#3d3f43", height: 30, marginRight: 5, paddingLeft: 5, paddingRight: 5, height: 28, cursor: "pointer", borderColor: "#3d3f43", color: "white", } -const searchClient = algoliasearch("JNSS5CFDZZ", "c8f882473ff42d41158430be09ec2b4e") +const searchClient = algoliasearch("JNSS5CFDZZ", "33e4e3564f4f060e96e0531957bed552") const SearchData = props => { const { serverside, globalUrl, userdata } = props let navigate = useNavigate(); diff --git a/frontend/src/components/Searchfield.jsx b/frontend/src/components/Searchfield.jsx index 4447c73b..57558241 100644 --- a/frontend/src/components/Searchfield.jsx +++ b/frontend/src/components/Searchfield.jsx @@ -38,7 +38,6 @@ import aa from 'search-insights' import { InstantSearch, Configure, connectSearchBox, connectHits, Index } from 'react-instantsearch-dom'; //import { InstantSearch, SearchBox, Hits, connectSearchBox, connectHits, Index } from 'react-instantsearch-dom'; -import { HotKeys } from 'react-hotkeys'; // https://www.algolia.com/doc/api-reference/widgets/search-box/react/ const chipStyle = { backgroundColor: "#3d3f43", height: 30, marginRight: 5, paddingLeft: 5, paddingRight: 5, height: 28, cursor: "pointer", borderColor: "#3d3f43", color: "white", @@ -168,4 +167,4 @@ const SearchField = props => { ) } -export default SearchField; +export default SearchField; \ No newline at end of file diff --git a/frontend/src/components/ShuffleCodeEditor1.jsx b/frontend/src/components/ShuffleCodeEditor1.jsx index 3059d936..c2438b97 100644 --- a/frontend/src/components/ShuffleCodeEditor1.jsx +++ b/frontend/src/components/ShuffleCodeEditor1.jsx @@ -61,8 +61,6 @@ import PaperComponent from "../components/PaperComponent.jsx"; import { padding, textAlign } from '@mui/system'; import data from '../frameworkStyle.jsx'; import { useNavigate, Link, useParams, useSearchParams } from "react-router-dom"; -import { tags as t } from '@lezer/highlight'; - import AceEditor from "react-ace"; import ace from "ace-builds"; @@ -149,7 +147,10 @@ const CodeEditor = (props) => { // Auto-indent JSON-like content (with safety hehe) const autoIndentContent = React.useCallback((content) => { - return content + if (!isFileEditor) { + console.log("Autoindent disabled") + return content + } // Safety checks :) if (!content || typeof content !== 'string' || content.trim().length === 0) { @@ -238,6 +239,24 @@ const CodeEditor = (props) => { expectedOutput(localcodedata) }, [localcodedata]) + useEffect(() => { + if (!isFileEditor) { + return + } + + if (codedata === undefined || codedata === null || typeof codedata !== 'string') { + return + } + + const indentedContent = autoIndentContent(codedata); + if (indentedContent !== undefined && indentedContent !== null) { + console.log("SETTING: ", indentedContent) + setlocalcodedata(indentedContent); + } else { + console.log("INDENT FAILED") + } + }, []) + // Auto-indent when codedata prop changes useEffect(() => { if (codedata && codedata !== localcodedata && typeof codedata === 'string') { @@ -1597,7 +1616,7 @@ const CodeEditor = (props) => { if (e.srcElement.className === "ace_content") { console.log("DRAG STOP IN CONTENT!", e.srcElement.className) - let usedposition = e.offsetY + const usedposition = e.offsetY if (usedposition === undefined || usedposition === null) { toast.info(`Error: LayerY is undefined or null. Please contact ${supportEmail}`) return @@ -1784,8 +1803,8 @@ const CodeEditor = (props) => { // zIndex: 12501, pointerEvents: "auto", color: theme.palette.DialogStyle.color, - minWidth: isMobile || isWorkflowEditor || fullScreenModeEnabled ? "100%" : isFileEditor ? "650px" : "80%", - maxWidth: isMobile || isWorkflowEditor || fullScreenModeEnabled ? "100%" : isFileEditor ? "650px" : "1100px", + minWidth: isMobile || isWorkflowEditor || fullScreenModeEnabled ? "100%" : isFileEditor ? 800 : "80%", + maxWidth: isMobile || isWorkflowEditor || fullScreenModeEnabled ? "100%" : isFileEditor ? 800 : "1100px", minHeight: isMobile || isWorkflowEditor || fullScreenModeEnabled ? "100%" : "auto", maxHeight: isMobile || isWorkflowEditor || fullScreenModeEnabled ? "100%" : "700px", border: "3px solid rgba(255,255,255,0.3)", @@ -1981,7 +2000,8 @@ const CodeEditor = (props) => { paddingLeft: 10, }} > - File Editor ({localcodedata.length}) + {/* cba positioning */} + File Editor ({localcodedata.length})                                                                             {validation === true ? Valid JSON : Invalid JSON}
@@ -2511,7 +2531,7 @@ const CodeEditor = (props) => { } - {(actionId || triggerId || conditionId) && !isWorkflowEditor && !isFileEditor ? + {/*(actionId || triggerId || conditionId) && !isWorkflowEditor && !isFileEditor ? <> { : null - } + */}
} @@ -2583,7 +2603,7 @@ const CodeEditor = (props) => { mode={isWorkflowEditor ? "yaml" : selectedAction === undefined ? "json" : selectedAction.name === "execute_python" ? "python" : selectedAction.name === "execute_bash" ? "bash" : "json"} theme="gruvbox" height={fullScreenModeEnabled ? "84vh" : isFileEditor ? 450 : isWorkflowEditor ? "90vh" : 550} - width={isFileEditor ? 650 : fullScreenModeEnabled ? "50vw" : isWorkflowEditor ? "90vw" : "100%"} + width={isFileEditor ? 800 : fullScreenModeEnabled ? isFileEditor ? "100%" : "50vw" : isWorkflowEditor ? "90vw" : "100%"} markers={markers} highlightActiveLine={false} @@ -2717,7 +2737,7 @@ const CodeEditor = (props) => {
: - {selectedAction.name === "execute_python" || selectedAction.name === "execute_bash" ? + {selectedAction?.name === "execute_python" || selectedAction?.name === "execute_bash" ? "Code to run" : triggerId ? `Output: ${triggerName?.replaceAll("_", " ").slice(0, 1).toUpperCase() + triggerName?.replaceAll("_", " ").slice(1)} (${triggerField})` : diff --git a/frontend/src/components/SubOrgDistributionDialog.jsx b/frontend/src/components/SubOrgDistributionDialog.jsx new file mode 100644 index 00000000..31740c80 --- /dev/null +++ b/frontend/src/components/SubOrgDistributionDialog.jsx @@ -0,0 +1,265 @@ +import React, { useState, useContext } from "react"; +import { + Dialog, + DialogTitle, + DialogContent, + Box, + TextField, + Button, + Typography, + List, + ListItem, + ListItemText, + Checkbox, + InputAdornment, +} from "@mui/material"; +import { Search as SearchIcon } from "@mui/icons-material"; +import { Context } from "../context/ContextApi.jsx"; +import { getTheme } from "../theme.jsx"; + +/** + * A reusable dialog for selecting/distributing sub-organizations. + * + * Props: + * open {boolean} - controls dialog visibility + * onClose {function} - called on Cancel or backdrop click (no args) + * title {string} - dialog title text + * extraInfo {string} - secondary line below the title (e.g. "Selected Key: xxx") + * orgs {Array} - ordered array of { id, name, image? } objects to display + * selectedOrgIds {string[]} - currently selected org IDs (controlled) + * onSelectionChange {function} - called with a updater fn (prev => next) when selection changes + * onSave {function} - called with the final selectedOrgIds array when Save is clicked + * disabled {boolean} - disables checkboxes and Save button (default: false) + */ +const SubOrgDistributionDialog = ({ + open, + onClose, + title, + extraInfo = null, + orgs = [], + selectedOrgIds = [], + onSelectionChange, + onSave, + disabled = false, +}) => { + const [searchQuery, setSearchQuery] = useState(""); + const { themeMode, brandColor } = useContext(Context); + const theme = getTheme(themeMode, brandColor); + + const safeOrgs = orgs || []; + + const filteredOrgs = safeOrgs.filter( + o => o && o.name.toLowerCase().includes(searchQuery.toLowerCase()) + ); + + const handleSelectAll = () => { + if (searchQuery) { + const filteredIds = filteredOrgs.map(o => o.id); + onSelectionChange(prev => [...new Set([...prev, ...filteredIds])]); + } else { + const allIds = safeOrgs.map(o => o.id); + onSelectionChange(prev => [...new Set([...prev, ...allIds])]); + } + }; + + const handleDeselectAll = () => { + if (searchQuery) { + const filteredIds = filteredOrgs.map(o => o.id); + onSelectionChange(prev => prev.filter(id => !filteredIds.includes(id))); + } else { + onSelectionChange([]); + } + }; + + const handleToggle = (id) => { + if (disabled) return; + onSelectionChange(prev => + prev.includes(id) ? prev.filter(i => i !== id) : [...prev, id] + ); + }; + + const handleClose = () => { + setSearchQuery(""); + onClose(); + }; + + const filteredSelected = filteredOrgs.filter(o => selectedOrgIds.includes(o.id)).length; + const countText = searchQuery + ? `${filteredSelected} of ${filteredOrgs.length} filtered selected` + : `${selectedOrgIds.length} of ${safeOrgs.length} selected`; + + const imageSize = 22; + const imageStyle = { width: imageSize, height: imageSize, pointerEvents: "none", marginRight: 10 }; + + return ( + + + + {title} + + {extraInfo && ( + + {extraInfo} + + )} + + + + setSearchQuery(e.target.value)} + InputProps={{ + startAdornment: ( + + + + ), + style: { color: theme.palette.textFieldStyle?.color }, + }} + style={{ backgroundColor: theme.palette.textFieldStyle?.backgroundColor, flexShrink: 0 }} + /> + +
+ + + + {countText} + +
+ +
+ + {filteredOrgs.map((org, index) => { + const isSelected = selectedOrgIds.includes(org.id); + const hasImage = org.image !== undefined; + return ( + handleToggle(org.id)} + sx={{ + cursor: disabled ? "default" : "pointer", + backgroundColor: index % 2 === 0 + ? "transparent" + : themeMode === "dark" ? "rgba(255,255,255,0.02)" : "rgba(0,0,0,0.02)", + "&:hover": { + backgroundColor: themeMode === "dark" ? "rgba(255,255,255,0.05)" : "rgba(0,0,0,0.05)", + }, + }} + > + + {hasImage && ( + org.image === "" ? ( + {org.name} + ) : ( + {org.name} + ) + )} + + + ); + })} + {filteredOrgs.length === 0 && ( + + + + )} + +
+ + + +
+ + +
+ + + ); +}; + +export default SubOrgDistributionDialog; diff --git a/frontend/src/components/TenantsTab.jsx b/frontend/src/components/TenantsTab.jsx index 16cd5f11..779cc910 100644 --- a/frontend/src/components/TenantsTab.jsx +++ b/frontend/src/components/TenantsTab.jsx @@ -1,5 +1,6 @@ import React, { memo, useContext, useEffect, useState } from 'react'; -import {getTheme} from "../theme.jsx"; +import { DataGrid } from '@mui/x-data-grid'; +import { getTheme } from "../theme.jsx"; import { Context } from '../context/ContextApi.jsx'; import { FormControl, @@ -24,9 +25,11 @@ import { IconButton, Modal, Checkbox, - } from "@mui/material"; - - import { + Select, + MenuItem, +} from "@mui/material"; + +import { Edit as EditIcon, Polyline as PolylineIcon, CheckCircle as CheckCircleIcon, @@ -34,11 +37,13 @@ import { Apps as AppsIcon, Business as BusinessIcon, Flag, - ArrowDropDown as ArrowDropDownIcon, + ArrowDropDown as ArrowDropDownIcon, VisibilityOff, Visibility, + KeyboardArrowLeft, + KeyboardArrowRight, - } from "@mui/icons-material"; +} from "@mui/icons-material"; import { toast } from 'react-toastify'; @@ -73,8 +78,15 @@ const TenantsTab = memo((props) => { const theme = getTheme(themeMode, brandColor); const [accountDeleteButtonClicked, setAccountDeleteButtonClicked] = useState(false); const [selectedSuborg, setSelectedSuborg] = useState(null); + const [rowsPerPage, setRowsPerPage] = useState(10); + const [nextCursor, setNextCursor] = useState(""); + const [currentCursor, setCurrentCursor] = useState(""); + const [cursorStack, setCursorStack] = useState([]); + const [localPage, setLocalPage] = useState(0); + const [loadingSubOrgs, setLoadingSubOrgs] = useState(false); + const [isChangingOrg, setIsChangingOrg] = useState(false); useEffect(() => { - if(parentOrg !== null && parentOrgFlag === null) { + if (parentOrg !== null && parentOrgFlag === null) { let regiontag = "UK"; let regionCode = "gb"; @@ -85,25 +97,25 @@ const TenantsTab = memo((props) => { regiontag = namesplit[namesplit.length - 1]; if (regiontag === "california") { - regiontag = "US"; - regionCode = "us"; + regiontag = "US"; + regionCode = "us"; } else if (regiontag === "frankfurt") { - regiontag = "EU-2"; - regionCode = "eu"; + regiontag = "EU-2"; + regionCode = "eu"; } else if (regiontag === "ca") { - regiontag = "CA"; - regionCode = "ca"; - }else if (regiontag === "au") { + regiontag = "CA"; + regionCode = "ca"; + } else if (regiontag === "au") { regiontag = "AUS"; regionCode = "au" } } setParentOrgFlag(regionCode); setParentOrgRegionName(regiontag); + } } - } }, [parentOrg, parentOrgFlag]); - + var syncList = [ { primary: "Workflows", @@ -127,19 +139,26 @@ const TenantsTab = memo((props) => { useEffect(() => { if (userdata.orgs !== undefined && userdata.orgs !== null && userdata.orgs.length > 0) { - handleGetSubOrgs(userdata.active_org.id); + handleGetSubOrgs(userdata.active_org.id, "", 100); } else console.log("error in user data") }, [userdata]); - const handleGetSubOrgs = (orgId) => { + const handleGetSubOrgs = (orgId, cursor = "", limit = 100, direction = "next") => { + const effectiveLimit = limit !== null ? limit : 100; if (orgId.length === 0) { toast("Organization ID not defined. Please contact us on https://shuffler.io if this persists logout."); return; } - fetch(`${globalUrl}/api/v1/orgs/${orgId}/suborgs`, { + setLoadingSubOrgs(true); + let url = `${globalUrl}/api/v1/orgs/${orgId}/suborgs?limit=${effectiveLimit}`; + if (cursor) { + url += `&cursor=${encodeURIComponent(cursor)}`; + } + + fetch(url, { method: "GET", credentials: "include", headers: { @@ -154,49 +173,87 @@ const TenantsTab = memo((props) => { }) .then((responseJson) => { if (responseJson.success === false) { - setLoadOrgs(false) + setLoadOrgs(false); + setLoadingSubOrgs(false); //toast("Failed getting your org. If this persists, please contact support."); } else { - const { subOrgs, parentOrg } = responseJson; - setLoadOrgs(false) - setSubOrgs(subOrgs); + const { subOrgs, parentOrg, cursor: responseCursor } = responseJson; + setLoadOrgs(false); + setLoadingSubOrgs(false); + setSubOrgs(subOrgs || []); setParentOrg(parentOrg); + setNextCursor(responseCursor || ""); + + if (direction === "prev") { + const len = (subOrgs || []).length; + setLocalPage(len > 0 ? Math.ceil(len / rowsPerPage) - 1 : 0); + } else { + setLocalPage(0); + } let regiontag = "UK"; let regionCode = "gb"; if (parentOrg?.region_url?.length > 0) { - const regionsplit = parentOrg?.region_url.split("."); - if (regionsplit.length > 2 && !regionsplit[0].includes("shuffler")) { - const namesplit = regionsplit[0].split("/"); - regiontag = namesplit[namesplit.length - 1]; + const regionsplit = parentOrg?.region_url.split("."); + if (regionsplit.length > 2 && !regionsplit[0].includes("shuffler")) { + const namesplit = regionsplit[0].split("/"); + regiontag = namesplit[namesplit.length - 1]; - if (regiontag === "california") { - regiontag = "US"; - regionCode = "us"; - } else if (regiontag === "frankfurt") { - regiontag = "EU-2"; - regionCode = "eu"; - } else if (regiontag === "ca") { - regiontag = "CA"; - regionCode = "ca"; - }else if (regiontag === "au") { - regiontag = "AUS"; - regionCode = "au" + if (regiontag === "california") { + regiontag = "US"; + regionCode = "us"; + } else if (regiontag === "frankfurt") { + regiontag = "EU-2"; + regionCode = "eu"; + } else if (regiontag === "ca") { + regiontag = "CA"; + regionCode = "ca"; + } else if (regiontag === "au") { + regiontag = "AUS"; + regionCode = "au" + } } - } setParentOrgFlag(regionCode); setParentOrgRegionName(regiontag); - } + } } }) .catch((error) => { console.log("Error getting sub orgs: ", error); //toast("Error getting sub organizations"); - setLoadOrgs(false) + setLoadOrgs(false); + setLoadingSubOrgs(false); }); }; + const handleNextPage = () => { + const maxLocalPage = Math.ceil(subOrgs.length / rowsPerPage) - 1; + if (localPage < maxLocalPage) { + setLocalPage(prev => prev + 1); + } else if (nextCursor && nextCursor !== currentCursor) { + setCursorStack(prev => [...prev, currentCursor]); + setCurrentCursor(nextCursor); + handleGetSubOrgs(userdata.active_org.id, nextCursor, 100, "next"); + } + }; + + const handlePrevPage = () => { + if (localPage > 0) { + setLocalPage(prev => prev - 1); + } else if (cursorStack.length > 0) { + const prevCursor = cursorStack[cursorStack.length - 1]; + setCursorStack(prev => prev.slice(0, -1)); + setCurrentCursor(prevCursor); + handleGetSubOrgs(userdata.active_org.id, prevCursor, 100, "prev"); + } + }; + + const handleChangeRowsPerPage = (newSize) => { + setRowsPerPage(Number(newSize)); + setLocalPage(0); + }; + const GridItem = (props) => { const [expanded, setExpanded] = React.useState(false); const [showEdit, setShowEdit] = React.useState(false); @@ -502,7 +559,7 @@ const TenantsTab = memo((props) => { const createSubOrg = (currentOrgId, name) => { const data = { name: name, org_id: currentOrgId }; const url = globalUrl + `/api/v1/orgs/${currentOrgId}/create_sub_org`; - setSuborglistOpen(true) + setSuborglistOpen(true) fetch(url, { mode: "cors", @@ -520,8 +577,8 @@ const TenantsTab = memo((props) => { if (responseJson["success"] === false) { if (responseJson.reason !== undefined) { toast.error(responseJson.reason, { - autoClose: 5000, - }) + autoClose: 5000, + }) } else { toast("Failed creating suborg. Please try again"); } @@ -554,6 +611,7 @@ const TenantsTab = memo((props) => { localStorage.setItem("globalUrl", ""); localStorage.setItem("getting_started_sidebar", "open"); + setIsChangingOrg(true); fetch(`${globalUrl}/api/v1/orgs/${orgId}/change`, { mode: "cors", credentials: "include", @@ -569,8 +627,8 @@ const TenantsTab = memo((props) => { if (response.status !== 200) { console.log("Error in response"); } else { - localStorage.setItem("apps", []) - } + localStorage.setItem("apps", []) + } return response.json(); }) @@ -607,212 +665,215 @@ const TenantsTab = memo((props) => { const [disabled, setDisabled] = useState(true); const [open, setOpen] = useState(true); const boxStyling = { - position: "relative", - top: "50%", - left: "50%", - transform: "translate(-50%, -50%)", - zIndex: "9999", - backgroundColor: theme.palette.backgroundColor, - color: theme.palette.text.primary, - padding: 20, - borderRadius: 5, - boxShadow: "0 0 10px rgba(0, 0, 0, 0.3)", - width: 430, - height: 430, + position: "relative", + top: "50%", + left: "50%", + transform: "translate(-50%, -50%)", + zIndex: "9999", + backgroundColor: theme.palette.backgroundColor, + color: theme.palette.text.primary, + padding: 20, + borderRadius: 5, + boxShadow: "0 0 10px rgba(0, 0, 0, 0.3)", + width: 430, + height: 430, }; - + const closeIconButtonStyling = { - color: theme.palette.text.primary, - border: "none", - backgroundColor: "transparent", - position: 'relative', - width: 20, - height: 20, - cursor: "pointer", - left: "calc(100% - 30px)", + color: theme.palette.text.primary, + border: "none", + backgroundColor: "transparent", + position: 'relative', + width: 20, + height: 20, + cursor: "pointer", + left: "calc(100% - 30px)", }; - + const handlePasswordVisibility = () => { - setShowPassword(!showPassword); + setShowPassword(!showPassword); }; - + const buttonStyle = { - marginTop: 20, - height: 50, - border: "none", - width: "100%", - fontSize: 16, - backgroundColor: disabled ? "gray" : "red", - color: theme.palette.text.primary, - cursor: disabled === false && "pointer", - }; - + marginTop: 20, + height: 50, + border: "none", + width: "100%", + fontSize: 16, + backgroundColor: disabled ? "gray" : "red", + color: theme.palette.text.primary, + cursor: disabled === false && "pointer", + }; + const handlePasswordChange = (e) => { - setPassword(e.target.value); + setPassword(e.target.value); }; - + const handleCheckBoxEvent = () => { - setUserDeleteAccepted((prev) => !prev); + setUserDeleteAccepted((prev) => !prev); }; - + useEffect(() => { - if (password.length > 8 && userDeleteAccepted) { - setDisabled(false); - } else { - setDisabled(true); - } + if (password.length > 8 && userDeleteAccepted) { + setDisabled(false); + } else { + setDisabled(true); + } }, [password, userDeleteAccepted]); - + const handleDeleteAccount = () => { - const baseURL = globalUrl; - - const url = `${baseURL}/api/v1/orgs/${selectedSuborg?.id}`; + const baseURL = globalUrl; - const data = { - password: password, - }; + const url = `${baseURL}/api/v1/orgs/${selectedSuborg?.id}`; - fetch(url, { - mode: "cors", - method: "DELETE", - body: JSON.stringify(data), - credentials: "include", - crossDomain: true, - withCredentials: true, - headers: { - "Content-Type": "application/json", - }, - }) - .then((response) => response.json()) - .then((data) => { - if (data.success) { - toast.success( - "Suborg deleted" - ); - handleGetSubOrgs(userdata.active_org.id); - setAccountDeleteButtonClicked(false); + const data = { + password: password, + }; - } else { - if (data.reason) { - toast.error(data.reason); - }else { - toast.error("Failed to delete suborg. Please try again or contact support@shuffler.io for help."); - } - } + fetch(url, { + mode: "cors", + method: "DELETE", + body: JSON.stringify(data), + credentials: "include", + crossDomain: true, + withCredentials: true, + headers: { + "Content-Type": "application/json", + }, }) - .catch((error) => { - console.error( - "There was a problem with your fetch operation:", - error - ); - }); + .then((response) => response.json()) + .then((data) => { + if (data.success) { + toast.success( + "Suborg deleted" + ); + setCursorStack([]); + setCurrentCursor(""); + setNextCursor(""); + handleGetSubOrgs(userdata.active_org.id); + setAccountDeleteButtonClicked(false); + + } else { + if (data.reason) { + toast.error(data.reason); + } else { + toast.error("Failed to delete suborg. Please try again or contact support@shuffler.io for help."); + } + } + }) + .catch((error) => { + console.error( + "There was a problem with your fetch operation:", + error + ); + }); }; - + return ( - -
- { - setAccountDeleteButtonClicked(false); - setSelectedSuborg(null); - }} - > - - -

Sub-Organization

- {/*
*/} -
- -
    -
  • - -
    • -
  • - -
    • -
-
- - -
-
- - +
+ { + setAccountDeleteButtonClicked(false); + setSelectedSuborg(null); + }} + > + + +

Sub-Organization

+ {/*
*/} +
+ +
    +
  • + +
    • +
  • + +
    • +
+
- {showPassword ? : } - - ), - }} - /> -
- -
-
- + + +
+
+ + + {showPassword ? : } + + ), + }} + /> +
+ +
+
+ ); - }; + }; const modalView = ( { backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, zIndex: 1000, '& .MuiDialogContent-root': { - backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, }, '& .MuiDialogTitle-root': { - backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, }, '& .MuiDialogActions-root': { - backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, }, }, }} @@ -880,7 +941,7 @@ const TenantsTab = memo((props) => { - - - -
- - Your Parent Organization - -
-
- - - {/* */} - - -
- - - - - {isCloud && ( - - )} - - - - - {loadOrgs ? ( - [...Array(3)].map((_, rowIndex) => ( - - {[ - { width: 100, minWidth: 100, maxWidth: 100 }, - { width: 250, minWidth: 50, maxWidth: 250 }, - { width: 400, minWidth: 400, maxWidth: 400 }, - { width: "28%", minWidth: "28%" }, - { width: 400, minWidth: 400, maxWidth: 400 }, - ].map((style, colIndex) => ( - - - - ))} - - )) - ) : parentOrg?.id?.length > 0 ? ( - - - } - style={{ - width: 100, - minWidth: 100, - maxWidth: 100, - display: "table-cell", - padding: "8px 8px 8px 20px", - textAlign: "center", - }} - /> - - {isCloud && ( - - {parentOrgFlag} - -
- } - style={{ display: "table-cell", padding: 8, verticalAlign: "middle" }} - /> - )} - - - - - - - } - style={{ display: "table-cell", verticalAlign: "middle" }} - /> - - ): ( - - {Array(5).fill().map((_, index) => ( - - ))} - - )} - -
-
- - {subOrgs.length > 0 && ( -
- - -
- - Sub Organizations of the Current Organization ({subOrgs.length}) - -
+ Change Active Org + + + + {!selectedOrganization?.creator_org?.length && ( + + )} +
+ ), + }, + ]; + + return ( +
+ {modalView} + {cloudSyncModal} + {accountDeleteButtonClicked && } +
+
+
+ Tenants + + Create, manage and change to sub-organizations (tenants)! {" "} + {isCloud + ? `You can only make a sub organization if you are a customer of shuffle or running a POC of the platform. Please contact ${supportEmail} to try it out.` + : ''}  + + Learn more + + +
+ + + + + +
+ + Your Parent Organization + +
+
+ {/* { }} /> */} -
- - {!suborglistOpen ? - - setSuborglistOpen(true)} - > - Show Sub-Organizations - - } - style={{ - width: 100, - minWidth: 100, - maxWidth: 100, - paddingLeft: 20, - display: "table-cell", - padding: "0px 8px 8px 8px", - textAlign: "center", - borderBottom: theme.palette.defaultBorder, - verticalAlign: "middle", - }} - /> - - : - - - - - {isCloud && ( - - )} - - - - {subOrgs.map((data, index) => { - let regiontag = "UK"; - let regionCode = "gb"; +
+ + + + + {isCloud && ( + + )} + + + - if (data.region_url?.length > 0) { - const regionsplit = data.region_url.split("."); - if (regionsplit.length > 2 && !regionsplit[0].includes("shuffler")) { - const namesplit = regionsplit[0].split("/"); - regiontag = namesplit[namesplit.length - 1]; - if (regiontag === "california") { - regiontag = "US"; - regionCode = "us"; - } else if (regiontag === "frankfurt") { - regiontag = "EU-2"; - regionCode = "eu"; - } else if (regiontag === "ca") { - regiontag = "CA"; - regionCode = "ca"; - } - } - } - var bgColor = themeMode === "dark" ? "#212121" : "#FFFFFF"; - if (index % 2 === 0) { - bgColor = themeMode === "dark" ? "#1A1A1A" : "#EAEAEA"; - } - - return ( - - } style={{ width: 100, - minWidth: 100, - maxWidth: 100, - display: "table-cell", - padding: "8px 8px 8px 20px", - textAlign: "center", }} /> - - - {isCloud && ( - - {regiontag} - -
- } - style={{ display: "table-cell", padding: 8, verticalAlign: "middle" }} - /> - )} - - - - - - - - {selectedOrganization?.creator_org?.length > 0 ? null : -
+ } + style={{ display: "table-cell", padding: 8, verticalAlign: "middle" }} + /> + )} + - Delete Org - } - - - } - style={{ display: "table-cell", verticalAlign: "middle" }} - /> - - )})} - - } - - + /> + + + + + + } + style={{ display: "table-cell", verticalAlign: "middle" }} + /> + + ) : ( + + {Array(5).fill().map((_, index) => ( + + ))} + + )} +
- )} - + {(subOrgs.length > 0 || cursorStack.length > 0 || loadingSubOrgs) && ( +
+ -
- + + Sub Organizations of the Current Organization ({subOrgs.length}) + +
+ +
+ {!suborglistOpen ? ( + + ) : ( + row.id} + sx={{ + border: "none", + color: theme.palette.text.primary, + '& .MuiDataGrid-columnHeaders': { + borderBottom: theme.palette.defaultBorder, + backgroundColor: theme.palette.platformColor, + }, + '& .MuiDataGrid-cell': { + borderBottom: theme.palette.defaultBorder, + display: "flex", + alignItems: "center", + }, + '& .MuiDataGrid-row': { + backgroundColor: theme.palette.platformColor, + }, + '& .MuiDataGrid-overlayWrapper': { + minHeight: subOrgs.length > 0 ? 0 : 100, + }, + }} + /> + )} +
+ {suborglistOpen && ( +
+ + Rows per page: + + + + + + = Math.ceil(subOrgs.length / rowsPerPage) - 1 && (!nextCursor || nextCursor === currentCursor))} + size="small" + sx={{ color: (loadingSubOrgs || (localPage >= Math.ceil(subOrgs.length / rowsPerPage) - 1 && (!nextCursor || nextCursor === currentCursor))) ? theme.palette.text.disabled : theme.palette.text.primary }} + > + + +
+ )} +
+ )} + + - All Tenants - -
+ /> - {/* + + All Tenants + +
+ + {/* */} -
- - {!allTenantsOpen ? - + + {!allTenantsOpen ? + - setAllTenantsOpen(true)} - > - Show ALL your tenants - - } - style={{ - width: 100, - minWidth: 100, - maxWidth: 100, - paddingLeft: 20, - display: "table-cell", - padding: "0px 8px 8px 8px", - textAlign: "center", - borderBottom: theme.palette.defaultBorder, - verticalAlign: "middle", - }} - /> - - : - - - - - {isCloud && ( - - )} - - - + }} + > + setAllTenantsOpen(true)} + > + Show ALL your tenants + + } + style={{ + width: 100, + minWidth: 100, + maxWidth: 100, + paddingLeft: 20, + display: "table-cell", + padding: "0px 8px 8px 8px", + textAlign: "center", + borderBottom: theme.palette.defaultBorder, + verticalAlign: "middle", + }} + /> + + : + + + + + {isCloud && ( + + )} + + + - {userdata?.orgs?.length <= 0 ? ( - [...Array(6)].map((_, rowIndex) => ( - - {Array(7) - .fill() - .map((_, colIndex) => ( - - - - ))} - - )) - ) : ( - userdata?.orgs?.length > 0 && - userdata.orgs.map((data, index) => { - let regiontag = "UK"; - let regionCode = "gb"; + {userdata?.orgs?.length <= 0 ? ( + [...Array(6)].map((_, rowIndex) => ( + + {Array(7) + .fill() + .map((_, colIndex) => ( + + + + ))} + + )) + ) : ( + userdata?.orgs?.length > 0 && + userdata.orgs.map((data, index) => { + let regiontag = "UK"; + let regionCode = "gb"; - if (data.region_url?.length > 0) { - const regionsplit = data.region_url.split("."); - if (regionsplit.length > 2 && !regionsplit[0].includes("shuffler")) { - const namesplit = regionsplit[0].split("/"); - regiontag = namesplit[namesplit.length - 1]; + if (data.region_url?.length > 0) { + const regionsplit = data.region_url.split("."); + if (regionsplit.length > 2 && !regionsplit[0].includes("shuffler")) { + const namesplit = regionsplit[0].split("/"); + regiontag = namesplit[namesplit.length - 1]; - if (regiontag === "california") { - regiontag = "US"; - regionCode = "us"; - } else if (regiontag === "frankfurt") { - regiontag = "EU-2"; - regionCode = "eu"; - } else if (regiontag === "ca") { - regiontag = "CA"; - regionCode = "ca"; - } - } - } + if (regiontag === "california") { + regiontag = "US"; + regionCode = "us"; + } else if (regiontag === "frankfurt") { + regiontag = "EU-2"; + regionCode = "eu"; + } else if (regiontag === "ca") { + regiontag = "CA"; + regionCode = "ca"; + } + } + } - var bgColor = themeMode === "dark" ? "#212121" : "#FFFFFF"; - if (index % 2 === 0) { - bgColor = themeMode === "dark" ? "#1A1A1A" : "#EAEAEA"; - } + var bgColor = themeMode === "dark" ? "#212121" : "#FFFFFF"; + if (index % 2 === 0) { + bgColor = themeMode === "dark" ? "#1A1A1A" : "#EAEAEA"; + } - return ( - - - } - style={{ - width: 100, - minWidth: 100, - maxWidth: 100, - display: "table-cell", - padding: "8px 8px 8px 20px", - textAlign: "center", - }} - /> - - {isCloud ? ( - - {regiontag} + return ( + + + } + style={{ + width: 100, + minWidth: 100, + maxWidth: 100, + display: "table-cell", + padding: "8px 8px 8px 20px", + textAlign: "center", + }} + /> + + {isCloud ? ( + + {regiontag} - -
- } - style={{ - display: "table-cell", - padding: 8, - verticalAlign: "middle", - }} - > - ) : null} - - { - handleClickChangeOrg(data?.id); - }} - > - Change Active Org - - } - style={{ - display: "table-cell", - padding: 8, - verticalAlign: "middle", - }} - > - - ); - }) - )} - } - -
+ +
+ } + style={{ + display: "table-cell", + padding: 8, + verticalAlign: "middle", + }} + > + ) : null} + + { + handleClickChangeOrg(data?.id); + }} + > + Change Active Org + + } + style={{ + display: "table-cell", + padding: 8, + verticalAlign: "middle", + }} + > + + ); + }) + )} + } + +
diff --git a/frontend/src/components/UserManagmentTab.jsx b/frontend/src/components/UserManagmentTab.jsx index 41ab96f0..a27fdedf 100644 --- a/frontend/src/components/UserManagmentTab.jsx +++ b/frontend/src/components/UserManagmentTab.jsx @@ -1,11 +1,10 @@ import React, { useState, useEffect, useContext, memo } from "react"; import { toast } from 'react-toastify'; import { Context } from "../context/ContextApi.jsx"; +import { Link } from "react-router-dom"; import { FormControl, InputLabel, - OutlinedInput, - Checkbox, Tooltip, Typography, Select, @@ -31,23 +30,12 @@ import { import { Cached as CachedIcon, Edit as EditIcon, - Style, } from "@mui/icons-material"; import ModeEditOutlineOutlinedIcon from '@mui/icons-material/ModeEditOutlineOutlined'; import ContentCopyOutlinedIcon from '@mui/icons-material/ContentCopyOutlined'; import {getTheme} from "../theme.jsx"; -const ITEM_HEIGHT = 48; -const ITEM_PADDING_TOP = 8; -const MenuProps = { - PaperProps: { - style: { - maxHeight: ITEM_HEIGHT * 4.5 + ITEM_PADDING_TOP, - width: 500, - }, - }, - getContentAnchorEl: () => null, -}; +import SubOrgDistributionDialog from "./SubOrgDistributionDialog.jsx"; const logsViewModal = false; const userdata = ""; @@ -76,10 +64,11 @@ const UserManagmentTab = memo((props) => { const [logsViewModal, setLogsViewModal] = React.useState(false); const [ipSelected, setIpSelected] = React.useState(""); const [userLogViewing, setUserLogViewing] = React.useState({}); + const [subOrgModalOpen, setSubOrgModalOpen] = React.useState(false); + const [pendingSubOrgs, setPendingSubOrgs] = React.useState([]); const { themeMode, supportEmail, brandColor } = useContext(Context); const theme = getTheme(themeMode, brandColor); - useEffect(() => { if (selectedOrganization?.mfa_required !== MFARequired) { setMFARequired(selectedOrganization?.mfa_required); @@ -247,30 +236,6 @@ const UserManagmentTab = memo((props) => { }); }; - const handleOrgEditChange = (event) => { - if (userdata.id === selectedUser.id) { - toast("Can't remove orgs from yourself"); - return; - } - - if (event.target.value.includes("ALL")) { - toast.info("Adding to available all sub-organizations. This may take a minute.") - event.target.value = selectedOrganization.child_orgs.map((org) => org.id) - } else if (event.target.value.includes("None")) { - toast.info("Removing from all sub-organizations. This may take a minute") - event.target.value = [] - } - - setMatchingOrganizations(event.target.value); - // Workaround for empty orgs - if (event.target.value.length === 0) { - event.target.value.push("REMOVE"); - } - - setUser(selectedUser.id, "suborgs", event.target.value); - //setUser(selectedUser.id, "suborgs", matchingOrganizations) - }; - const userOrgEdit = selectedUser.id !== undefined && selectedUser?.orgs !== undefined && @@ -278,44 +243,44 @@ const UserManagmentTab = memo((props) => { selectedOrganization?.child_orgs !== undefined && selectedOrganization?.child_orgs !== null && selectedOrganization?.child_orgs?.length > 0 ? ( - - - Accessible Sub-Organizations ( - {selectedUser?.orgs ? selectedUser?.orgs?.length - 1 : 0}) - - - + ) : null; + const subOrgManagementDialog = ( + setSubOrgModalOpen(false)} + title={`Manage Sub-Organizations for ${selectedUser?.username || ''}`} + orgs={selectedOrganization?.child_orgs || []} + selectedOrgIds={pendingSubOrgs} + onSelectionChange={setPendingSubOrgs} + onSave={(ids) => { + if (userdata.id === selectedUser.id) { + toast("Can't modify orgs for yourself"); + return; + } + const newValue = ids.length === 0 ? ["REMOVE"] : [...ids]; + setMatchingOrganizations([...ids]); + setUser(selectedUser.id, "suborgs", newValue); + setSubOrgModalOpen(false); + setSelectedUserModalOpen(false); + }} + disabled={selectedUser?.id === userdata?.id} + /> + ); + const getUsers = () => { fetch(globalUrl + "/api/v1/getusers", { method: "GET", @@ -1117,6 +1082,8 @@ const UserManagmentTab = memo((props) => { }); }; + var previousreferrer = "" + var nextreferrer = "" const logview = logsViewModal ? ( { onChange={(event) => { setIpSelected(event.target.value); getLogs(event.target.value, userLogViewing.id); - - }} > {(() => { const uniqueIPs = new Set(); + console.log("Login info: ", userLogViewing.login_info) return userLogViewing.login_info.map((data, index) => { - if ( - data.ip.includes("127.0.0.1") || - uniqueIPs.has(data.ip) - ) { - return null; + console.log("Data: ", data) + if (data.ip.includes("127.0.0.1") || uniqueIPs.has(data.ip)) { + return null } - uniqueIPs.add(data.ip); + uniqueIPs.add(data.ip) return ( - {data.ip} + {data?.timestamp ? new Date(data.timestamp * 1000).toLocaleString() : "N/A"} - {data?.ip} ); }); @@ -1229,12 +1193,13 @@ const UserManagmentTab = memo((props) => { minWidth: 700, maxWidth: 700, overflow: "hidden", - marginLeft: 10, + marginLeft: 50, }} /> {logs.map((data, index) => { - //console.log("LOG: ", data) + previousreferrer = nextreferrer + nextreferrer = data.referer return ( // redirect user to logs @@ -1243,6 +1208,8 @@ const UserManagmentTab = memo((props) => { key={index} style={{ backgroundColor: index % 2 === 0 ? "#1f2023" : "#27292d", + paddingTop: data.referer !== previousreferrer ? 50 : 0, + borderTop: data.referer !== previousreferrer ? `1px solid rgba(255,255,255,0.3)` : "none", }} > { }} /> - + + + )})} @@ -1290,6 +1260,7 @@ const UserManagmentTab = memo((props) => {
{modalView} {editUserModal} + {subOrgManagementDialog} {logview}
diff --git a/frontend/src/components/WorkflowGrid.jsx b/frontend/src/components/WorkflowGrid.jsx index e4a34134..9e843206 100644 --- a/frontend/src/components/WorkflowGrid.jsx +++ b/frontend/src/components/WorkflowGrid.jsx @@ -3,6 +3,7 @@ import React, { useEffect, useState } from 'react'; import {Link} from 'react-router-dom'; import theme from '../theme.jsx'; import { removeQuery } from '../components/ScrollToTop.jsx'; +import SearchContactForm from '../components/SearchContactForm.jsx'; import { Search as SearchIcon, CloudQueue as CloudQueueIcon, Code as CodeIcon } from '@mui/icons-material'; @@ -25,66 +26,23 @@ import { useDebouncedCallback } from "../utils/useDebouncedCallback.jsx"; import WorkflowPaper from "../components/WorkflowPaper.jsx" import WorkflowPaperNew from "../components/WorkflowPaperNew.jsx" -const searchClient = algoliasearch("JNSS5CFDZZ", "c8f882473ff42d41158430be09ec2b4e") +const searchClient = algoliasearch("JNSS5CFDZZ", "eb5fd80aa6ed5ab4730d836cff3ea283") const AppGrid = props => { const { maxRows, showName, showSuggestion, isMobile, globalUrl, parsedXs, alternativeView, onlyResults, inputsearch } = props - const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; + const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; const rowHandler = maxRows === undefined || maxRows === null ? 50 : maxRows const xs = parsedXs === undefined || parsedXs === null ? isMobile ? 6 : 4 : parsedXs //const [apps, setApps] = React.useState([]); //const [filteredApps, setFilteredApps] = React.useState([]); - const [formMail, setFormMail] = React.useState(""); - const [message, setMessage] = React.useState(""); - const [formMessage, setFormMessage] = React.useState(""); const [usecases, setUsecases] = React.useState([]); const [localMessage, setLocalMessage] = React.useState(""); - const buttonStyle = {borderRadius: 30, height: 50, width: 220, margin: isMobile ? "15px auto 15px auto" : 20, fontSize: 18,} - const innerColor = "rgba(255,255,255,0.65)" const borderRadius = 3 window.title = "Shuffle | Workflows | Discover your use-case" - const submitContact = (email, message) => { - const data = { - "firstname": "", - "lastname": "", - "title": "", - "companyname": "", - "email": email, - "phone": "", - "message": message, - } - - const errorMessage = "Something went wrong. Please contact frikky@shuffler.io directly." - - fetch(globalUrl+"/api/v1/contact", { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(data), - }) - .then(response => response.json()) - .then(response => { - if (response.success === true) { - setFormMessage(response.reason) - //toast("Thanks for submitting!") - } else { - setFormMessage(errorMessage) - } - - setFormMail("") - setMessage("") - }) - .catch(error => { - setFormMessage(errorMessage) - console.log(error) - }); - } - const handleKeysetting = (categorydata, workflows) => { console.log("Workflows: ", workflows) //workflows[0].category = ["detect"] @@ -229,10 +187,16 @@ const AppGrid = props => { placeholder="Find Workflows..." id="shuffle_search_field" onChange={(event) => { - removeQuery("q") const value = event.currentTarget.value setInputValue(value) debouncedRefine(value) + const urlSearchParams = new URLSearchParams(window.location.search) + if (value) { + urlSearchParams.set("q", value) + } else { + urlSearchParams.delete("q") + } + window.history.replaceState(null, "", value ? `?${urlSearchParams.toString()}` : window.location.pathname) }} onKeyDown={(event) => { if(event.key === "Enter") { @@ -333,64 +297,10 @@ const AppGrid = props => { {showSuggestion === true ? -
- - Can't find what you're looking for? - -
- setFormMail(e.target.value)} - /> - setMessage(e.target.value)} - /> -
- - {formMessage} -
- : null + + : null } - {onlyResults === true ? null : + {/* {onlyResults === true ? null : Search by @@ -399,7 +309,7 @@ const AppGrid = props => { Algolia logo - } + } */}
) } diff --git a/frontend/src/components/Workflowsearch.jsx b/frontend/src/components/Workflowsearch.jsx index 1b342645..44f77b98 100644 --- a/frontend/src/components/Workflowsearch.jsx +++ b/frontend/src/components/Workflowsearch.jsx @@ -10,7 +10,7 @@ import algoliasearch from 'algoliasearch'; import { InstantSearch, connectSearchBox, connectHits } from 'react-instantsearch-dom'; import { Grid, Paper, TextField, ButtonBase, InputAdornment, Typography, Button, Tooltip} from '@mui/material'; -const searchClient = algoliasearch("JNSS5CFDZZ", "c8f882473ff42d41158430be09ec2b4e") +const searchClient = algoliasearch("JNSS5CFDZZ", "33e4e3564f4f060e96e0531957bed552") const WorkflowSearch = props => { const { maxRows, showName, showSuggestion, isMobile, globalUrl, parsedXs, newSelectedApp, setNewSelectedApp, defaultSearch, showSearch, ConfiguredHits, selectAble, } = props const rowHandler = maxRows === undefined || maxRows === null ? 50 : maxRows diff --git a/frontend/src/components/ssoTab.jsx b/frontend/src/components/ssoTab.jsx index c361e1b1..1d019800 100644 --- a/frontend/src/components/ssoTab.jsx +++ b/frontend/src/components/ssoTab.jsx @@ -1,16 +1,16 @@ import { useEffect, useContext } from "react"; import React from "react"; -import { - Typography, - Switch, - Button, - Tooltip, - TextField, - Grid, +import { + Typography, + Switch, + Button, + Tooltip, + TextField, + Grid, Checkbox } from "@mui/material"; import { makeStyles } from "@mui/styles"; -import { Link } from "react-router-dom"; +import { Link, useSearchParams } from "react-router-dom"; import theme from "../theme.jsx"; import { toast } from "react-toastify"; import { Context } from "../context/ContextApi.jsx"; @@ -28,7 +28,13 @@ const SSOTab = ({selectedOrganization, userdata, isEditOrgTab, globalUrl, handle // Check if user is admin const isAdmin = userdata?.active_org?.role === "admin" || userdata?.support === true; - + + // Read region_url override from URL params (only allow shuffler.io domains) + const [searchParams] = useSearchParams(); + const rawRegionUrl = searchParams.get("region_url"); + const regionUrlOverride = rawRegionUrl && rawRegionUrl.includes("shuffler.io") ? rawRegionUrl : null; + const effectiveGlobalUrl = regionUrlOverride || globalUrl; + // State for tracking user SSO connection status const [users, setUsers] = React.useState([]); const [userSSOConnected, setUserSSOConnected] = React.useState(false); @@ -109,7 +115,7 @@ const SSOTab = ({selectedOrganization, userdata, isEditOrgTab, globalUrl, handle // Function to fetch users and check current user's SSO status const checkUserSSOStatus = () => { setCheckingSSOStatus(true); - fetch(globalUrl + "/api/v1/getusers", { + fetch(effectiveGlobalUrl + "/api/v1/getusers", { method: "GET", headers: { "Content-Type": "application/json", @@ -309,7 +315,7 @@ const SSOTab = ({selectedOrganization, userdata, isEditOrgTab, globalUrl, handle }; const HandleTestSSO = () => { - const url = `${globalUrl}/api/v1/orgs/${selectedOrganization?.id}/change`; + const url = `${effectiveGlobalUrl}/api/v1/orgs/${selectedOrganization?.id}/change`; const data = { org_id: selectedOrganization?.id, sso: true, @@ -366,7 +372,7 @@ const SSOTab = ({selectedOrganization, userdata, isEditOrgTab, globalUrl, handle }; const HandleDisconnectSSO = () => { - const url = `${globalUrl}/api/v1/disconnect_sso`; + const url = `${effectiveGlobalUrl}/api/v1/disconnect_sso`; const data = { org_id: selectedOrganization?.id, }; @@ -444,6 +450,11 @@ const SSOTab = ({selectedOrganization, userdata, isEditOrgTab, globalUrl, handle : "Connect your account with this org's SSO!" } + {regionUrlOverride && ( + + Using region override: {regionUrlOverride} + + )} { - // Handle "system" mode by checking user's system preference - let resolvedMode = themeMode; - if (themeMode === "system" || !themeMode) { - resolvedMode = window?.matchMedia?.("(prefers-color-scheme: dark)")?.matches ? "dark" : "light"; - } - // Ensure mode is only "dark" or "light" - if (resolvedMode !== "dark" && resolvedMode !== "light") { - resolvedMode = "dark"; - } - - return createTheme({ +export const getTheme = (themeMode, brandColor) => + createTheme({ palette: { - mode: resolvedMode, + mode: themeMode, main: brandColor || "#FF8544", primary: { main: brandColor || "#FF8544", @@ -177,35 +167,36 @@ export const getTheme = (themeMode, brandColor) => { contrastText:"#000000", }, text: { - primary: resolvedMode === "dark" ? "#ffffff" : "#1A1A1A", - secondary: resolvedMode === "dark" ? "#9E9E9E" : "#616161", + primary: themeMode === "dark" ? "#ffffff" : "#1A1A1A", + secondary: themeMode === "dark" ? "#9E9E9E" : "#616161", }, - type: resolvedMode, - inputColor: resolvedMode === "dark" ? "rgba(39,41,45,1)" : "rgba(245, 245, 245, 1)", - textColor: resolvedMode === "dark" ? "#F1F1F1" : "#1A1A1A", - textPrimary: resolvedMode === "dark" ? "rgba(255, 255, 255, 0.8)" : "rgba(26, 26, 26, 0.8)", - surfaceColor: resolvedMode === "dark" ? "#27292d" : "#EFEFEF", - platformColor: resolvedMode === "dark" ? "#212121" : "#ffffff", - backgroundColor: resolvedMode === "dark" ? "#1a1a1a" : "#f1f1f1", - cytoscapeBackgroundColor: resolvedMode === "dark" ? "#161616" : "#f5f5f5", - distributionColor: resolvedMode === "dark" ? "#40E0D0" : "#008080", - cardBackgroundColor: resolvedMode === "dark" ? "#1e1e1e" : "#eaeaea", - cardHoverColor: resolvedMode === "dark" ? "#323232" : "#F0F0F0", - hoverColor: resolvedMode === "dark" ? "#323232" : "#D6D6D6", - usecaseCardColor: resolvedMode === "dark" ? "#2f2f2f" : "rgba(245, 245, 245, 1)", - usecaseCardHoverColor: resolvedMode === "dark" ? "#2F2F2F" : "rgba(245, 245, 245, 1)", - usecaseDialogFieldColor: resolvedMode === "dark" ? "#2B2B2B" : "#F5F5F5", - accentColor: resolvedMode === "dark" ? "#ff8544" : "#ff8544", - green: resolvedMode === "dark" ? "#5cc879" : "#008000", - defaultBorder: resolvedMode === "dark" ? '1px solid #494949' : '1px solid #CCCCCC', + type: themeMode, + inputColor: themeMode === "dark" ? "rgba(39,41,45,1)" : "rgba(245, 245, 245, 1)", + textColor: themeMode === "dark" ? "#F1F1F1" : "#1A1A1A", + textPrimary: themeMode === "dark" ? "rgba(255, 255, 255, 0.8)" : "rgba(26, 26, 26, 0.8)", + surfaceColor: themeMode === "dark" ? "#27292d" : "#EFEFEF", + platformColor: themeMode === "dark" ? "#212121" : "#ffffff", + backgroundColor: themeMode === "dark" ? "#1a1a1a" : "#f1f1f1", + cytoscapeBackgroundColor: themeMode === "dark" ? "#161616" : "#f5f5f5", + distributionColor: themeMode === "dark" ? "#40E0D0" : "#008080", + cardBackgroundColor: themeMode === "dark" ? "#1e1e1e" : "#eaeaea", + cardHoverColor: themeMode === "dark" ? "#323232" : "#F0F0F0", + hoverColor: themeMode === "dark" ? "#323232" : "#D6D6D6", + usecaseCardColor: themeMode === "dark" ? "#2f2f2f" : "rgba(245, 245, 245, 1)", + usecaseCardHoverColor: themeMode === "dark" ? "#2F2F2F" : "rgba(245, 245, 245, 1)", + usecaseDialogFieldColor: themeMode === "dark" ? "#2B2B2B" : "#F5F5F5", + accentColor: themeMode === "dark" ? "#ff8544" : "#ff8544", + green: themeMode === "dark" ? "#5cc879" : "#008000", + defaultBorder: themeMode === "dark" ? '1px solid #494949' : '1px solid #CCCCCC', linkColor: brandColor === "#ff8544" ? "#f86a3e" : brandColor, - slateGrayColor: resolvedMode === "dark" ? "#494949" : "#CCCCCC", - parsedAppPaperColor: resolvedMode === "dark" ? "#2f2f2f" : "#CCCCCC", - + slateGrayColor: themeMode === "dark" ? "#494949" : "#CCCCCC", + parsedAppPaperColor: themeMode === "dark" ? "#2f2f2f" : "#CCCCCC", + welcomeCardSubtextColor: themeMode === "dark" ? "#C8C8C8" : "#2f2f2f", + deleteColor: themeMode === "dark" ? "#FD4C62" : "#d32f2f", borderRadius: 10, - loaderColor: resolvedMode === "dark" ? "#1a1a1a" : "#E0E0E0", + loaderColor: themeMode === "dark" ? "#1a1a1a" : "#E0E0E0", jsonIconStyle: "round", - jsonTheme: resolvedMode === "dark" ? "summerfruit" : { + jsonTheme: themeMode === "dark" ? "summerfruit" : { base00: "#ffffff", // background base01: "#f0f0f0", // very light grey base02: "#f5f5f5", // light grey @@ -225,11 +216,11 @@ export const getTheme = (themeMode, brandColor) => { }, jsonCollapseStringsAfterLength: 100, drawer: { - backgroundColor: resolvedMode === "dark" ? "#262626" : "#f9f9f9" + backgroundColor: themeMode === "dark" ? "#262626" : "#f9f9f9" }, actionSidebarField: { - backgroundColor: resolvedMode === "dark" ? "#2F2F2F" : "#F1F1F1", - color: resolvedMode === "dark" ? "#ffffff" : "#000000", + backgroundColor: themeMode === "dark" ? "#2F2F2F" : "#F1F1F1", + color: themeMode === "dark" ? "#ffffff" : "#000000", borderRadius: 8, height: 40, border: "none", @@ -238,53 +229,53 @@ export const getTheme = (themeMode, brandColor) => { padding: 5, width: "98%", borderRadius: 5, - border: resolvedMode === "dark" ? "1px solid rgba(255,255,255,0.7)" : "1px solid rgba(0,0,0,0.3)", - backgroundColor: resolvedMode === "dark" + border: themeMode === "dark" ? "1px solid rgba(255,255,255,0.7)" : "1px solid rgba(0,0,0,0.3)", + backgroundColor: themeMode === "dark" ? "#1A1A1A" : "#f1f1f1", - color: resolvedMode === "dark" + color: themeMode === "dark" ? "#F1F1F1" : "#1A1A1A", overflowX: "auto", }, textFieldStyle: { - backgroundColor: resolvedMode === "dark" ? "#212121" : "#FFFFFF", - color: resolvedMode === "dark" ? "#ffffff" : "#000000", + backgroundColor: themeMode === "dark" ? "#212121" : "#FFFFFF", + color: themeMode === "dark" ? "#ffffff" : "#000000", borderRadius: "5px", height: 40, - border: resolvedMode === "dark" ? "1px solid #4D4D4D" : "1px solid #E0E0E0", + border: themeMode === "dark" ? "1px solid #4D4D4D" : "1px solid #E0E0E0", }, DialogStyle: { - backgroundColor: resolvedMode === "dark" ? "#212121" : "#ffffff", + backgroundColor: themeMode === "dark" ? "#212121" : "#ffffff", borderRadius: 2, - boxShadow: resolvedMode === "dark" ? "0px 0px 10px 0px rgba(0,0,0,0.75)" : "0px 0px 10px 0px rgba(0,0,0,0.2)", - border: resolvedMode === "dark" ? "1px solid #494949" : "1px solid #cccccc", + boxShadow: themeMode === "dark" ? "0px 0px 10px 0px rgba(0,0,0,0.75)" : "0px 0px 10px 0px rgba(0,0,0,0.2)", + border: themeMode === "dark" ? "1px solid #494949" : "1px solid #cccccc", }, innerTextfieldStyle: { height: 40, fontSize: 16, - backgroundColor: resolvedMode === "dark" ? "#212121" : "#f5f5f5", + backgroundColor: themeMode === "dark" ? "#212121" : "#f5f5f5", }, tooltip: { - backgroundColor: resolvedMode === "dark" ? "#212121" : "#ffffff", - color: resolvedMode === "dark" ? "#ffffff" : "#000000", - border: resolvedMode === "dark" ? "1px solid #494949" : "1px solid #cccccc", + backgroundColor: themeMode === "dark" ? "#212121" : "#ffffff", + color: themeMode === "dark" ? "#ffffff" : "#000000", + border: themeMode === "dark" ? "1px solid #494949" : "1px solid #cccccc", }, chipStyle: { - backgroundColor: resolvedMode === "dark" ? "#333333" : "#F5F5F5", - borderColor: resolvedMode === "dark" ? "#444444" : "#E0E0E0", - color: resolvedMode === "dark" ? "#FFFFFF" : "#333333", + backgroundColor: themeMode === "dark" ? "#333333" : "#F5F5F5", + borderColor: themeMode === "dark" ? "#444444" : "#E0E0E0", + color: themeMode === "dark" ? "#FFFFFF" : "#333333", }, defaultImage: "/images/no_image.png", singulOrange: "/images/singul_orange.png", singulGreen: "/images/singul_green.png", singulBlackWhite: "/icons/workflow-page/shuffle_agent.png", - scrollbarColor: resolvedMode === "dark" ? "#494949 #2f2f2f": "#c1c1c1 #f1f1f1", - scrollbarColorTransparent: resolvedMode === "dark" ? '#494949 transparent': "#c1c1c1 transparent", + scrollbarColor: themeMode === "dark" ? "#494949 #2f2f2f": "#c1c1c1 #f1f1f1", + scrollbarColorTransparent: themeMode === "dark" ? '#494949 transparent': "#c1c1c1 transparent", }, typography: { fontFamily: `"inter", "Roboto", "Helvetica", "Arial", sans-serif`, - color: resolvedMode === "dark" ? "#ffffff" : "#000000", + color: themeMode === "dark" ? "#ffffff" : "#000000", useNextVariants: true, fontWeightLight: 300, fontWeightRegular: 400, @@ -292,36 +283,36 @@ export const getTheme = (themeMode, brandColor) => { fontWeightSemiBold: 600, fontWeightBold: 700, allVariants: { - color: resolvedMode === "dark" ? "#ffffff" : "#1A1A1A", + color: themeMode === "dark" ? "#ffffff" : "#1A1A1A", }, h1: { fontSize: 40, - color: resolvedMode === "dark" ? "#ffffff" : "#1A1A1A" + color: themeMode === "dark" ? "#ffffff" : "#1A1A1A" }, h2: { fontSize: 36, - color: resolvedMode === "dark" ? "#ffffff" : "#1A1A1A" + color: themeMode === "dark" ? "#ffffff" : "#1A1A1A" }, h3: { fontSize: 32, - color: resolvedMode === "dark" ? "#ffffff" : "#1A1A1A" + color: themeMode === "dark" ? "#ffffff" : "#1A1A1A" }, h4: { fontSize: 30, fontWeight: 500, - color: resolvedMode === "dark" ? "#ffffff" : "#1A1A1A" + color: themeMode === "dark" ? "#ffffff" : "#1A1A1A" }, h6: { fontSize: 22, - color: resolvedMode === "dark" ? "#ffffff" : "#1A1A1A" + color: themeMode === "dark" ? "#ffffff" : "#1A1A1A" }, body1: { fontSize: 16, - color: resolvedMode === "dark" ? "#ffffff" : "#1A1A1A" + color: themeMode === "dark" ? "#ffffff" : "#1A1A1A" }, body2: { fontSize: 14, - color: resolvedMode === "dark" ? "#ffffff" : "#1A1A1A" + color: themeMode === "dark" ? "#ffffff" : "#1A1A1A" }, }, components: { @@ -336,7 +327,7 @@ export const getTheme = (themeMode, brandColor) => { { props: { variant: 'text', color: 'primary' }, style: { - color: resolvedMode === "dark" ? "#ffffff" : "#1A1A1A", + color: themeMode === "dark" ? "#ffffff" : "#1A1A1A", whiteSpace: "nowrap", textWrap: "normal", }, @@ -344,7 +335,7 @@ export const getTheme = (themeMode, brandColor) => { { props: { variant: 'text', color: 'secondary' }, style: { - color: resolvedMode === "dark" ? "#9E9E9E" : "#616161", + color: themeMode === "dark" ? "#9E9E9E" : "#616161", whiteSpace: "nowrap", textWrap: "normal", }, @@ -352,24 +343,24 @@ export const getTheme = (themeMode, brandColor) => { { props: { variant: 'contained', color: 'primary' }, style: { - backgroundColor: resolvedMode === "dark" ? brandColor || '#ff8544' : brandColor || '#FF7C35', - color: resolvedMode === "dark" ? '#1a1a1a': '#FFFFFF', + backgroundColor: themeMode === "dark" ? brandColor || '#ff8544' : brandColor || '#FF7C35', + color: themeMode === "dark" ? '#1a1a1a': '#FFFFFF', borderRadius: '4px', whiteSpace: "nowrap", textWrap: "normal", transition: 'background-color 0.2s ease-in-out', '&:hover': { fontWeight: 600, - backgroundColor: resolvedMode === 'dark' ? brandColor || "#ff955c" : brandColor || '#FF8D4F', - color: resolvedMode === "dark" ? '#1a1a1a': '#FFFFFF', + backgroundColor: themeMode === 'dark' ? brandColor || "#ff955c" : brandColor || '#FF8D4F', + color: themeMode === "dark" ? '#1a1a1a': '#FFFFFF', }, }, }, { props: { variant: 'contained', color: 'secondary' }, style: { - backgroundColor: resolvedMode === "dark" ? '#494949' : '#C9C9C9', - color: resolvedMode === "dark" ? '#ffffff' : '#4C4C4C', + backgroundColor: themeMode === "dark" ? '#494949' : '#C9C9C9', + color: themeMode === "dark" ? '#ffffff' : '#4C4C4C', borderRadius: '4px', boxShadow: 'none', whiteSpace: "nowrap", @@ -377,23 +368,23 @@ export const getTheme = (themeMode, brandColor) => { textWrap: "normal", '&:hover': { fontWeight: 600, - border: resolvedMode === "dark" ? '1px solid #f1f1f1' : 'none', - backgroundColor: resolvedMode === "dark" ? '#494949' : '#C9C9C9', - color: resolvedMode === "dark" ? '#ffffff' : '#4C4C4C', + border: themeMode === "dark" ? '1px solid #f1f1f1' : 'none', + backgroundColor: themeMode === "dark" ? '#494949' : '#C9C9C9', + color: themeMode === "dark" ? '#ffffff' : '#4C4C4C', }, }, }, { props: { variant: 'outlined', color: 'primary' }, style: { - borderColor: resolvedMode === "dark" ? brandColor || "#ff8544" : brandColor || "#cc5f1f", - color: resolvedMode === "dark" ? brandColor || "#ff8544" : brandColor || "#cc5f1f", + borderColor: themeMode === "dark" ? brandColor || "#ff8544" : brandColor || "#cc5f1f", + color: themeMode === "dark" ? brandColor || "#ff8544" : brandColor || "#cc5f1f", whiteSpace: "nowrap", fontWeight: 'normal', textWrap: "normal", '&:hover': { - backgroundColor: resolvedMode === "dark" ? brandColor || "#ff8544" : "#ffe8dc", - color: resolvedMode === "dark" ? "#1a1a1a" : "#8a3d00", + backgroundColor: themeMode === "dark" ? brandColor || "#ff8544" : "#ffe8dc", + color: themeMode === "dark" ? "#1a1a1a" : "#8a3d00", fontWeight: 600, }, }, @@ -402,14 +393,14 @@ export const getTheme = (themeMode, brandColor) => { props: { variant: 'outlined', color: 'secondary' }, style: { border: '1px solid #C5C5C5', - color: resolvedMode === "dark" ? '#C5C5C5' : '#2D2D2D', + color: themeMode === "dark" ? '#C5C5C5' : '#2D2D2D', whiteSpace: "nowrap", textWrap: "normal", '&:hover': { - backgroundColor: resolvedMode === "dark" ? '#C5C5C5' : '#EFEFEF', - borderColor: resolvedMode === "dark" ? '#C5C5C5' : '#2D2D2D', + backgroundColor: themeMode === "dark" ? '#C5C5C5' : '#EFEFEF', + borderColor: themeMode === "dark" ? '#C5C5C5' : '#2D2D2D', fontWeight: 600, - color: resolvedMode === "dark" ? '#1a1a1a' : '#1A1A1A', + color: themeMode === "dark" ? '#1a1a1a' : '#1A1A1A', }, }, }, @@ -432,8 +423,8 @@ export const getTheme = (themeMode, brandColor) => { background: 'linear-gradient(90deg, #e6743a 0%, #d4456e 50%, #8a4de8 100%)', }, '&:disabled': { - background: resolvedMode === "dark" ? '#494949' : '#C9C9C9', - color: resolvedMode === "dark" ? '#9E9E9E' : '#616161', + background: themeMode === "dark" ? '#494949' : '#C9C9C9', + color: themeMode === "dark" ? '#9E9E9E' : '#616161', }, }, }, @@ -478,9 +469,9 @@ export const getTheme = (themeMode, brandColor) => { }, '&:disabled': { background: 'transparent', - color: resolvedMode === "dark" ? '#9E9E9E' : '#616161', + color: themeMode === "dark" ? '#9E9E9E' : '#616161', '&::before': { - background: resolvedMode === "dark" ? '#494949' : '#C9C9C9', + background: themeMode === "dark" ? '#494949' : '#C9C9C9', }, }, }, @@ -490,7 +481,7 @@ export const getTheme = (themeMode, brandColor) => { MuiTab: { styleOverrides: { root: { - color: resolvedMode === "dark" ? "#C5C5C5" : "#1A1A1A", + color: themeMode === "dark" ? "#C5C5C5" : "#1A1A1A", }, }, }, @@ -499,7 +490,7 @@ export const getTheme = (themeMode, brandColor) => { overrides: { MuiMenu: { list: { - backgroundColor: resolvedMode === "dark" ? "#27292d" : "#ffffff", + backgroundColor: themeMode === "dark" ? "#27292d" : "#ffffff", }, }, MuiCssBaseline: { @@ -545,5 +536,4 @@ export const getTheme = (themeMode, brandColor) => { }, }, }); -} diff --git a/frontend/src/views/AgentUI.jsx b/frontend/src/views/AgentUI.jsx index 6b2fbbd9..7e3ccc3f 100644 --- a/frontend/src/views/AgentUI.jsx +++ b/frontend/src/views/AgentUI.jsx @@ -44,6 +44,7 @@ import { Add as AddIcon, Warning as WarningIcon, Pause as PauseIcon, + Chat as ChatIcon, } from '@mui/icons-material' import { @@ -71,6 +72,7 @@ const AgentUI = (props) => { const [newSelectedApp, setNewSelectedApp] = React.useState({}) const [appPickerAnchor, setAppPickerAnchor] = React.useState(null) const [chosenApps, setChosenApps] = useState([]) + const [planningEnabled, setPlanningEnabled] = useState([]) const activateApp = (appId) => { if (appId === undefined || appId === null || appId === "") { @@ -188,11 +190,11 @@ const AgentUI = (props) => { const agentWrapperStyle = { width: "100%", - maxHeight: "100vh", + minHeight: "100vh", margin: "auto", backgroundColor: theme.palette.backgroundColor, - paddingBottom: showAgentStarter ? 0 : 1500, + paddingBottom: showAgentStarter ? 0 : 50, } @@ -248,21 +250,25 @@ const AgentUI = (props) => { return } + // If no node_id provided, look for the AI Agent node if (node_id === undefined || node_id === null || node_id === "") { - // Look for AI agent - /* for (var key in execution_data.results) { const item = execution_data.results[key] - if (item?.action?.app_name !== "AI Agent") { - continue + if (item?.action?.app_name === "AI Agent") { + node_id = item?.action?.id + break } - - node_id = item?.action?.id - break } - */ if (node_id === undefined || node_id === null || node_id === "") { + // Fallback: if only one result, use it + if (execution_data?.results?.length === 1) { + setAgentActionResult(execution_data.results[0]) + const validatedData = validateJson(execution_data.results[0].result) + if (validatedData.valid) { + setData(validatedData.result) + } + } return } } @@ -270,6 +276,11 @@ const AgentUI = (props) => { var found = false for (var key in execution_data.results) { const item = execution_data.results[key] + + if (item?.action?.app_name !== "AI Agent") { + continue + } + if (item?.action?.id !== node_id) { continue } @@ -289,16 +300,6 @@ const AgentUI = (props) => { if (found === false) { toast.warn("Failed to find the relevant AI Agent result") - - if (execution_data?.results?.length === 1) { - setAgentActionResult(execution_data.results[0]) - const validatedData = validateJson(execution_data.results[0].result) - if (validatedData.valid) { - setData(validatedData.result) - } else { - toast.warn("Action output result is not valid JSON!") - } - } } } @@ -475,7 +476,7 @@ const AgentUI = (props) => { getAppAuth() }, []) - const maxTimelineWidth = 375 + const maxTimelineWidth = 275 const submitQuestions = (decisionId, questionAnswers, isContinuation) => { console.log("Submitting questions: ", decisionId, questionAnswers) @@ -521,8 +522,12 @@ const AgentUI = (props) => { const executionId = params.get("execution_id") const nodeId = params.get("node_id") const authorization = params.get("authorization") + const workflowIdParam = params.get("workflow_id") - const url = `${globalUrl}/api/v1/workflows/${executionId}/run?reference_execution=${executionId}&authorization=${authorization}&answer=true¬e=${encodeURIComponent(JSON.stringify(newArgument))}&agentic=true&decision_id=${decisionId}` + // workflow_id param > execution.workflow.id > executionId + const foundWorkflowId = workflowIdParam !== undefined && workflowIdParam !== null && workflowIdParam !== "" ? workflowIdParam : execution?.workflow?.id !== undefined && execution?.workflow?.id !== null && execution?.workflow?.id !== "" ? execution.workflow.id : executionId + + const url = `${globalUrl}/api/v1/workflows/${foundWorkflowId}/run?reference_execution=${executionId}&authorization=${authorization}&answer=true¬e=${encodeURIComponent(JSON.stringify(newArgument))}&agentic=true&decision_id=${decisionId}&node_id=${nodeId}` fetch(url, { method: "GET", credentials: "include", @@ -893,15 +898,32 @@ const AgentUI = (props) => { />
- {itemLabel} + + {itemLabel} +
{
: null} - {questions?.length > 0 && item?.status === "RUNNING" || item?.status === "WAITING" ? + {item.category !== "agent" && questions?.length > 0 && (item?.status === "RUNNING" || item?.status === "WAITING") ?
{questions.map((q, questionIndex) => { return ( @@ -1189,7 +1211,25 @@ const AgentUI = (props) => { const [continuationText, setContinuationText] = useState("") - var actionResult = execution?.results?.length > 0 ? execution.results[0] : execution + // Find the AI Agent result specifically, not just results[0] + var actionResult = null + if (execution?.results?.length > 0) { + for (var key in execution.results) { + const item = execution.results[key] + if (item?.action?.app_name === "AI Agent") { + actionResult = item + break + } + } + + // Fallback to first result if no AI Agent found + if (actionResult === null) { + actionResult = execution.results[0] + } + } else { + actionResult = execution + } + const validate = validateJson(actionResult?.result) if (validate.valid === true) { actionResult.result = validate.result @@ -1246,6 +1286,11 @@ const AgentUI = (props) => { for (var key in agent_data?.decisions) { const item = agent_data.decisions[key] + if (item.run_details === undefined) { + console.log("Skipping item without run_details:", item) + continue + } + if (item.run_details.started_at === undefined || item.run_details.started_at === null) { item.run_details.started_at = originalStartTime } @@ -1495,6 +1540,7 @@ const AgentUI = (props) => { parsedAction = parsedAction.slice(0, -1) // Remove last comma } + /* const data = { "id": uuid, "name":"agent", @@ -1517,9 +1563,23 @@ const AgentUI = (props) => { "name":"action", "value": parsedAction, } - ]} - + ], + "planning_mode": planningEnabled, + } const url = `${globalUrl}/api/v1/apps/agent_starter/run` + */ + + const data = { + "jsonrpc": "2.0", + "method": "tools/call", + "params": { + "tool_name": parsedAction, + "input": { + "text": inputText, + }, + } + } + const url = `${globalUrl}/api/v1/agent` fetch(url, { method: "POST", body: JSON.stringify(data), @@ -1604,8 +1664,8 @@ const AgentUI = (props) => { return ( -
-
+
+
{ agentRequestLoading ? : - - + + @@ -1678,12 +1742,29 @@ const AgentUI = (props) => { }} /> -
+
+ {/* + + + } label="Planning Mode" + style={chipStyle} + variant={planningEnabled ? "contained" : "outlined"} + onClick={() => { + setPlanningEnabled(!planningEnabled) + }} + disabled={true} + /> + + + */} } label="Select Apps / MCPs" style={chipStyle} + variant={"outlined"} onClick={() => { setAppPickerAnchor(document.getElementById("add_app_chip")) }} @@ -1794,18 +1875,21 @@ const AgentUI = (props) => { {chosenApps?.map((app, index) => { return( - + { - window.open(`/apps/${app.id}`, '_blank', 'noopener,noreferrer'); + if (app?.id !== undefined) { + window.open(`/apps/${app.id}`, '_blank', 'noopener,noreferrer'); + } }} style={{ cursor: "pointer", width: 30, height: 30, + backgroundColor: theme.palette.backgroundColor, }} /> diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index db69e886..f995697f 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -231,7 +231,7 @@ export const triggers = [ status: "uninitialized", trigger_type: "SCHEDULE", errors: null, - large_image: "data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iNDgiIGhlaWdodD0iNDgiIHZpZXdCb3g9IjAgMCA0OCA0OCIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPHJlY3Qgd2lkdGg9IjQ4IiBoZWlnaHQ9IjQ4IiByeD0iOCIgZmlsbD0iIzIxQTBCRCIvPgo8Y2lyY2xlIGN4PSIyNCIgY3k9IjI0IiByPSI4Ljc1IiBzdHJva2U9IndoaXRlIiBzdHJva2Utd2lkdGg9IjEuNSIvPgo8cGF0aCBkPSJNMjguNSAyNEgyNC4yNUMyNC4xMTE5IDI0IDI0IDIzLjg4ODEgMjQgMjMuNzVWMjAuNSIgc3Ryb2tlPSJ3aGl0ZSIgc3Ryb2tlLXdpZHRoPSIxLjUiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIvPgo8L3N2Zz4K", + large_image: "data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iNDgiIGhlaWdodD0iNDgiIHZpZXdCb3g9IjAgMCA0OCA0OCIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPHJlY3Qgd2lkdGg9IjQ4IiBoZWlnaHQ9IjQ4IiByeD0iOCIgZmlsbD0iI0UzQTQxQiIvPgo8cmVjdCB3aWR0aD0iMjQiIGhlaWdodD0iMjQiIHRyYW5zZm9ybT0idHJhbnNsYXRlKDEyIDEyKSIgZmlsbD0iI0UzQTQxQiIvPgo8Y2lyY2xlIGN4PSIyNCIgY3k9IjI0IiByPSI4Ljc1IiBzdHJva2U9IndoaXRlIiBzdHJva2Utd2lkdGg9IjEuNSIvPgo8cGF0aCBkPSJNMjguNSAyNEgyNC4yNUMyNC4xMTE5IDI0IDI0IDIzLjg4ODEgMjQgMjMuNzVWMjAuNSIgc3Ryb2tlPSJ3aGl0ZSIgc3Ryb2tlLXdpZHRoPSIxLjUiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIvPgo8L3N2Zz4=", label: "Schedule", is_valid: true, environment: "onprem", @@ -300,6 +300,11 @@ export const triggers = [ "name": "subflow", "example": "", "value": "", + }, + { + "name": "subflow_failure", + "example": "", + "value": "", } ], status: "running", @@ -498,6 +503,31 @@ export function setActionState(actionId, updates, workflowId = null) { } } +// Will use this function to remove the action data when the node will get removed from the cytoscape. +export function removeActionState(actionId, workflowId = null) { + if (!actionId) return; + + try { + const stored = localStorage.getItem(ACTION_STATES_STORAGE_KEY); + if (!stored) return; + + const allStates = JSON.parse(stored); + + if (workflowId && allStates[workflowId]) { + delete allStates[workflowId][actionId]; + + // Clean up empty workflow objects + if (Object.keys(allStates[workflowId]).length === 0) { + delete allStates[workflowId]; + } + } + + localStorage.setItem(ACTION_STATES_STORAGE_KEY, JSON.stringify(allStates)); + } catch (e) { + console.error("Failed to remove action state:", e); + } +} + const splitter = "|~|"; const svgSize = 24; const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); @@ -505,7 +535,7 @@ const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); //const referenceUrl = "https://shuffler.io/functions/webhooks/" //const referenceUrl = window.location.origin+"/api/v1/hooks/" -const searchClient = algoliasearch("JNSS5CFDZZ", "c8f882473ff42d41158430be09ec2b4e") +const searchClient = algoliasearch("JNSS5CFDZZ", "33e4e3564f4f060e96e0531957bed552") const AngularWorkflow = (defaultprops) => { const { globalUrl, setCookie, isLoggedIn, isLoaded, userdata, data_id, ReactGA, } = defaultprops; const {themeMode, supportEmail, brandColor} = useContext(Context) @@ -566,6 +596,8 @@ const AngularWorkflow = (defaultprops) => { const [originalWorkflow, setOriginalWorkflow] = React.useState({}); const [originalSelectedEnvironment, setOriginalSelectedEnvironment] = React.useState({}); const [subworkflow, setSubworkflow] = React.useState({}); + const [subworkflowFailure, setSubworkflowFailure] = React.useState({}); + const [subworkflowFailureStartnode, setSubworkflowFailureStartnode] = React.useState(""); const [subworkflowStartnode, setSubworkflowStartnode] = React.useState(""); const [leftViewOpen, setLeftViewOpen] = React.useState(isMobile ? false : true); const [leftBarSize, setLeftBarSize] = React.useState(isMobile ? 0 : 235) @@ -1130,7 +1162,14 @@ const AngularWorkflow = (defaultprops) => { "description": "Translates your JSON data into a standard formats, then stores it in the Shuffle Datastore", "label": "Translate standard", "example": "{\"source_data\": \"{\\\"event\\\": \\\"login\\\", \\\"user\\\": \\\"john_doe\\\", \\\"timestamp\\\": \\\"2023-10-01T12:00:00Z\\\"}\", \"standard\": \"OCSF\"}", - "parameters": [{ + "parameters": [ + { + "name": "app_name", + "value": "", + "required": true, + "multiline": false, + }, + { "name": "source_data", "value": "", "required": true, @@ -1150,7 +1189,14 @@ const AngularWorkflow = (defaultprops) => { "name": "Cases", "description": "Available actions for case management", "label": "Cases", - "parameters": [{ + "parameters": [ + { + "name": "app_name", + "value": "", + "required": true, + "multiline": false, + }, + { "name": "action", "value": "list_tickets", "options": [ @@ -1175,7 +1221,14 @@ const AngularWorkflow = (defaultprops) => { "name": "Communication", "description": "Available actions for communication", "label": "Communication", - "parameters": [{ + "parameters": [ + { + "name": "app_name", + "value": "", + "required": true, + "multiline": false, + }, + { "name": "action", "value": "list_messages", "options": [ @@ -1210,7 +1263,8 @@ const AngularWorkflow = (defaultprops) => { "disable_user", "get_identity", "get_asset", - "search_identity" + "search_identity", + "list_users", ], "required": true, }, @@ -2199,6 +2253,26 @@ const AngularWorkflow = (defaultprops) => { } } + if (param.name === "subflow_failure" && param.value !== undefined && param.value !== null && param.value.length > 0) { + if (param.value === workflow?.id) { + setSubworkflowFailure(workflow); + } else { + const sub = responseJson.find((data) => data?.id === param.value); + if (sub !== undefined) { + setSubworkflowFailure(sub); + + // Populate startnode if set + const startnodeParam = trigger.parameters.find((p) => p.name === "subflow_failure_startnode"); + if (startnodeParam && startnodeParam.value && sub.actions) { + const foundAction = sub.actions.find((a) => a?.id === startnodeParam.value); + if (foundAction) { + setSubworkflowFailureStartnode(foundAction); + } + } + } + } + } + if (param.name === "startnode" && param.value !== undefined && param.value !== null) { if (Object.getOwnPropertyNames(baseSubflow).length > 0) { @@ -2344,7 +2418,7 @@ const AngularWorkflow = (defaultprops) => { return } - setExecutionsLoading(true); + setExecutionsLoading(true); var url = `${globalUrl}/api/v2/workflows/${id}/executions` var method = "GET" @@ -2825,6 +2899,7 @@ const AngularWorkflow = (defaultprops) => { stop() return } + //console.log(responseJson) // Loop nodes and find results // Update on every interval? idk @@ -3971,6 +4046,8 @@ const AngularWorkflow = (defaultprops) => { if (actionAppname === appname) { workflow.actions[actionkey].selectedAuthentication = item; workflow.actions[actionkey].authentication_id = item.id; + selectedAction.selectedAuthentication = item; + selectedAction.authentication_id = item.id; appUpdates = true; } } @@ -5195,7 +5272,8 @@ const AngularWorkflow = (defaultprops) => { if (responseJson.public) { - setAppAuthentication([]) + // Delay setting appAuthentication to prevent race condition with graph setup + setTimeout(() => setAppAuthentication([]), 100) setLeftBarSize(300) if (Object.getOwnPropertyNames(creatorProfile).length === 0) { @@ -5586,7 +5664,7 @@ const AngularWorkflow = (defaultprops) => { } ReactDOM.unstable_batchedUpdates(() => { - setRightSideBarOpen(true); + // setRightSideBarOpen(true); setLastSaved(false); /* @@ -6772,7 +6850,7 @@ const AngularWorkflow = (defaultprops) => { } //event.target.unselect(); - setRightSideBarOpen(true); + // setRightSideBarOpen(true); return } else if (data.buttonType === "copy") { @@ -6864,7 +6942,7 @@ const AngularWorkflow = (defaultprops) => { if (sourcenode !== null && sourcenode !== undefined) { const sourcedata = sourcenode.data() - if (sourcedata.trigger_type !== "SUBFLOW" && sourcedata.trigger_type !== "USERINPUT") { + if (sourcedata?.trigger_type !== "SUBFLOW" && sourcedata?.trigger_type !== "USERINPUT") { continue } @@ -7129,12 +7207,12 @@ const AngularWorkflow = (defaultprops) => { const tmpAuth = JSON.parse(JSON.stringify(newAppAuth)); - const curappName = curapp.name.toLowerCase() + const curappName = curapp.name.toLowerCase().replaceAll(" ", "_") for (let tmpAuthKey in tmpAuth) { var item = tmpAuth[tmpAuthKey]; const newfields = {}; - if (item.app.name.toLowerCase() !== curappName) { + if (item.app.name.toLowerCase().replaceAll(" ", "_") !== curappName) { continue } @@ -7516,7 +7594,6 @@ const AngularWorkflow = (defaultprops) => { setSelectedTriggerIndex(trigger_index) setSelectedTrigger(data) - //setSelectedActionEnvironment(data.env) }, 25) } else if (data.type === "COMMENT") { if (selectedNodes?.length > 1) { @@ -7829,7 +7906,7 @@ const AngularWorkflow = (defaultprops) => { continue } - const paramname = param.name.toLowerCase().trim().replaceAll("_", " "); + const paramname = param.name?.toLowerCase()?.trim()?.replaceAll("_", " "); const foundresult = GetParamMatch(paramname, exampledata, ""); if (foundresult.length > 0) { @@ -7866,10 +7943,7 @@ const AngularWorkflow = (defaultprops) => { continue } - const paramname = param.name - .toLowerCase() - .trim() - .replaceAll("_", " "); + const paramname = param.name?.toLowerCase()?.trim()?.replaceAll("_", " "); const foundresult = GetParamMatch(paramname, exampledata, ""); if (foundresult.length > 0) { @@ -8169,11 +8243,11 @@ const AngularWorkflow = (defaultprops) => { if (workflow.branches[branchkey].destination_id === edge.target && workflow.branches[branchkey].source_id === edge.source) { console.log("That branch already exists: ", workflow.branches[branchkey]) - const foundbranch = cy.getElementById(workflow.branches[branchkey].id) + //const foundbranch = cy.getElementById(workflow.branches[branchkey].id) + const foundbranch = cy.getElementById(edge.id) if (foundbranch !== undefined && foundbranch !== null && foundbranch.data() !== undefined && foundbranch.data() !== null) { console.log("Removing branch: ", foundbranch.data()) - - event.target.remove() + //event.target.remove() found = true break @@ -8545,6 +8619,10 @@ const AngularWorkflow = (defaultprops) => { workflow.actions = workflow.actions.filter((a) => a.id !== data.id); workflow.triggers = workflow.triggers.filter((a) => a.id !== data.id); + + // Clean up action state from localStorage + removeActionState(data.id, workflow.id); + if (workflow.start === data.id && workflow.actions.length > 0) { // FIXME - should check branches connected to startnode, as picking random // is just confusing @@ -8654,7 +8732,7 @@ const AngularWorkflow = (defaultprops) => { if ((event.ctrlKey || event.metaKey) && !event.shiftKey) { // If any modal/sidebar is open, let browser handle normal copy - if (isAnyModalOrSidebarOpen) { + if (isAnyModalOrSidebarOpen || event.target?.closest('.MuiDialog-root, .MuiModal-root, [role="dialog"]')) { return } @@ -10105,8 +10183,8 @@ const AngularWorkflow = (defaultprops) => { // Calculates how a branch should curve (it's still weird~) // https://codepen.io/guillaumethomas/pen/xxbbBKO const calculateEdgeCurve = (sourcenodePosition, destinationnodePosition) => { - const xParsed = destinationnodePosition.x - sourcenodePosition.x - const yParsed = destinationnodePosition.y - sourcenodePosition.y + const xParsed = destinationnodePosition?.x - sourcenodePosition?.x + const yParsed = destinationnodePosition?.y - sourcenodePosition?.y const z = Math.sqrt(xParsed * xParsed + yParsed * yParsed) const costheta = xParsed / z @@ -10273,7 +10351,7 @@ const AngularWorkflow = (defaultprops) => { action.iconBackground = iconInfo.iconBackgroundColor action.fillstyle = "linear-gradient" } - }else if(!action.isStartNode) { + } else if(!action.isStartNode) { // This is to round the corners of the image // If action has no large_image (e.g. imported/synced workflow where it was stripped), // inject it from the available apps in the sidebar @@ -10283,6 +10361,7 @@ const AngularWorkflow = (defaultprops) => { apps.find((a) => a.name === action.app_name) imageSource = (foundApp && foundApp.large_image) ? foundApp.large_image : "" } + const originalBase64 = imageSource !== "" ? imageSource : theme.palette.defaultImage const roundedImage = await roundBase64Image(originalBase64, 16); action = {...action, large_image: roundedImage} @@ -11424,10 +11503,10 @@ const AngularWorkflow = (defaultprops) => { // No matter what, it's being stopped. if (!responseJson.success) { if (responseJson.reason !== undefined) { - toast("Failed to stop schedule: " + responseJson.reason); + toast.warn("Failed to stop schedule: " + responseJson.reason); } } else { - toast("Successfully stopped schedule"); + toast.success("Successfully stopped schedule"); } if (triggerindex !== undefined && triggerindex !== null && triggerindex >= 0) { @@ -13161,7 +13240,7 @@ const AngularWorkflow = (defaultprops) => { if (queryID !== undefined && queryID !== null) { aa('init', { appId: "JNSS5CFDZZ", - apiKey: "c8f882473ff42d41158430be09ec2b4e", + apiKey: "33e4e3564f4f060e96e0531957bed552", }) const timestamp = new Date().getTime() @@ -16032,7 +16111,7 @@ const AngularWorkflow = (defaultprops) => { onClick={() => { // Change Direction of the branch target/source const foundBranch = cy.getElementById(selectedEdge.id) - if (foundBranch !== undefined && foundBranch !== null) { + if (foundBranch !== undefined && foundBranch !== null && foundBranch?.length > 0) { const source = foundBranch.data("source") const target = foundBranch.data("target") @@ -19292,6 +19371,236 @@ const AngularWorkflow = (defaultprops) => { /> ) : null} + {workflow?.triggers && + workflow?.triggers[selectedTriggerIndex] && + workflow?.triggers[selectedTriggerIndex].parameters + ? ( +
+ On Decline + + Optionally trigger a workflow when the user declines + + {workflows === undefined || + workflows === null || + workflows.length === 0 ? null : ( + option.id === value.id} + getOptionLabel={(option) => { + if (option === undefined || option === null || option.name === undefined || option.name === null) { + return "No Workflow Selected"; + } + const newname = (option.name.charAt(0).toUpperCase() + option.name.substring(1)).replaceAll("_", " "); + return newname; + }} + options={ + [{ + "id": "", + "name": "No Workflow Selected", + }].concat(workflows) + } + fullWidth + onChange={(event, newValue) => { + if (newValue === null || newValue === undefined || newValue.id === undefined) { + return + } + + var failureParamIndex = workflow.triggers[selectedTriggerIndex].parameters.findIndex((param) => param.name === "subflow_failure") + if (failureParamIndex === -1) { + workflow.triggers[selectedTriggerIndex].parameters.push({ + "name": "subflow_failure", + "value": "", + }) + failureParamIndex = workflow.triggers[selectedTriggerIndex].parameters.length - 1 + } + + workflow.triggers[selectedTriggerIndex].parameters[failureParamIndex].value = newValue.id + setSubworkflowFailureStartnode("") + + // Fetch workflow to get actions for startnode selection + if (newValue.id.length > 0 && (newValue.actions === undefined || newValue.actions === null || newValue.actions.length === 0)) { + fetch(`${globalUrl}/api/v1/workflows/${newValue.id}`, { + method: "GET", + headers: { "Content-Type": "application/json" }, + credentials: "include", + }) + .then((resp) => resp.json()) + .then((responseJson) => { + if (responseJson.id !== undefined) { + setSubworkflowFailure(responseJson) + + // Default startnode + const startAction = responseJson.actions?.find((a) => a.id === responseJson.start) + if (startAction) { + setSubworkflowFailureStartnode(startAction) + } + } + }) + .catch((error) => { + console.log("Failed fetching decline workflow: ", error) + }) + } else { + setSubworkflowFailure(newValue) + const startAction = newValue.actions?.find((a) => a.id === newValue.start) + if (startAction) { + setSubworkflowFailureStartnode(startAction) + } + } + + setWorkflow(workflow) + setUpdate(Math.random()) + setLastSaved(false) + event.target.blur() + }} + renderOption={(props, data, state) => { + return ( + + + {data.name} + + ) + }} + renderInput={(params) => { + return ( +
+ + {subworkflowFailure === null || subworkflowFailure === undefined || subworkflowFailure?.id === undefined || subworkflowFailure?.id === null || subworkflowFailure?.id.length === 0 ? null : + + + + + + } +
+ ); + }} + /> + )} + + {subworkflowFailure?.actions !== undefined && subworkflowFailure?.actions !== null && subworkflowFailure?.actions?.length > 0 ? ( + option.id === value.id} + getOptionLabel={(option) => { + if (option === undefined || option === null || option.label === undefined || option.label === null) { + return "Default"; + } + const newname = (option.label.charAt(0).toUpperCase() + option.label.substring(1)).replaceAll("_", " "); + return newname; + }} + options={subworkflowFailure.actions} + fullWidth + onChange={(event, newValue) => { + setSubworkflowFailureStartnode(newValue) + + var startnodeParamIndex = workflow.triggers[selectedTriggerIndex].parameters.findIndex((param) => param.name === "subflow_failure_startnode") + if (startnodeParamIndex === -1) { + workflow.triggers[selectedTriggerIndex].parameters.push({ + "name": "subflow_failure_startnode", + "value": "", + }) + startnodeParamIndex = workflow.triggers[selectedTriggerIndex].parameters.length - 1 + } + + workflow.triggers[selectedTriggerIndex].parameters[startnodeParamIndex].value = newValue?.id || "" + setWorkflow(workflow) + setUpdate(Math.random()) + setLastSaved(false) + }} + renderOption={(props, action, state) => { + return ( + + {action.label} + + ) + }} + renderInput={(params) => { + return ( + + ); + }} + /> + ) : null} +
+ ) : null} +
Required Input-Questions @@ -19978,6 +20287,7 @@ const AngularWorkflow = (defaultprops) => { "&.Mui-selected": { backgroundColor: themeMode === "dark" ? "#1e1e1e" : "#CCCCCC", color: theme.palette.text.primary, + borderRadius: "6px !important", fontWeight: 600, "&:hover": { backgroundColor: themeMode === "dark" ? "rgba(0,0,0,0.3)" : "rgba(0,0,0,0.1)", @@ -20020,7 +20330,6 @@ const AngularWorkflow = (defaultprops) => { justifyContent: "space-between", width: "100%", position: "relative", - minHeight: 80, }}> {/* Left: Workflow Name Container */}
{ }} > {workflow?.name !== undefined && workflow?.name !== null && workflow?.name?.length > 0 ? - + : null } {workflow.name} + {/* Warning Messages */} + {!distributedFromParent || userdata?.support === true ? + isCorrectOrg ? null : + + Warning: { + toast.info("Changing to correct organisation. Please wait a few seconds.") + changeOrg() + }} + >Change Active Organization to edit this Workflow. + + : + + suborgWorkflows?.length === 0 ? + + Warning: This workflow is controlled by your parent org and may not be editable. + + : + null + } + {parentWorkflows === undefined || parentWorkflows === null || parentWorkflows.length === 0 ? null : + + }
{/* Center: Build/Debug Toggle */} @@ -20237,6 +20602,7 @@ const AngularWorkflow = (defaultprops) => { saveWorkflow(workflow, undefined, undefined, e.target.value) /* Standard re-loads */ + setAllTriggers(undefined) setSelectedTriggerIndex(-1) getEnvironments(e.target.value) @@ -20561,7 +20927,7 @@ const AngularWorkflow = (defaultprops) => { id="execution_location" style={{ color: theme.palette.text.primary }} > - Runtime Location + Runtime Location ({selectedActionEnvironment?.Name}) setSelectedRegion(e.target.value)} - style={{ color: '#fff', backgroundColor: '#1a1a1a', height: 32, fontSize: 13, minWidth: 140 }} - sx={{ '& .MuiOutlinedInput-notchedOutline': { borderColor: '#333' } }} - > - London - California - EU (Frankfurt) - Canada - Australia - - - )} - {!isCloud && ( - - )} -
- -
- {backendVersion && Backend v{backendVersion}} - {lastUpdated && Last check: {lastUpdated}} - {isCloud ? 'Cloud' : 'On-Premises'} -
- - {/* === Legend (Moved to top) === */} -
- {[{ color: '#00F670', label: '≥ SLO Healthy' }, { color: '#FFD700', label: '95–SLO Degraded' }, { color: '#FF354C', label: '< 95% Outage' }].map(({ color, label }) => ( -
-
- {label} -
- ))} - Click any bar to view failure details · White marker = SLO target -
-
- -
-
- - {[['24hr', '24h'], ['7day', '7d'], ['30d', '30d'], ['90d', '90d'], ['180d', '180d'], ['365d', '365d']].map(([key, label]) => { - const isDisabled = isHealthLoading || ['90d', '180d', '365d'].includes(key); - const isActive = !customRange && selectedRange === key; - return ( - - ); - })} - - - {/* Calendar range picker button */} - - - -
- - {/* Show active custom range label */} - {customRange && ( -
- - {customRange.startLabel} – {customRange.endLabel} - - -
- )} -
- - {/* Calendar popover */} - setCalendarAnchor(null)} - anchorOrigin={{ vertical: 'bottom', horizontal: 'right' }} - transformOrigin={{ vertical: 'top', horizontal: 'right' }} - PaperProps={{ style: { backgroundColor: '#111', border: '1px solid #252525', borderRadius: 12, minWidth: 310, overflow: 'hidden', boxShadow: '0 12px 40px rgba(0,0,0,0.7)', display: 'flex', flexDirection: 'column' } }} +
+ {/* Health Bar Chart Section */} +
+ + + + + + + + - -
- + {isFixingOpensearchPrefix ? 'Fixing Opensearch Prefix...' : 'Fix Opensearch Prefix'} +
- {/* Loading */} - {isHealthLoading && } + {/* Loading Bar for HealthBarChart */} + {isHealthLoading && ( + + )} - {/* === System status banner === */} - {!isHealthLoading && filteredHealthData.length > 0 && ( -
- {systemStatus === 'operational' - ? - : systemStatus === 'degraded' - ? - : } +
+
+
+ +
+ Workflow Health + Operational +
+
- - {systemStatus === 'operational' ? 'All Systems Operational' : systemStatus === 'degraded' ? 'Degraded Performance Detected' : 'Some Services Affected'} - - - {filteredHealthData.length} health check{filteredHealthData.length !== 1 ? 's' : ''} · {activRangeLabel} - + {averageUptime.toFixed(2)}% + Success Rate
+
+ +
- {/* SLO summary dots */} -
- {activeServices.map(cfg => { - const uptime = computeAvgUptime(serviceCharts[cfg.key]); - const dotColor = uptime >= cfg.sloTarget ? '#00F670' : uptime >= 95 ? '#FFD700' : '#FF354C'; - return ( - -
-
- {cfg.label} - {uptime.toFixed(2)}% -
- - ); - })} + {userdata.support_access && ( +
+
+ Live Executions + + + + + + {/* */} +
+ {/* Loading Bar for LiveExecutionsChart */} + {isLiveExecutionsLoading && ( + + )} +
)} - {/* === Service health rows (vertical) === */} -
- {activeServices.map(cfg => renderServiceCard(cfg))} -
- {/* === Failure details panel === */} - {renderFailureDetails()} - - {/* Legend removed and moved to the top */} - - {/* === Live Executions === */} - {userdata.support_access && ( - <> -
-
-
- Live Executions - - {[['1h', '1h'], ['7h', '7h'], ['1d', '1d'], ['7d', '7d']].map(([key, label]) => ( - - ))} - -
- {isLiveExecutionsLoading && } - -
- - )} -
); }; -export default HealthPage; \ No newline at end of file +export default HealthPage; diff --git a/frontend/src/components/LeftSideBar.jsx b/frontend/src/components/LeftSideBar.jsx index 6050f092..056867ae 100644 --- a/frontend/src/components/LeftSideBar.jsx +++ b/frontend/src/components/LeftSideBar.jsx @@ -52,12 +52,6 @@ const ShuffleLogo = "/images/Shuffle_logo.png"; const detectionIcon = "/icons/detection.svg"; const documentationIcon = "/icons/documentation.svg"; const ExpandMoreAndLessIcon = "/icons/expandMoreIcon.svg"; -const shuffleSecurityLogo = ( - - - -); - const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, SHUFFLE_VERSION }) => { @@ -93,17 +87,6 @@ const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, SHUFFLE_V ); const [activeOrgData, setActiveOrgData] = useState(null); const [isProdStatusOn, setIsProdStatusOn] = useState(false); - const [productAnchorEl, setProductAnchorEl] = useState(null); - - const handleProductClick = (event) => { - event.preventDefault(); - setProductAnchorEl((prev) => (prev ? null : event.currentTarget)); - }; - - const handleProductClose = () => { - setProductAnchorEl(null); - }; - const userOrgs = React.useMemo(() => { return orgOptions.find((option) => option.name === selectedOrg); }, [selectedOrg, orgOptions]); @@ -137,7 +120,7 @@ const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, SHUFFLE_V setCurrentSelectedTheme(userdata?.theme); } }, [userdata]); - + const CustomPopper = (props) => { @@ -856,8 +839,6 @@ const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, SHUFFLE_V regiontag = "EU-2"; } else if (regiontag === "ca"){ regiontag = "CA"; - } else if (regiontag === "uk"){ - regiontag = "UK"; } } } @@ -1041,7 +1022,7 @@ const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, SHUFFLE_V }} > - - { - !showPartnerLogo && e.preventDefault(); - }} - > - Shuffle Logo - {!showPartnerLogo && expandLeftNav && ( - - )} - - - - - - { - if (isCloud) { - //ReactGA.event({ - // category: "sidebar", - // action: "click_shuffle_security", - // label: "", - //}) - - window.location.href = "https://security.shuffler.io/incidents?utm_source=shuffler_sidebar"; - } else { - const { protocol, hostname } = window.location; - - var newPort = 3002; - if (protocol === "https") { - newPort = 3444 - } - - const newUrl = `${protocol}//${hostname}:${newPort}/incidents`; - window.location.href = newUrl; - } - }} - sx={{ - borderRadius: "8px", - padding: "10px 12px", - border: "1px solid transparent", - "&:hover": { - backgroundColor: themeMode === "dark" ? "#2C2C2C" : "#F5F5F5", - }, - display: "flex", - gap: "12px", - alignItems: "center", - }} - > - - {shuffleSecurityLogo} - - - Shuffle{" "} - Security - - - { - handleProductClose(); - window.location.href = - isCloud && !showPartnerLogo ? "/" : "/workflows"; - }} - sx={{ - borderRadius: "8px", - padding: "10px 12px", - border: - themeMode === "dark" - ? "1px solid rgba(242, 100, 2, 0.3)" - : "1px solid rgba(242, 100, 2, 0.2)", - backgroundColor: - themeMode === "dark" - ? "rgba(242, 100, 2, 0.05)" - : "rgba(242, 100, 2, 0.02)", - "&:hover": { - backgroundColor: - themeMode === "dark" - ? "rgba(242, 100, 2, 0.1)" - : "rgba(242, 100, 2, 0.06)", - }, - display: "flex", - gap: "12px", - alignItems: "center", - }} - > - - Shuffle - - - Shuffle{" "} - Core - - - + + Shuffle Logo + + { !isCloud && expandLeftNav && ( @@ -1944,8 +1747,7 @@ const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, SHUFFLE_V
) ) -}); +}); \ No newline at end of file diff --git a/frontend/src/components/LicencePopup.jsx b/frontend/src/components/LicencePopup.jsx index b113716c..54611352 100644 --- a/frontend/src/components/LicencePopup.jsx +++ b/frontend/src/components/LicencePopup.jsx @@ -330,7 +330,7 @@ const LicencePopup = (props) => { body: JSON.stringify({ org_id: selectedOrganization.id, editing: "subscription_update", - subscription_index: subscription.id, + subscription_index: 0, subscription: subscription, }), mode: "cors", @@ -554,7 +554,7 @@ const LicencePopup = (props) => { const payload = { org_id: selectedOrganization.id, editing: "subscription_update", - subscription_index: subscription.id, + subscription_index: 0, subscription: { ...form, // Ensure backend gets array of features @@ -696,15 +696,6 @@ const LicencePopup = (props) => { helperText={errors.amount || "0 for Free"} /> - setForm({ ...form, reference: e.target.value })} - fullWidth - placeholder="sub_1234567890abcdef" - helperText="Stripe subscription reference ID" - /> - { : localSub?.currency + localSub?.amount : "Free"; - const calculateAppRunsFromPrice = (amount) => { - const price = parseInt(amount) || 0; - if (price === 0) return 2000; // Free plan - - // Calculate Stripe quantity from price ($32 per unit) - const stripeQuantity = Math.max(1, Math.round(price / 32)); - - // Backend logic: (quantity * 10000) + 2000 - return (stripeQuantity * 10000) + 2000; - }; - if (typeof window === "undefined" || window.location === undefined) { return null; } @@ -1272,7 +1252,7 @@ const LicencePopup = (props) => { )} - {(isPaidPlan || localSub?.amount === "0") ? ( + {isPaidPlan ? (
{ ) : null} {localSub.cancellationdate !== 0 ? ( - - {`Cancelled on ${new Date( - (localSub.cancellationdate || localSub.CancellationDate) * - 1000 - ).toLocaleDateString(undefined, { - day: "2-digit", - month: "short", - year: "numeric", - })}`} - + + {`Cancelled on ${new Date( + (localSub.cancellationdate || localSub.CancellationDate) * + 1000 + ).toLocaleDateString(undefined, { + day: "2-digit", + month: "short", + year: "numeric", + })}`} + ) : null} - {localSub.amount !== "0" && ( - - {`Purchased on ${new Date( - (localSub.startdate || localSub.Startdate) * 1000 - ).toLocaleDateString(undefined, { - day: "2-digit", - month: "short", - year: "numeric", - })}`} - - )} - { (isCloud || (!isCloud && selectedOrganization.cloud_sync)) && (
- {localSub?.active ? "App Runs" : "App Runs included in plan"} + App Runs - {localSub?.active ? ( - // Active plan - show current usage and progress bar -
- - {usedAppRuns?.toLocaleString?.() || usedAppRuns} of{" "} - {appRunsLimit?.toLocaleString?.() || appRunsLimit} - - - - -
- ) : ( - // Inactive plan - show only the plan's app runs capacity +
- {calculateAppRunsFromPrice(localSub.amount)?.toLocaleString?.() || calculateAppRunsFromPrice(localSub.amount)} + {usedAppRuns?.toLocaleString?.() || usedAppRuns} of{" "} + {appRunsLimit?.toLocaleString?.() || appRunsLimit} - )} + + + +
) } @@ -1527,6 +1480,7 @@ const LicencePopup = (props) => { }} > {isCloud && + localSub.name.toLowerCase().includes("scale") && localSub?.reference && localSub.reference.length > 0 ? (
- + {activeMainTab === "setup" && ( + <> + {(isIntegration || isAgent) && selectedAction && Object.getOwnPropertyNames(selectedAction).length > 0 && selectedActionParameters?.length > 0 ? + apps !== undefined && apps !== null && apps.length > 0 && wrapperapp !== undefined && newimage !== undefined ? +
+
{ + selectedAction.example = "noapp" + selectedAction.large_image = newimage + if (cy !== undefined && cy !== null) { + const foundnode = cy.getElementById(selectedAction.id) + if (foundnode !== undefined && foundnode !== null) { + foundnode.data("large_image", newimage) + } + } + + /* + const iconInfo = GetIconInfo(selectedAction) + if (iconInfo !== undefined && iconInfo !== null) { + selectedAction.fillGradient = iconInfo.fillGradient + + selectedAction.iconBackground = iconInfo.iconBackgroundColor + selectedAction.fillstyle = "linear-gradient" + } + */ + + const paramIndex = selectedAction.parameters !== undefined && selectedAction.parameters !== null ? selectedAction.parameters.findIndex((param) => param.name === "app_name") : -1 + if (paramIndex === -1) { + console.log("Couldn't find app_name parameter") + selectedAction.parameters.push({ + name: "app_name", + value: wrapperapp.name, + autocompleted: false, + }) + } else { + selectedAction.parameters[paramIndex].value = wrapperapp.name + } + + setSelectedAction(selectedAction) + setUpdate(Math.random()) + + }}> + +
+ +
+ +
+ + {apps.map((app, appIndex) => { + // Forces it into every category (for now) + // This is to make it possible to "use" shuffle for Singul natively + if (app.name === "Shuffle Tools") { + if (actionname == "Intel" || actionname == "Intel") { + app.categories = [actionname] + } + } + + if (app.categories === undefined || app.categories === null || app.categories.length === 0) { + return null + } + + var newactionname = actionname.toLowerCase() + if (isAgent === true) { + newactionname = "ai" + } + + var found = false + for (var key in app.categories) { + + var localnewactionname = newactionname + if (newactionname == "comms") { + localnewactionname = "communication" + } + + if (app.categories[key].toLowerCase() !== localnewactionname) { + continue + } + + found = true + break + } + + if (!found) { + return null + } + + var isAppSelected = false + const paramIndex = selectedAction.parameters.findIndex((param) => param.name === "app_name") + if (paramIndex > -1) { + // Check the actual value and if it's the same + if (selectedAction.parameters[paramIndex].value === app.name) { + isAppSelected = true + } + } + + return ( +
{ + selectedAction.example = "" + selectedAction.large_image = app.large_image + if (cy !== undefined && cy !== null) { + const foundnode = cy.getElementById(selectedAction.id) + if (foundnode !== undefined && foundnode !== null) { + foundnode.data("large_image", app.large_image) + } + } + + if (paramIndex === -1) { + console.log("Couldn't find app_name parameter") + selectedAction.parameters.push({ + name: "app_name", + value: app.name, + autocompleted: false, + }) + } else { + selectedAction.parameters[paramIndex].value = app.name + } + + setSelectedAction(selectedAction) + setUpdate(Math.random()) + + + var requiresAuth = app?.authentication?.required + if (requiresAuth && appAuthentication?.length > 0) { + for (var key in appAuthentication) { + if (appAuthentication[key]?.app?.name === app?.name) { + requiresAuth = false + break + } + } + } + + setRequiresAuthentication(requiresAuth); + }}> + + + +
+ ) + })} +
+ : null + : + null + } - {activeMainTab === "setup" && ( - <>
Name @@ -2652,21 +2657,7 @@ const ParsedAction = (props) => { placeholder={selectedAction.execution_delay} value={delay} onChange={(event) => { - // Check if positive number - if (isNaN(event.target.value) || Number(event.target.value) < 0) { - toast.error("Please enter a valid positive number for delay.") - return - } - - // Check if first number is 0 - if (event.target.value.length > 1 && event.target.value.charAt(0) === "0") { - event.target.value = event.target.value.substring(1) - } - setDelay(event.target.value) - selectedAction.execution_delay = event.target.value - setSelectedAction(selectedAction) - setUpdate(Math.random()) }} /> @@ -2711,13 +2702,9 @@ const ParsedAction = (props) => { fullWidth variant="contained" onClick={() => { - if (isCloud) { - ReactGA.event({ - category: "Integration", - action: "Authenticate", - label: `${selectedApp?.name} - Open 1`, - }) - } + //if (authenticationType.type === "oauth2" && authenticationType.redirect_uri !== undefined && authenticationType.redirect_uri !== null) { + // return null + //} setAuthenticationModalOpen(true); }} @@ -2731,7 +2718,19 @@ const ParsedAction = (props) => { ) : null} {/* Change made in new release when we added Tabs system in it */} - {appMayNeedAuth ? ( + {( + (selectedAction.authentication !== undefined && + selectedAction.authentication !== null && + selectedAction.authentication.length > 0) || + (selectedApp.name !== undefined && + (((selectedAction.authentication === undefined || + selectedAction.authentication === null || + selectedAction.authentication.length === 0)) || + isAgent || + isIntegration) && + requiresAuthentication) + ) ? ( +
{ 0 && selectedAction?.selectedAuthentication && typeof selectedAction.selectedAuthentication === 'object' && Object.getOwnPropertyNames(selectedAction.selectedAuthentication).length !== 0 ? ( + workflow?.suborg_distribution?.length > 0 && selectedAction?.selectedAuthentication && Object.getOwnPropertyNames(selectedAction.selectedAuthentication).length !== 0 ? (
{ labelId="select-app-auth" value={ selectedAction?.authentication_id === "authgroups" ? "authgroups" : - (selectedAction?.selectedAuthentication === null || !selectedAction?.selectedAuthentication || typeof selectedAction.selectedAuthentication !== 'object' || Object.getOwnPropertyNames(selectedAction.selectedAuthentication).length === 0) + !selectedAction?.selectedAuthentication || Object.getOwnPropertyNames(selectedAction.selectedAuthentication).length === 0 ? "No selection" : selectedAction?.selectedAuthentication } @@ -2998,14 +2997,6 @@ const ParsedAction = (props) => { variant="outlined" style={{}} onClick={() => { - if (isCloud) { - ReactGA.event({ - category: "Integration", - action: "Authenticate", - label: `${selectedApp?.name} - Open 2`, - }) - } - setAuthenticationModalOpen(true); }} > @@ -3014,18 +3005,68 @@ const ParsedAction = (props) => {
- + {requiresAuthentication && (!selectedAction.authentication_id || selectedAction.authentication_id === "") ? ( +
+ + Authentication needed. + + + Some steps in this workflow won’t run until you connect your account. + + +
+ ) : null}
) : null} - {selectedAction.authentication_id === "authgroups" && (authGroups === undefined || authGroups === null || authGroups.length === 0) ? - - - Create your first Authentication group - - - : null} + {selectedAction.authentication_id === "authgroups" && (authGroups === undefined || authGroups === null || authGroups.length === 0) ? + + + Create your first Authentication group + + + : null} {/*showEnvironment !== undefined && showEnvironment && environments.length > 1 && !isIntegration ? ( @@ -3118,8 +3159,6 @@ const ParsedAction = (props) => {
) : null*/} - - {workflow.execution_variables !== undefined && workflow.execution_variables !== null && workflow.execution_variables.length > 0 ? (
Runtime variable (optional) @@ -3431,80 +3470,11 @@ const ParsedAction = (props) => { ) : null}
- {appMayNeedAuth && !hasAuth && !isAgent && !isIntegration ? ( -
- - Authentication needed {isIntegration || isAgent ? `` : "."} - - - This step may not work until you authenticate it. - - -
- ) : null} - {activeMainTab === "setup" && ( - - - - setAnchorEl(null)} - - anchorOrigin={{ - vertical: 'bottom', - horizontal: 'left', - }} - transformOrigin={{ - vertical: 'top', - horizontal: 'left', - }} - - style={{ - zIndex: 20000, - marginTop: 2, - border: "1px solid rgba(255,255,255,0.3)", - }} - - PaperProps={{ - style: { - maxHeight: 600, - maxWidth: 450, - } - }} - > - { - e.preventDefault() - e.stopPropagation() - - const paramIndex = selectedAction.parameters.findIndex((param) => param.name === "app_name") - if (paramIndex === -1) { - selectedAction.parameters.push({ - "description": "The name of the app to run the LLM query against", - "id": "", - "name": "app_name", - "example": "", - "value": "", - "multiline": false, - "multiselect": false, - "options": null, - "action_field": "", - "variant": "STATIC_VALUE", - "required": true, - "configuration": false, - "tags": null, - "schema": { - "type": "" - }, - "skip_multicheck": false, - "value_replace": null, - "unique_toggled": false, - "error": "", - "hidden": false, - - "custom_value": true, - }) - } else { - selectedAction.parameters[paramIndex].custom_value = true - } - - // Overwrite params for custom value handling - const newSelectedActionParameters = JSON.parse(JSON.stringify(selectedAction?.parameters)) - setSelectedActionParameters(newSelectedActionParameters) - setSelectedAction(selectedAction) - setAnchorEl(null) - }} - selected={false} - style={{ - margin: 7, - display: "flex", - minWidth: 400, - maxWidth: 400, - cursor: "pointer", - }} - > - - Custom Value - - - - - {apps.map((item, index) => { - const parsedName = (item.name?.charAt(0).toUpperCase() + item.name?.substring(1)).replace(/_/g, " ") - return ( - { - //handleSelect(item) - setSelectedActionLocal(selectedAction, item) - setAnchorEl(null) - }} - selected={false} - style={{ - margin: 7, - display: "flex", - minWidth: 400, - maxWidth: 400, - cursor: "pointer", - }} - > - - - {parsedName} - - - ) - })} - - - - {apps.map((app, appIndex) => { - // Forces it into every category (for now) - // This is to make it possible to "use" shuffle for Singul natively - if (app.name === "Shuffle Tools") { - if (actionname == "Intel" || actionname == "Intel") { - app.categories = [actionname] - } - } - - var isAppSelected = false - const paramIndex = selectedAction.parameters.findIndex((param) => param.name === "app_name") - if (paramIndex > -1) { - // Check the actual value and if it's the same - //if (selectedAction.parameters[paramIndex].value === app.name) { - if (selectedAction.parameters[paramIndex].value.includes(app.name)) { - isAppSelected = true - } - } - - if (app.categories === undefined || app.categories === null || app.categories.length === 0) { - if (!isAppSelected) { - return null - } - } - - var newactionname = actionname.toLowerCase() - if (isAgent === true) { - //newactionname = "ai" - } else { - var found = false - for (var key in app.categories) { - - var localnewactionname = newactionname - if (newactionname == "comms") { - localnewactionname = "communication" - } - - if (app.categories[key].toLowerCase() !== localnewactionname) { - continue - } - - found = true - break - } - - if (!found && !isAppSelected) { - return null - } - } - - - return ( -
{ - setSelectedActionLocal(selectedAction, app) - }}> - - - -
- ) - })} - -
-
- : null - : null - } +
@@ -3824,7 +3524,7 @@ const ParsedAction = (props) => { marginBottom: hideExtraTypes ? 50 : 200, }} > { - selectedActionParameters !== undefined && selectedActionParameters !== null && selectedAction && selectedAction !== null && typeof selectedAction === 'object' && Object.getOwnPropertyNames(selectedAction).length > 0 && selectedActionParameters.length > 0 ? + selectedActionParameters !== undefined && selectedActionParameters !== null && selectedAction && Object.getOwnPropertyNames(selectedAction).length > 0 && selectedActionParameters.length > 0 ?
{/* { } if ((isIntegration || isAgent) && data.name === "app_name") { - if (data?.custom_value === true) { - //data.label = "Allowed MCPs" - } else { - return null - } - } + return null + } /* // Somehow autogenerate from the app itself @@ -4131,9 +3827,13 @@ const ParsedAction = (props) => { } if (selectedAction.name === "custom_action" && data.name === "body") { - const methodParam = selectedActionParameters.find(p => p.name === "method") || selectedAction.parameters?.find(p => p.name === "method"); - if (methodParam?.value?.toUpperCase() === "GET") { - return null + for (var key in selectedActionParameters) { + const param = selectedActionParameters[key] + if (param.name === "method") { + if (param.value === "GET") { + return null + } + } } } @@ -4197,7 +3897,6 @@ const ParsedAction = (props) => { backgroundColor: themeMode === "dark" ? "#161616" : "#CCCCCC", color: theme.palette.text.primary, fontWeight: 600, - borderRadius: "6px !important", "&:hover": { backgroundColor: themeMode === "dark" ? "rgba(0,0,0,0.3)" : "rgba(0,0,0,0.1)", }, @@ -4623,7 +4322,7 @@ const ParsedAction = (props) => { } - if ((multiline === undefined || multiline === false) && (data.name.startsWith("${") && data.name.endsWith("}"))) { + if ((multiline === undefined || multiline === false) && ((data?.autocompleted === true || data?.field_active === true) || data.name.startsWith("${") && data.name.endsWith("}"))) { multiline = true } @@ -5128,16 +4827,6 @@ const ParsedAction = (props) => { fullWidth id={"rightside_field_" + count} onChange={(e) => { - if (e.target.value.includes("custom_shuffle_action")) { - data.options = [] - selectedActionParameters[count].options = [] - setSelectedActionParameters(selectedActionParameters) - selectedAction.parameters = selectedActionParameters - setSelectedAction(selectedAction) - setUpdate(Math.random()) - return - } - changeActionParameter(e, count, data); setUpdate(Math.random()); }} @@ -5182,22 +4871,6 @@ const ParsedAction = (props) => { ); } )} - - - {isAgent || isIntegration ? - - Custom Value - - : null} ); } else if (data.variant === "STATIC_VALUE") { @@ -5590,14 +5263,12 @@ const ParsedAction = (props) => { const buttonTitle = `Authenticate the ${selectedApp?.name?.replaceAll("_", " ")} API` const hasAutocomplete = data?.autocompleted === true - const isPathField = selectedAction?.name === "custom_action" && data?.name === "path" - if (data.variant === undefined || data.variant === null) { data.variant = "STATIC_VALUE" } - var isFirstOptional = optionalFound === false && data.configuration === false && data.required === false && !isPathField ? true : false - if (optionalFound === false && data.configuration === false && data.required === false && !isPathField) { + var isFirstOptional = optionalFound === false && data.configuration === false && data.required === false ? true : false + if (optionalFound === false && data.configuration === false && data.required === false) { optionalFound = true } @@ -5624,7 +5295,7 @@ const ParsedAction = (props) => { } } - const isOptional = (data.configuration === false && data.required === false) && !isPathField + const isOptional = data.configuration === false && data.required === false return (
@@ -5681,13 +5352,6 @@ const ParsedAction = (props) => { color: theme.palette.textPrimary, }} onClick={() => { - if (isCloud) { - ReactGA.event({ - category: "Integration", - action: "Authenticate", - label: `${selectedApp?.name} - Open 4`, - }) - } setAuthenticationModalOpen(true); }} /> diff --git a/frontend/src/components/PartnersUsecasesTab.jsx b/frontend/src/components/PartnersUsecasesTab.jsx index 85c40d56..9cbc31a1 100644 --- a/frontend/src/components/PartnersUsecasesTab.jsx +++ b/frontend/src/components/PartnersUsecasesTab.jsx @@ -846,7 +846,6 @@ const PartnersUsecasesTab = ({ isCloud, globalUrl, userdata, partnerData, setPar setIsLoading(true); if(!isCloud || !userdata?.active_org?.is_partner) { // If the user is not a partner or if it's not a cloud environment do not make api call :) - setIsLoading(false); return; } // Load usecase data from API diff --git a/frontend/src/components/RuntimeDebugger.jsx b/frontend/src/components/RuntimeDebugger.jsx index 51a3aaaf..ba531c5b 100644 --- a/frontend/src/components/RuntimeDebugger.jsx +++ b/frontend/src/components/RuntimeDebugger.jsx @@ -1087,9 +1087,6 @@ const RuntimeDebugger = (props) => { options={[{ "name": "Agent Runs", "id": "AGENT", - },{ - "name": "Sensor Actions", - "id": "SENSOR_ACTION", }].concat(workflows)} fullWidth style={{ diff --git a/frontend/src/components/SearchContactForm.jsx b/frontend/src/components/SearchContactForm.jsx deleted file mode 100644 index ff5747a5..00000000 --- a/frontend/src/components/SearchContactForm.jsx +++ /dev/null @@ -1,121 +0,0 @@ -import React, { useState } from "react"; -import theme from "../theme.jsx"; -import { TextField, Typography, Button } from "@mui/material"; - -const SearchContactForm = ({ globalUrl, isMobile, tabName }) => { - const [formMail, setFormMail] = useState(""); - const [message, setMessage] = useState(""); - const [formMessage, setFormMessage] = useState(""); - - const submitContact = (email, message) => { - const data = { - firstname: "", - lastname: "", - title: "", - companyname: "", - email: email, - phone: "", - message: message, - }; - - const errorMessage = - "Something went wrong. Please contact frikky@shuffler.io directly."; - - fetch(globalUrl + "/api/v1/contact", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(data), - }) - .then((response) => response.json()) - .then((response) => { - setFormMessage( - response?.success === true ? response.reason : errorMessage - ); - setFormMail(""); - setMessage(""); - }) - .catch(() => { - setFormMessage(errorMessage); - }); - }; - - return ( -
- - Can't find what you're looking for? - -
- setFormMail(e.target.value)} - /> - setMessage(e.target.value)} - /> -
- - - {formMessage} - -
- ); -}; - -export default SearchContactForm; diff --git a/frontend/src/components/SearchData.jsx b/frontend/src/components/SearchData.jsx index f7a7dd33..de209b4b 100644 --- a/frontend/src/components/SearchData.jsx +++ b/frontend/src/components/SearchData.jsx @@ -47,7 +47,7 @@ const chipStyle = { backgroundColor: "#3d3f43", height: 30, marginRight: 5, paddingLeft: 5, paddingRight: 5, height: 28, cursor: "pointer", borderColor: "#3d3f43", color: "white", } -const searchClient = algoliasearch("JNSS5CFDZZ", "33e4e3564f4f060e96e0531957bed552") +const searchClient = algoliasearch("JNSS5CFDZZ", "c8f882473ff42d41158430be09ec2b4e") const SearchData = props => { const { serverside, globalUrl, userdata } = props let navigate = useNavigate(); diff --git a/frontend/src/components/Searchfield.jsx b/frontend/src/components/Searchfield.jsx index 57558241..4447c73b 100644 --- a/frontend/src/components/Searchfield.jsx +++ b/frontend/src/components/Searchfield.jsx @@ -38,6 +38,7 @@ import aa from 'search-insights' import { InstantSearch, Configure, connectSearchBox, connectHits, Index } from 'react-instantsearch-dom'; //import { InstantSearch, SearchBox, Hits, connectSearchBox, connectHits, Index } from 'react-instantsearch-dom'; +import { HotKeys } from 'react-hotkeys'; // https://www.algolia.com/doc/api-reference/widgets/search-box/react/ const chipStyle = { backgroundColor: "#3d3f43", height: 30, marginRight: 5, paddingLeft: 5, paddingRight: 5, height: 28, cursor: "pointer", borderColor: "#3d3f43", color: "white", @@ -167,4 +168,4 @@ const SearchField = props => { ) } -export default SearchField; \ No newline at end of file +export default SearchField; diff --git a/frontend/src/components/ShuffleCodeEditor1.jsx b/frontend/src/components/ShuffleCodeEditor1.jsx index c2438b97..3059d936 100644 --- a/frontend/src/components/ShuffleCodeEditor1.jsx +++ b/frontend/src/components/ShuffleCodeEditor1.jsx @@ -61,6 +61,8 @@ import PaperComponent from "../components/PaperComponent.jsx"; import { padding, textAlign } from '@mui/system'; import data from '../frameworkStyle.jsx'; import { useNavigate, Link, useParams, useSearchParams } from "react-router-dom"; +import { tags as t } from '@lezer/highlight'; + import AceEditor from "react-ace"; import ace from "ace-builds"; @@ -147,10 +149,7 @@ const CodeEditor = (props) => { // Auto-indent JSON-like content (with safety hehe) const autoIndentContent = React.useCallback((content) => { - if (!isFileEditor) { - console.log("Autoindent disabled") - return content - } + return content // Safety checks :) if (!content || typeof content !== 'string' || content.trim().length === 0) { @@ -239,24 +238,6 @@ const CodeEditor = (props) => { expectedOutput(localcodedata) }, [localcodedata]) - useEffect(() => { - if (!isFileEditor) { - return - } - - if (codedata === undefined || codedata === null || typeof codedata !== 'string') { - return - } - - const indentedContent = autoIndentContent(codedata); - if (indentedContent !== undefined && indentedContent !== null) { - console.log("SETTING: ", indentedContent) - setlocalcodedata(indentedContent); - } else { - console.log("INDENT FAILED") - } - }, []) - // Auto-indent when codedata prop changes useEffect(() => { if (codedata && codedata !== localcodedata && typeof codedata === 'string') { @@ -1616,7 +1597,7 @@ const CodeEditor = (props) => { if (e.srcElement.className === "ace_content") { console.log("DRAG STOP IN CONTENT!", e.srcElement.className) - const usedposition = e.offsetY + let usedposition = e.offsetY if (usedposition === undefined || usedposition === null) { toast.info(`Error: LayerY is undefined or null. Please contact ${supportEmail}`) return @@ -1803,8 +1784,8 @@ const CodeEditor = (props) => { // zIndex: 12501, pointerEvents: "auto", color: theme.palette.DialogStyle.color, - minWidth: isMobile || isWorkflowEditor || fullScreenModeEnabled ? "100%" : isFileEditor ? 800 : "80%", - maxWidth: isMobile || isWorkflowEditor || fullScreenModeEnabled ? "100%" : isFileEditor ? 800 : "1100px", + minWidth: isMobile || isWorkflowEditor || fullScreenModeEnabled ? "100%" : isFileEditor ? "650px" : "80%", + maxWidth: isMobile || isWorkflowEditor || fullScreenModeEnabled ? "100%" : isFileEditor ? "650px" : "1100px", minHeight: isMobile || isWorkflowEditor || fullScreenModeEnabled ? "100%" : "auto", maxHeight: isMobile || isWorkflowEditor || fullScreenModeEnabled ? "100%" : "700px", border: "3px solid rgba(255,255,255,0.3)", @@ -2000,8 +1981,7 @@ const CodeEditor = (props) => { paddingLeft: 10, }} > - {/* cba positioning */} - File Editor ({localcodedata.length})                                                                             {validation === true ? Valid JSON : Invalid JSON} + File Editor ({localcodedata.length})
@@ -2531,7 +2511,7 @@ const CodeEditor = (props) => { } - {/*(actionId || triggerId || conditionId) && !isWorkflowEditor && !isFileEditor ? + {(actionId || triggerId || conditionId) && !isWorkflowEditor && !isFileEditor ? <> { : null - */} + }
} @@ -2603,7 +2583,7 @@ const CodeEditor = (props) => { mode={isWorkflowEditor ? "yaml" : selectedAction === undefined ? "json" : selectedAction.name === "execute_python" ? "python" : selectedAction.name === "execute_bash" ? "bash" : "json"} theme="gruvbox" height={fullScreenModeEnabled ? "84vh" : isFileEditor ? 450 : isWorkflowEditor ? "90vh" : 550} - width={isFileEditor ? 800 : fullScreenModeEnabled ? isFileEditor ? "100%" : "50vw" : isWorkflowEditor ? "90vw" : "100%"} + width={isFileEditor ? 650 : fullScreenModeEnabled ? "50vw" : isWorkflowEditor ? "90vw" : "100%"} markers={markers} highlightActiveLine={false} @@ -2737,7 +2717,7 @@ const CodeEditor = (props) => {
: - {selectedAction?.name === "execute_python" || selectedAction?.name === "execute_bash" ? + {selectedAction.name === "execute_python" || selectedAction.name === "execute_bash" ? "Code to run" : triggerId ? `Output: ${triggerName?.replaceAll("_", " ").slice(0, 1).toUpperCase() + triggerName?.replaceAll("_", " ").slice(1)} (${triggerField})` : diff --git a/frontend/src/components/SubOrgDistributionDialog.jsx b/frontend/src/components/SubOrgDistributionDialog.jsx deleted file mode 100644 index 31740c80..00000000 --- a/frontend/src/components/SubOrgDistributionDialog.jsx +++ /dev/null @@ -1,265 +0,0 @@ -import React, { useState, useContext } from "react"; -import { - Dialog, - DialogTitle, - DialogContent, - Box, - TextField, - Button, - Typography, - List, - ListItem, - ListItemText, - Checkbox, - InputAdornment, -} from "@mui/material"; -import { Search as SearchIcon } from "@mui/icons-material"; -import { Context } from "../context/ContextApi.jsx"; -import { getTheme } from "../theme.jsx"; - -/** - * A reusable dialog for selecting/distributing sub-organizations. - * - * Props: - * open {boolean} - controls dialog visibility - * onClose {function} - called on Cancel or backdrop click (no args) - * title {string} - dialog title text - * extraInfo {string} - secondary line below the title (e.g. "Selected Key: xxx") - * orgs {Array} - ordered array of { id, name, image? } objects to display - * selectedOrgIds {string[]} - currently selected org IDs (controlled) - * onSelectionChange {function} - called with a updater fn (prev => next) when selection changes - * onSave {function} - called with the final selectedOrgIds array when Save is clicked - * disabled {boolean} - disables checkboxes and Save button (default: false) - */ -const SubOrgDistributionDialog = ({ - open, - onClose, - title, - extraInfo = null, - orgs = [], - selectedOrgIds = [], - onSelectionChange, - onSave, - disabled = false, -}) => { - const [searchQuery, setSearchQuery] = useState(""); - const { themeMode, brandColor } = useContext(Context); - const theme = getTheme(themeMode, brandColor); - - const safeOrgs = orgs || []; - - const filteredOrgs = safeOrgs.filter( - o => o && o.name.toLowerCase().includes(searchQuery.toLowerCase()) - ); - - const handleSelectAll = () => { - if (searchQuery) { - const filteredIds = filteredOrgs.map(o => o.id); - onSelectionChange(prev => [...new Set([...prev, ...filteredIds])]); - } else { - const allIds = safeOrgs.map(o => o.id); - onSelectionChange(prev => [...new Set([...prev, ...allIds])]); - } - }; - - const handleDeselectAll = () => { - if (searchQuery) { - const filteredIds = filteredOrgs.map(o => o.id); - onSelectionChange(prev => prev.filter(id => !filteredIds.includes(id))); - } else { - onSelectionChange([]); - } - }; - - const handleToggle = (id) => { - if (disabled) return; - onSelectionChange(prev => - prev.includes(id) ? prev.filter(i => i !== id) : [...prev, id] - ); - }; - - const handleClose = () => { - setSearchQuery(""); - onClose(); - }; - - const filteredSelected = filteredOrgs.filter(o => selectedOrgIds.includes(o.id)).length; - const countText = searchQuery - ? `${filteredSelected} of ${filteredOrgs.length} filtered selected` - : `${selectedOrgIds.length} of ${safeOrgs.length} selected`; - - const imageSize = 22; - const imageStyle = { width: imageSize, height: imageSize, pointerEvents: "none", marginRight: 10 }; - - return ( - - - - {title} - - {extraInfo && ( - - {extraInfo} - - )} - - - - setSearchQuery(e.target.value)} - InputProps={{ - startAdornment: ( - - - - ), - style: { color: theme.palette.textFieldStyle?.color }, - }} - style={{ backgroundColor: theme.palette.textFieldStyle?.backgroundColor, flexShrink: 0 }} - /> - -
- - - - {countText} - -
- -
- - {filteredOrgs.map((org, index) => { - const isSelected = selectedOrgIds.includes(org.id); - const hasImage = org.image !== undefined; - return ( - handleToggle(org.id)} - sx={{ - cursor: disabled ? "default" : "pointer", - backgroundColor: index % 2 === 0 - ? "transparent" - : themeMode === "dark" ? "rgba(255,255,255,0.02)" : "rgba(0,0,0,0.02)", - "&:hover": { - backgroundColor: themeMode === "dark" ? "rgba(255,255,255,0.05)" : "rgba(0,0,0,0.05)", - }, - }} - > - - {hasImage && ( - org.image === "" ? ( - {org.name} - ) : ( - {org.name} - ) - )} - - - ); - })} - {filteredOrgs.length === 0 && ( - - - - )} - -
- - - -
- - -
- - - ); -}; - -export default SubOrgDistributionDialog; diff --git a/frontend/src/components/TenantsTab.jsx b/frontend/src/components/TenantsTab.jsx index 779cc910..16cd5f11 100644 --- a/frontend/src/components/TenantsTab.jsx +++ b/frontend/src/components/TenantsTab.jsx @@ -1,6 +1,5 @@ import React, { memo, useContext, useEffect, useState } from 'react'; -import { DataGrid } from '@mui/x-data-grid'; -import { getTheme } from "../theme.jsx"; +import {getTheme} from "../theme.jsx"; import { Context } from '../context/ContextApi.jsx'; import { FormControl, @@ -25,11 +24,9 @@ import { IconButton, Modal, Checkbox, - Select, - MenuItem, -} from "@mui/material"; - -import { + } from "@mui/material"; + + import { Edit as EditIcon, Polyline as PolylineIcon, CheckCircle as CheckCircleIcon, @@ -37,13 +34,11 @@ import { Apps as AppsIcon, Business as BusinessIcon, Flag, - ArrowDropDown as ArrowDropDownIcon, + ArrowDropDown as ArrowDropDownIcon, VisibilityOff, Visibility, - KeyboardArrowLeft, - KeyboardArrowRight, -} from "@mui/icons-material"; + } from "@mui/icons-material"; import { toast } from 'react-toastify'; @@ -78,15 +73,8 @@ const TenantsTab = memo((props) => { const theme = getTheme(themeMode, brandColor); const [accountDeleteButtonClicked, setAccountDeleteButtonClicked] = useState(false); const [selectedSuborg, setSelectedSuborg] = useState(null); - const [rowsPerPage, setRowsPerPage] = useState(10); - const [nextCursor, setNextCursor] = useState(""); - const [currentCursor, setCurrentCursor] = useState(""); - const [cursorStack, setCursorStack] = useState([]); - const [localPage, setLocalPage] = useState(0); - const [loadingSubOrgs, setLoadingSubOrgs] = useState(false); - const [isChangingOrg, setIsChangingOrg] = useState(false); useEffect(() => { - if (parentOrg !== null && parentOrgFlag === null) { + if(parentOrg !== null && parentOrgFlag === null) { let regiontag = "UK"; let regionCode = "gb"; @@ -97,25 +85,25 @@ const TenantsTab = memo((props) => { regiontag = namesplit[namesplit.length - 1]; if (regiontag === "california") { - regiontag = "US"; - regionCode = "us"; + regiontag = "US"; + regionCode = "us"; } else if (regiontag === "frankfurt") { - regiontag = "EU-2"; - regionCode = "eu"; + regiontag = "EU-2"; + regionCode = "eu"; } else if (regiontag === "ca") { - regiontag = "CA"; - regionCode = "ca"; - } else if (regiontag === "au") { + regiontag = "CA"; + regionCode = "ca"; + }else if (regiontag === "au") { regiontag = "AUS"; regionCode = "au" } } setParentOrgFlag(regionCode); setParentOrgRegionName(regiontag); - } } + } }, [parentOrg, parentOrgFlag]); - + var syncList = [ { primary: "Workflows", @@ -139,26 +127,19 @@ const TenantsTab = memo((props) => { useEffect(() => { if (userdata.orgs !== undefined && userdata.orgs !== null && userdata.orgs.length > 0) { - handleGetSubOrgs(userdata.active_org.id, "", 100); + handleGetSubOrgs(userdata.active_org.id); } else console.log("error in user data") }, [userdata]); - const handleGetSubOrgs = (orgId, cursor = "", limit = 100, direction = "next") => { - const effectiveLimit = limit !== null ? limit : 100; + const handleGetSubOrgs = (orgId) => { if (orgId.length === 0) { toast("Organization ID not defined. Please contact us on https://shuffler.io if this persists logout."); return; } - setLoadingSubOrgs(true); - let url = `${globalUrl}/api/v1/orgs/${orgId}/suborgs?limit=${effectiveLimit}`; - if (cursor) { - url += `&cursor=${encodeURIComponent(cursor)}`; - } - - fetch(url, { + fetch(`${globalUrl}/api/v1/orgs/${orgId}/suborgs`, { method: "GET", credentials: "include", headers: { @@ -173,87 +154,49 @@ const TenantsTab = memo((props) => { }) .then((responseJson) => { if (responseJson.success === false) { - setLoadOrgs(false); - setLoadingSubOrgs(false); + setLoadOrgs(false) //toast("Failed getting your org. If this persists, please contact support."); } else { - const { subOrgs, parentOrg, cursor: responseCursor } = responseJson; - setLoadOrgs(false); - setLoadingSubOrgs(false); - setSubOrgs(subOrgs || []); + const { subOrgs, parentOrg } = responseJson; + setLoadOrgs(false) + setSubOrgs(subOrgs); setParentOrg(parentOrg); - setNextCursor(responseCursor || ""); - - if (direction === "prev") { - const len = (subOrgs || []).length; - setLocalPage(len > 0 ? Math.ceil(len / rowsPerPage) - 1 : 0); - } else { - setLocalPage(0); - } let regiontag = "UK"; let regionCode = "gb"; if (parentOrg?.region_url?.length > 0) { - const regionsplit = parentOrg?.region_url.split("."); - if (regionsplit.length > 2 && !regionsplit[0].includes("shuffler")) { - const namesplit = regionsplit[0].split("/"); - regiontag = namesplit[namesplit.length - 1]; + const regionsplit = parentOrg?.region_url.split("."); + if (regionsplit.length > 2 && !regionsplit[0].includes("shuffler")) { + const namesplit = regionsplit[0].split("/"); + regiontag = namesplit[namesplit.length - 1]; - if (regiontag === "california") { - regiontag = "US"; - regionCode = "us"; - } else if (regiontag === "frankfurt") { - regiontag = "EU-2"; - regionCode = "eu"; - } else if (regiontag === "ca") { - regiontag = "CA"; - regionCode = "ca"; - } else if (regiontag === "au") { - regiontag = "AUS"; - regionCode = "au" - } + if (regiontag === "california") { + regiontag = "US"; + regionCode = "us"; + } else if (regiontag === "frankfurt") { + regiontag = "EU-2"; + regionCode = "eu"; + } else if (regiontag === "ca") { + regiontag = "CA"; + regionCode = "ca"; + }else if (regiontag === "au") { + regiontag = "AUS"; + regionCode = "au" } + } setParentOrgFlag(regionCode); setParentOrgRegionName(regiontag); - } + } } }) .catch((error) => { console.log("Error getting sub orgs: ", error); //toast("Error getting sub organizations"); - setLoadOrgs(false); - setLoadingSubOrgs(false); + setLoadOrgs(false) }); }; - const handleNextPage = () => { - const maxLocalPage = Math.ceil(subOrgs.length / rowsPerPage) - 1; - if (localPage < maxLocalPage) { - setLocalPage(prev => prev + 1); - } else if (nextCursor && nextCursor !== currentCursor) { - setCursorStack(prev => [...prev, currentCursor]); - setCurrentCursor(nextCursor); - handleGetSubOrgs(userdata.active_org.id, nextCursor, 100, "next"); - } - }; - - const handlePrevPage = () => { - if (localPage > 0) { - setLocalPage(prev => prev - 1); - } else if (cursorStack.length > 0) { - const prevCursor = cursorStack[cursorStack.length - 1]; - setCursorStack(prev => prev.slice(0, -1)); - setCurrentCursor(prevCursor); - handleGetSubOrgs(userdata.active_org.id, prevCursor, 100, "prev"); - } - }; - - const handleChangeRowsPerPage = (newSize) => { - setRowsPerPage(Number(newSize)); - setLocalPage(0); - }; - const GridItem = (props) => { const [expanded, setExpanded] = React.useState(false); const [showEdit, setShowEdit] = React.useState(false); @@ -559,7 +502,7 @@ const TenantsTab = memo((props) => { const createSubOrg = (currentOrgId, name) => { const data = { name: name, org_id: currentOrgId }; const url = globalUrl + `/api/v1/orgs/${currentOrgId}/create_sub_org`; - setSuborglistOpen(true) + setSuborglistOpen(true) fetch(url, { mode: "cors", @@ -577,8 +520,8 @@ const TenantsTab = memo((props) => { if (responseJson["success"] === false) { if (responseJson.reason !== undefined) { toast.error(responseJson.reason, { - autoClose: 5000, - }) + autoClose: 5000, + }) } else { toast("Failed creating suborg. Please try again"); } @@ -611,7 +554,6 @@ const TenantsTab = memo((props) => { localStorage.setItem("globalUrl", ""); localStorage.setItem("getting_started_sidebar", "open"); - setIsChangingOrg(true); fetch(`${globalUrl}/api/v1/orgs/${orgId}/change`, { mode: "cors", credentials: "include", @@ -627,8 +569,8 @@ const TenantsTab = memo((props) => { if (response.status !== 200) { console.log("Error in response"); } else { - localStorage.setItem("apps", []) - } + localStorage.setItem("apps", []) + } return response.json(); }) @@ -665,215 +607,212 @@ const TenantsTab = memo((props) => { const [disabled, setDisabled] = useState(true); const [open, setOpen] = useState(true); const boxStyling = { - position: "relative", - top: "50%", - left: "50%", - transform: "translate(-50%, -50%)", - zIndex: "9999", - backgroundColor: theme.palette.backgroundColor, - color: theme.palette.text.primary, - padding: 20, - borderRadius: 5, - boxShadow: "0 0 10px rgba(0, 0, 0, 0.3)", - width: 430, - height: 430, + position: "relative", + top: "50%", + left: "50%", + transform: "translate(-50%, -50%)", + zIndex: "9999", + backgroundColor: theme.palette.backgroundColor, + color: theme.palette.text.primary, + padding: 20, + borderRadius: 5, + boxShadow: "0 0 10px rgba(0, 0, 0, 0.3)", + width: 430, + height: 430, }; - + const closeIconButtonStyling = { - color: theme.palette.text.primary, - border: "none", - backgroundColor: "transparent", - position: 'relative', - width: 20, - height: 20, - cursor: "pointer", - left: "calc(100% - 30px)", + color: theme.palette.text.primary, + border: "none", + backgroundColor: "transparent", + position: 'relative', + width: 20, + height: 20, + cursor: "pointer", + left: "calc(100% - 30px)", }; - + const handlePasswordVisibility = () => { - setShowPassword(!showPassword); + setShowPassword(!showPassword); }; - + const buttonStyle = { - marginTop: 20, - height: 50, - border: "none", - width: "100%", - fontSize: 16, - backgroundColor: disabled ? "gray" : "red", - color: theme.palette.text.primary, - cursor: disabled === false && "pointer", - }; - + marginTop: 20, + height: 50, + border: "none", + width: "100%", + fontSize: 16, + backgroundColor: disabled ? "gray" : "red", + color: theme.palette.text.primary, + cursor: disabled === false && "pointer", + }; + const handlePasswordChange = (e) => { - setPassword(e.target.value); + setPassword(e.target.value); }; - + const handleCheckBoxEvent = () => { - setUserDeleteAccepted((prev) => !prev); + setUserDeleteAccepted((prev) => !prev); }; - + useEffect(() => { - if (password.length > 8 && userDeleteAccepted) { - setDisabled(false); - } else { - setDisabled(true); - } + if (password.length > 8 && userDeleteAccepted) { + setDisabled(false); + } else { + setDisabled(true); + } }, [password, userDeleteAccepted]); - + const handleDeleteAccount = () => { - const baseURL = globalUrl; + const baseURL = globalUrl; + + const url = `${baseURL}/api/v1/orgs/${selectedSuborg?.id}`; - const url = `${baseURL}/api/v1/orgs/${selectedSuborg?.id}`; + const data = { + password: password, + }; - const data = { - password: password, - }; + fetch(url, { + mode: "cors", + method: "DELETE", + body: JSON.stringify(data), + credentials: "include", + crossDomain: true, + withCredentials: true, + headers: { + "Content-Type": "application/json", + }, + }) + .then((response) => response.json()) + .then((data) => { + if (data.success) { + toast.success( + "Suborg deleted" + ); + handleGetSubOrgs(userdata.active_org.id); + setAccountDeleteButtonClicked(false); - fetch(url, { - mode: "cors", - method: "DELETE", - body: JSON.stringify(data), - credentials: "include", - crossDomain: true, - withCredentials: true, - headers: { - "Content-Type": "application/json", - }, + } else { + if (data.reason) { + toast.error(data.reason); + }else { + toast.error("Failed to delete suborg. Please try again or contact support@shuffler.io for help."); + } + } }) - .then((response) => response.json()) - .then((data) => { - if (data.success) { - toast.success( - "Suborg deleted" - ); - setCursorStack([]); - setCurrentCursor(""); - setNextCursor(""); - handleGetSubOrgs(userdata.active_org.id); - setAccountDeleteButtonClicked(false); - - } else { - if (data.reason) { - toast.error(data.reason); - } else { - toast.error("Failed to delete suborg. Please try again or contact support@shuffler.io for help."); - } - } - }) - .catch((error) => { - console.error( - "There was a problem with your fetch operation:", - error - ); - }); + .catch((error) => { + console.error( + "There was a problem with your fetch operation:", + error + ); + }); }; - + return ( - -
- { - setAccountDeleteButtonClicked(false); - setSelectedSuborg(null); - }} - > - - -

Sub-Organization

- {/*
*/} -
- -
    -
  • - -
    • -
  • - -
    • -
-
+
+ { + setAccountDeleteButtonClicked(false); + setSelectedSuborg(null); + }} + > + + +

Sub-Organization

+ {/*
*/} +
+ +
    +
  • + +
    • +
  • + +
    • +
+
+ + +
+
+ + - - -
-
- - - {showPassword ? : } - - ), - }} - /> -
- -
-
- + {showPassword ? : } + + ), + }} + /> +
+ +
+
+ ); - }; + }; const modalView = ( { backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, zIndex: 1000, '& .MuiDialogContent-root': { - backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, }, '& .MuiDialogTitle-root': { - backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, }, '& .MuiDialogActions-root': { - backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, }, }, }} @@ -941,7 +880,7 @@ const TenantsTab = memo((props) => { - - - {!selectedOrganization?.creator_org?.length && ( - - )} -
- ), - }, - ]; - return ( -
+
{modalView} {cloudSyncModal} - {accountDeleteButtonClicked && } -
-
-
- Tenants - - Create, manage and change to sub-organizations (tenants)! {" "} - {isCloud - ? `You can only make a sub organization if you are a customer of shuffle or running a POC of the platform. Please contact ${supportEmail} to try it out.` - : ''}  - - Learn more - - -
- - - - - -
- } +
+
+
+ Tenants + + Create, manage and change to sub-organizations (tenants)! {" "} + {isCloud + ? `You can only make a sub organization if you are a customer of shuffle or running a POC of the platform. Please contact ${supportEmail} to try it out.` + : ''}  + - Your Parent Organization - -
- + + + + +
+ + Your Parent Organization + +
+
+ {/* { /> */} -
- - - - - {isCloud && ( - - )} - - - - - {loadOrgs ? ( - [...Array(3)].map((_, rowIndex) => ( - - {[ - { width: 100, minWidth: 100, maxWidth: 100 }, - { width: 250, minWidth: 50, maxWidth: 250 }, - { width: 400, minWidth: 400, maxWidth: 400 }, - { width: "28%", minWidth: "28%" }, - { width: 400, minWidth: 400, maxWidth: 400 }, - ].map((style, colIndex) => ( - - - - ))} - - )) - ) : parentOrg?.id?.length > 0 ? ( - - - } - style={{ - width: 100, - minWidth: 100, - maxWidth: 100, - display: "table-cell", - padding: "8px 8px 8px 20px", - textAlign: "center", - }} - /> - - {isCloud && ( - - {parentOrgFlag} - -
- } - style={{ display: "table-cell", padding: 8, verticalAlign: "middle" }} - /> - )} - - - - - - - } - style={{ display: "table-cell", verticalAlign: "middle" }} - /> - - ) : ( - - {Array(5).fill().map((_, index) => ( - - ))} - - )} - -
-
- - {(subOrgs.length > 0 || cursorStack.length > 0 || loadingSubOrgs) && ( -
- - -
- - Sub Organizations of the Current Organization ({subOrgs.length}) - -
- -
- {!suborglistOpen ? ( - - ) : ( - row.id} - sx={{ - border: "none", - color: theme.palette.text.primary, - '& .MuiDataGrid-columnHeaders': { - borderBottom: theme.palette.defaultBorder, - backgroundColor: theme.palette.platformColor, - }, - '& .MuiDataGrid-cell': { - borderBottom: theme.palette.defaultBorder, - display: "flex", - alignItems: "center", - }, - '& .MuiDataGrid-row': { - backgroundColor: theme.palette.platformColor, - }, - '& .MuiDataGrid-overlayWrapper': { - minHeight: subOrgs.length > 0 ? 0 : 100, - }, - }} - /> - )} -
- {suborglistOpen && ( -
- - Rows per page: - - - - - - = Math.ceil(subOrgs.length / rowsPerPage) - 1 && (!nextCursor || nextCursor === currentCursor))} - size="small" - sx={{ color: (loadingSubOrgs || (localPage >= Math.ceil(subOrgs.length / rowsPerPage) - 1 && (!nextCursor || nextCursor === currentCursor))) ? theme.palette.text.disabled : theme.palette.text.primary }} - > - - -
- )} -
- )} - - + - -
- + - All Tenants - + + + {isCloud && ( + + )} + + + + + {loadOrgs ? ( + [...Array(3)].map((_, rowIndex) => ( + + {[ + { width: 100, minWidth: 100, maxWidth: 100 }, + { width: 250, minWidth: 50, maxWidth: 250 }, + { width: 400, minWidth: 400, maxWidth: 400 }, + { width: "28%", minWidth: "28%" }, + { width: 400, minWidth: 400, maxWidth: 400 }, + ].map((style, colIndex) => ( + + + + ))} + + )) + ) : parentOrg?.id?.length > 0 ? ( + + + } + style={{ + width: 100, + minWidth: 100, + maxWidth: 100, + display: "table-cell", + padding: "8px 8px 8px 20px", + textAlign: "center", + }} + /> + + {isCloud && ( + + {parentOrgFlag} + +
+ } + style={{ display: "table-cell", padding: 8, verticalAlign: "middle" }} + /> + )} + + + + + + + } + style={{ display: "table-cell", verticalAlign: "middle" }} + /> + + ): ( + + {Array(5).fill().map((_, index) => ( + + ))} + + )} + +
- {/* 0 && ( +
+ + +
+ + Sub Organizations of the Current Organization ({subOrgs.length}) + +
+ + {/* */} + +
+ + {!suborglistOpen ? + + setSuborglistOpen(true)} + > + Show Sub-Organizations + + } + style={{ + width: 100, + minWidth: 100, + maxWidth: 100, + paddingLeft: 20, + display: "table-cell", + padding: "0px 8px 8px 8px", + textAlign: "center", + borderBottom: theme.palette.defaultBorder, + verticalAlign: "middle", + }} + /> + + : + + + + + {isCloud && ( + + )} + + + + {subOrgs.map((data, index) => { + let regiontag = "UK"; + let regionCode = "gb"; + + if (data.region_url?.length > 0) { + const regionsplit = data.region_url.split("."); + if (regionsplit.length > 2 && !regionsplit[0].includes("shuffler")) { + const namesplit = regionsplit[0].split("/"); + regiontag = namesplit[namesplit.length - 1]; + if (regiontag === "california") { + regiontag = "US"; + regionCode = "us"; + } else if (regiontag === "frankfurt") { + regiontag = "EU-2"; + regionCode = "eu"; + } else if (regiontag === "ca") { + regiontag = "CA"; + regionCode = "ca"; + } + } + } + var bgColor = themeMode === "dark" ? "#212121" : "#FFFFFF"; + if (index % 2 === 0) { + bgColor = themeMode === "dark" ? "#1A1A1A" : "#EAEAEA"; + } + + return ( + + } style={{ width: 100, + minWidth: 100, + maxWidth: 100, + display: "table-cell", + padding: "8px 8px 8px 20px", + textAlign: "center", }} /> + + + {isCloud && ( + + {regiontag} + +
+ } + style={{ display: "table-cell", padding: 8, verticalAlign: "middle" }} + /> + )} + + + + + + + + {selectedOrganization?.creator_org?.length > 0 ? null : + } + + + } + style={{ display: "table-cell", verticalAlign: "middle" }} + /> + + )})} + + } + + +
+
+ )} + + + +
+ + All Tenants + +
+ + {/* */} -
- - {!allTenantsOpen ? - + + {!allTenantsOpen ? + - setAllTenantsOpen(true)} - > - Show ALL your tenants - - } - style={{ - width: 100, - minWidth: 100, - maxWidth: 100, - paddingLeft: 20, - display: "table-cell", - padding: "0px 8px 8px 8px", - textAlign: "center", - borderBottom: theme.palette.defaultBorder, - verticalAlign: "middle", - }} - /> - - : - - - - - {isCloud && ( - - )} - - - + }} + > + setAllTenantsOpen(true)} + > + Show ALL your tenants + + } + style={{ + width: 100, + minWidth: 100, + maxWidth: 100, + paddingLeft: 20, + display: "table-cell", + padding: "0px 8px 8px 8px", + textAlign: "center", + borderBottom: theme.palette.defaultBorder, + verticalAlign: "middle", + }} + /> + + : + + + + + {isCloud && ( + + )} + + + - {userdata?.orgs?.length <= 0 ? ( - [...Array(6)].map((_, rowIndex) => ( - - {Array(7) - .fill() - .map((_, colIndex) => ( - - - - ))} - - )) - ) : ( - userdata?.orgs?.length > 0 && - userdata.orgs.map((data, index) => { - let regiontag = "UK"; - let regionCode = "gb"; + {userdata?.orgs?.length <= 0 ? ( + [...Array(6)].map((_, rowIndex) => ( + + {Array(7) + .fill() + .map((_, colIndex) => ( + + + + ))} + + )) + ) : ( + userdata?.orgs?.length > 0 && + userdata.orgs.map((data, index) => { + let regiontag = "UK"; + let regionCode = "gb"; - if (data.region_url?.length > 0) { - const regionsplit = data.region_url.split("."); - if (regionsplit.length > 2 && !regionsplit[0].includes("shuffler")) { - const namesplit = regionsplit[0].split("/"); - regiontag = namesplit[namesplit.length - 1]; + if (data.region_url?.length > 0) { + const regionsplit = data.region_url.split("."); + if (regionsplit.length > 2 && !regionsplit[0].includes("shuffler")) { + const namesplit = regionsplit[0].split("/"); + regiontag = namesplit[namesplit.length - 1]; - if (regiontag === "california") { - regiontag = "US"; - regionCode = "us"; - } else if (regiontag === "frankfurt") { - regiontag = "EU-2"; - regionCode = "eu"; - } else if (regiontag === "ca") { - regiontag = "CA"; - regionCode = "ca"; - } - } - } + if (regiontag === "california") { + regiontag = "US"; + regionCode = "us"; + } else if (regiontag === "frankfurt") { + regiontag = "EU-2"; + regionCode = "eu"; + } else if (regiontag === "ca") { + regiontag = "CA"; + regionCode = "ca"; + } + } + } - var bgColor = themeMode === "dark" ? "#212121" : "#FFFFFF"; - if (index % 2 === 0) { - bgColor = themeMode === "dark" ? "#1A1A1A" : "#EAEAEA"; - } + var bgColor = themeMode === "dark" ? "#212121" : "#FFFFFF"; + if (index % 2 === 0) { + bgColor = themeMode === "dark" ? "#1A1A1A" : "#EAEAEA"; + } - return ( - - - } - style={{ - width: 100, - minWidth: 100, - maxWidth: 100, - display: "table-cell", - padding: "8px 8px 8px 20px", - textAlign: "center", - }} - /> - - {isCloud ? ( - - {regiontag} + return ( + + + } + style={{ + width: 100, + minWidth: 100, + maxWidth: 100, + display: "table-cell", + padding: "8px 8px 8px 20px", + textAlign: "center", + }} + /> + + {isCloud ? ( + + {regiontag} - -
- } - style={{ - display: "table-cell", - padding: 8, - verticalAlign: "middle", - }} - > - ) : null} - - { - handleClickChangeOrg(data?.id); - }} - > - Change Active Org - - } - style={{ - display: "table-cell", - padding: 8, - verticalAlign: "middle", - }} - > - - ); - }) - )} - } - -
+ +
+ } + style={{ + display: "table-cell", + padding: 8, + verticalAlign: "middle", + }} + > + ) : null} + + { + handleClickChangeOrg(data?.id); + }} + > + Change Active Org + + } + style={{ + display: "table-cell", + padding: 8, + verticalAlign: "middle", + }} + > + + ); + }) + )} + } + +
diff --git a/frontend/src/components/UserManagmentTab.jsx b/frontend/src/components/UserManagmentTab.jsx index a27fdedf..41ab96f0 100644 --- a/frontend/src/components/UserManagmentTab.jsx +++ b/frontend/src/components/UserManagmentTab.jsx @@ -1,10 +1,11 @@ import React, { useState, useEffect, useContext, memo } from "react"; import { toast } from 'react-toastify'; import { Context } from "../context/ContextApi.jsx"; -import { Link } from "react-router-dom"; import { FormControl, InputLabel, + OutlinedInput, + Checkbox, Tooltip, Typography, Select, @@ -30,12 +31,23 @@ import { import { Cached as CachedIcon, Edit as EditIcon, + Style, } from "@mui/icons-material"; import ModeEditOutlineOutlinedIcon from '@mui/icons-material/ModeEditOutlineOutlined'; import ContentCopyOutlinedIcon from '@mui/icons-material/ContentCopyOutlined'; import {getTheme} from "../theme.jsx"; -import SubOrgDistributionDialog from "./SubOrgDistributionDialog.jsx"; +const ITEM_HEIGHT = 48; +const ITEM_PADDING_TOP = 8; +const MenuProps = { + PaperProps: { + style: { + maxHeight: ITEM_HEIGHT * 4.5 + ITEM_PADDING_TOP, + width: 500, + }, + }, + getContentAnchorEl: () => null, +}; const logsViewModal = false; const userdata = ""; @@ -64,11 +76,10 @@ const UserManagmentTab = memo((props) => { const [logsViewModal, setLogsViewModal] = React.useState(false); const [ipSelected, setIpSelected] = React.useState(""); const [userLogViewing, setUserLogViewing] = React.useState({}); - const [subOrgModalOpen, setSubOrgModalOpen] = React.useState(false); - const [pendingSubOrgs, setPendingSubOrgs] = React.useState([]); const { themeMode, supportEmail, brandColor } = useContext(Context); const theme = getTheme(themeMode, brandColor); + useEffect(() => { if (selectedOrganization?.mfa_required !== MFARequired) { setMFARequired(selectedOrganization?.mfa_required); @@ -236,6 +247,30 @@ const UserManagmentTab = memo((props) => { }); }; + const handleOrgEditChange = (event) => { + if (userdata.id === selectedUser.id) { + toast("Can't remove orgs from yourself"); + return; + } + + if (event.target.value.includes("ALL")) { + toast.info("Adding to available all sub-organizations. This may take a minute.") + event.target.value = selectedOrganization.child_orgs.map((org) => org.id) + } else if (event.target.value.includes("None")) { + toast.info("Removing from all sub-organizations. This may take a minute") + event.target.value = [] + } + + setMatchingOrganizations(event.target.value); + // Workaround for empty orgs + if (event.target.value.length === 0) { + event.target.value.push("REMOVE"); + } + + setUser(selectedUser.id, "suborgs", event.target.value); + //setUser(selectedUser.id, "suborgs", matchingOrganizations) + }; + const userOrgEdit = selectedUser.id !== undefined && selectedUser?.orgs !== undefined && @@ -243,44 +278,44 @@ const UserManagmentTab = memo((props) => { selectedOrganization?.child_orgs !== undefined && selectedOrganization?.child_orgs !== null && selectedOrganization?.child_orgs?.length > 0 ? ( - + + + Accessible Sub-Organizations ( + {selectedUser?.orgs ? selectedUser?.orgs?.length - 1 : 0}) + + + ) : null; - const subOrgManagementDialog = ( - setSubOrgModalOpen(false)} - title={`Manage Sub-Organizations for ${selectedUser?.username || ''}`} - orgs={selectedOrganization?.child_orgs || []} - selectedOrgIds={pendingSubOrgs} - onSelectionChange={setPendingSubOrgs} - onSave={(ids) => { - if (userdata.id === selectedUser.id) { - toast("Can't modify orgs for yourself"); - return; - } - const newValue = ids.length === 0 ? ["REMOVE"] : [...ids]; - setMatchingOrganizations([...ids]); - setUser(selectedUser.id, "suborgs", newValue); - setSubOrgModalOpen(false); - setSelectedUserModalOpen(false); - }} - disabled={selectedUser?.id === userdata?.id} - /> - ); - const getUsers = () => { fetch(globalUrl + "/api/v1/getusers", { method: "GET", @@ -1082,8 +1117,6 @@ const UserManagmentTab = memo((props) => { }); }; - var previousreferrer = "" - var nextreferrer = "" const logview = logsViewModal ? ( { onChange={(event) => { setIpSelected(event.target.value); getLogs(event.target.value, userLogViewing.id); + + }} > {(() => { const uniqueIPs = new Set(); - console.log("Login info: ", userLogViewing.login_info) return userLogViewing.login_info.map((data, index) => { - console.log("Data: ", data) - if (data.ip.includes("127.0.0.1") || uniqueIPs.has(data.ip)) { - return null + if ( + data.ip.includes("127.0.0.1") || + uniqueIPs.has(data.ip) + ) { + return null; } - uniqueIPs.add(data.ip) + uniqueIPs.add(data.ip); return ( - {data?.timestamp ? new Date(data.timestamp * 1000).toLocaleString() : "N/A"} - {data?.ip} + {data.ip} ); }); @@ -1193,13 +1229,12 @@ const UserManagmentTab = memo((props) => { minWidth: 700, maxWidth: 700, overflow: "hidden", - marginLeft: 50, + marginLeft: 10, }} /> {logs.map((data, index) => { - previousreferrer = nextreferrer - nextreferrer = data.referer + //console.log("LOG: ", data) return ( // redirect user to logs @@ -1208,8 +1243,6 @@ const UserManagmentTab = memo((props) => { key={index} style={{ backgroundColor: index % 2 === 0 ? "#1f2023" : "#27292d", - paddingTop: data.referer !== previousreferrer ? 50 : 0, - borderTop: data.referer !== previousreferrer ? `1px solid rgba(255,255,255,0.3)` : "none", }} > { }} /> - - - + )})} @@ -1260,7 +1290,6 @@ const UserManagmentTab = memo((props) => {
{modalView} {editUserModal} - {subOrgManagementDialog} {logview}
diff --git a/frontend/src/components/WorkflowGrid.jsx b/frontend/src/components/WorkflowGrid.jsx index 9e843206..e4a34134 100644 --- a/frontend/src/components/WorkflowGrid.jsx +++ b/frontend/src/components/WorkflowGrid.jsx @@ -3,7 +3,6 @@ import React, { useEffect, useState } from 'react'; import {Link} from 'react-router-dom'; import theme from '../theme.jsx'; import { removeQuery } from '../components/ScrollToTop.jsx'; -import SearchContactForm from '../components/SearchContactForm.jsx'; import { Search as SearchIcon, CloudQueue as CloudQueueIcon, Code as CodeIcon } from '@mui/icons-material'; @@ -26,23 +25,66 @@ import { useDebouncedCallback } from "../utils/useDebouncedCallback.jsx"; import WorkflowPaper from "../components/WorkflowPaper.jsx" import WorkflowPaperNew from "../components/WorkflowPaperNew.jsx" -const searchClient = algoliasearch("JNSS5CFDZZ", "eb5fd80aa6ed5ab4730d836cff3ea283") +const searchClient = algoliasearch("JNSS5CFDZZ", "c8f882473ff42d41158430be09ec2b4e") const AppGrid = props => { const { maxRows, showName, showSuggestion, isMobile, globalUrl, parsedXs, alternativeView, onlyResults, inputsearch } = props - const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; + const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; const rowHandler = maxRows === undefined || maxRows === null ? 50 : maxRows const xs = parsedXs === undefined || parsedXs === null ? isMobile ? 6 : 4 : parsedXs //const [apps, setApps] = React.useState([]); //const [filteredApps, setFilteredApps] = React.useState([]); + const [formMail, setFormMail] = React.useState(""); + const [message, setMessage] = React.useState(""); + const [formMessage, setFormMessage] = React.useState(""); const [usecases, setUsecases] = React.useState([]); const [localMessage, setLocalMessage] = React.useState(""); + const buttonStyle = {borderRadius: 30, height: 50, width: 220, margin: isMobile ? "15px auto 15px auto" : 20, fontSize: 18,} + const innerColor = "rgba(255,255,255,0.65)" const borderRadius = 3 window.title = "Shuffle | Workflows | Discover your use-case" + const submitContact = (email, message) => { + const data = { + "firstname": "", + "lastname": "", + "title": "", + "companyname": "", + "email": email, + "phone": "", + "message": message, + } + + const errorMessage = "Something went wrong. Please contact frikky@shuffler.io directly." + + fetch(globalUrl+"/api/v1/contact", { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(data), + }) + .then(response => response.json()) + .then(response => { + if (response.success === true) { + setFormMessage(response.reason) + //toast("Thanks for submitting!") + } else { + setFormMessage(errorMessage) + } + + setFormMail("") + setMessage("") + }) + .catch(error => { + setFormMessage(errorMessage) + console.log(error) + }); + } + const handleKeysetting = (categorydata, workflows) => { console.log("Workflows: ", workflows) //workflows[0].category = ["detect"] @@ -187,16 +229,10 @@ const AppGrid = props => { placeholder="Find Workflows..." id="shuffle_search_field" onChange={(event) => { + removeQuery("q") const value = event.currentTarget.value setInputValue(value) debouncedRefine(value) - const urlSearchParams = new URLSearchParams(window.location.search) - if (value) { - urlSearchParams.set("q", value) - } else { - urlSearchParams.delete("q") - } - window.history.replaceState(null, "", value ? `?${urlSearchParams.toString()}` : window.location.pathname) }} onKeyDown={(event) => { if(event.key === "Enter") { @@ -297,10 +333,64 @@ const AppGrid = props => { {showSuggestion === true ? - - : null +
+ + Can't find what you're looking for? + +
+ setFormMail(e.target.value)} + /> + setMessage(e.target.value)} + /> +
+ + {formMessage} +
+ : null } - {/* {onlyResults === true ? null : + {onlyResults === true ? null : Search by @@ -309,7 +399,7 @@ const AppGrid = props => { Algolia logo - } */} + }
) } diff --git a/frontend/src/components/Workflowsearch.jsx b/frontend/src/components/Workflowsearch.jsx index 44f77b98..1b342645 100644 --- a/frontend/src/components/Workflowsearch.jsx +++ b/frontend/src/components/Workflowsearch.jsx @@ -10,7 +10,7 @@ import algoliasearch from 'algoliasearch'; import { InstantSearch, connectSearchBox, connectHits } from 'react-instantsearch-dom'; import { Grid, Paper, TextField, ButtonBase, InputAdornment, Typography, Button, Tooltip} from '@mui/material'; -const searchClient = algoliasearch("JNSS5CFDZZ", "33e4e3564f4f060e96e0531957bed552") +const searchClient = algoliasearch("JNSS5CFDZZ", "c8f882473ff42d41158430be09ec2b4e") const WorkflowSearch = props => { const { maxRows, showName, showSuggestion, isMobile, globalUrl, parsedXs, newSelectedApp, setNewSelectedApp, defaultSearch, showSearch, ConfiguredHits, selectAble, } = props const rowHandler = maxRows === undefined || maxRows === null ? 50 : maxRows diff --git a/frontend/src/components/ssoTab.jsx b/frontend/src/components/ssoTab.jsx index 1d019800..c361e1b1 100644 --- a/frontend/src/components/ssoTab.jsx +++ b/frontend/src/components/ssoTab.jsx @@ -1,16 +1,16 @@ import { useEffect, useContext } from "react"; import React from "react"; -import { - Typography, - Switch, - Button, - Tooltip, - TextField, - Grid, +import { + Typography, + Switch, + Button, + Tooltip, + TextField, + Grid, Checkbox } from "@mui/material"; import { makeStyles } from "@mui/styles"; -import { Link, useSearchParams } from "react-router-dom"; +import { Link } from "react-router-dom"; import theme from "../theme.jsx"; import { toast } from "react-toastify"; import { Context } from "../context/ContextApi.jsx"; @@ -28,13 +28,7 @@ const SSOTab = ({selectedOrganization, userdata, isEditOrgTab, globalUrl, handle // Check if user is admin const isAdmin = userdata?.active_org?.role === "admin" || userdata?.support === true; - - // Read region_url override from URL params (only allow shuffler.io domains) - const [searchParams] = useSearchParams(); - const rawRegionUrl = searchParams.get("region_url"); - const regionUrlOverride = rawRegionUrl && rawRegionUrl.includes("shuffler.io") ? rawRegionUrl : null; - const effectiveGlobalUrl = regionUrlOverride || globalUrl; - + // State for tracking user SSO connection status const [users, setUsers] = React.useState([]); const [userSSOConnected, setUserSSOConnected] = React.useState(false); @@ -115,7 +109,7 @@ const SSOTab = ({selectedOrganization, userdata, isEditOrgTab, globalUrl, handle // Function to fetch users and check current user's SSO status const checkUserSSOStatus = () => { setCheckingSSOStatus(true); - fetch(effectiveGlobalUrl + "/api/v1/getusers", { + fetch(globalUrl + "/api/v1/getusers", { method: "GET", headers: { "Content-Type": "application/json", @@ -315,7 +309,7 @@ const SSOTab = ({selectedOrganization, userdata, isEditOrgTab, globalUrl, handle }; const HandleTestSSO = () => { - const url = `${effectiveGlobalUrl}/api/v1/orgs/${selectedOrganization?.id}/change`; + const url = `${globalUrl}/api/v1/orgs/${selectedOrganization?.id}/change`; const data = { org_id: selectedOrganization?.id, sso: true, @@ -372,7 +366,7 @@ const SSOTab = ({selectedOrganization, userdata, isEditOrgTab, globalUrl, handle }; const HandleDisconnectSSO = () => { - const url = `${effectiveGlobalUrl}/api/v1/disconnect_sso`; + const url = `${globalUrl}/api/v1/disconnect_sso`; const data = { org_id: selectedOrganization?.id, }; @@ -450,11 +444,6 @@ const SSOTab = ({selectedOrganization, userdata, isEditOrgTab, globalUrl, handle : "Connect your account with this org's SSO!" } - {regionUrlOverride && ( - - Using region override: {regionUrlOverride} - - )} - createTheme({ +export const getTheme = (themeMode, brandColor) => { + // Handle "system" mode by checking user's system preference + let resolvedMode = themeMode; + if (themeMode === "system" || !themeMode) { + resolvedMode = window?.matchMedia?.("(prefers-color-scheme: dark)")?.matches ? "dark" : "light"; + } + // Ensure mode is only "dark" or "light" + if (resolvedMode !== "dark" && resolvedMode !== "light") { + resolvedMode = "dark"; + } + + return createTheme({ palette: { - mode: themeMode, + mode: resolvedMode, main: brandColor || "#FF8544", primary: { main: brandColor || "#FF8544", @@ -167,36 +177,35 @@ export const getTheme = (themeMode, brandColor) => contrastText:"#000000", }, text: { - primary: themeMode === "dark" ? "#ffffff" : "#1A1A1A", - secondary: themeMode === "dark" ? "#9E9E9E" : "#616161", + primary: resolvedMode === "dark" ? "#ffffff" : "#1A1A1A", + secondary: resolvedMode === "dark" ? "#9E9E9E" : "#616161", }, - type: themeMode, - inputColor: themeMode === "dark" ? "rgba(39,41,45,1)" : "rgba(245, 245, 245, 1)", - textColor: themeMode === "dark" ? "#F1F1F1" : "#1A1A1A", - textPrimary: themeMode === "dark" ? "rgba(255, 255, 255, 0.8)" : "rgba(26, 26, 26, 0.8)", - surfaceColor: themeMode === "dark" ? "#27292d" : "#EFEFEF", - platformColor: themeMode === "dark" ? "#212121" : "#ffffff", - backgroundColor: themeMode === "dark" ? "#1a1a1a" : "#f1f1f1", - cytoscapeBackgroundColor: themeMode === "dark" ? "#161616" : "#f5f5f5", - distributionColor: themeMode === "dark" ? "#40E0D0" : "#008080", - cardBackgroundColor: themeMode === "dark" ? "#1e1e1e" : "#eaeaea", - cardHoverColor: themeMode === "dark" ? "#323232" : "#F0F0F0", - hoverColor: themeMode === "dark" ? "#323232" : "#D6D6D6", - usecaseCardColor: themeMode === "dark" ? "#2f2f2f" : "rgba(245, 245, 245, 1)", - usecaseCardHoverColor: themeMode === "dark" ? "#2F2F2F" : "rgba(245, 245, 245, 1)", - usecaseDialogFieldColor: themeMode === "dark" ? "#2B2B2B" : "#F5F5F5", - accentColor: themeMode === "dark" ? "#ff8544" : "#ff8544", - green: themeMode === "dark" ? "#5cc879" : "#008000", - defaultBorder: themeMode === "dark" ? '1px solid #494949' : '1px solid #CCCCCC', + type: resolvedMode, + inputColor: resolvedMode === "dark" ? "rgba(39,41,45,1)" : "rgba(245, 245, 245, 1)", + textColor: resolvedMode === "dark" ? "#F1F1F1" : "#1A1A1A", + textPrimary: resolvedMode === "dark" ? "rgba(255, 255, 255, 0.8)" : "rgba(26, 26, 26, 0.8)", + surfaceColor: resolvedMode === "dark" ? "#27292d" : "#EFEFEF", + platformColor: resolvedMode === "dark" ? "#212121" : "#ffffff", + backgroundColor: resolvedMode === "dark" ? "#1a1a1a" : "#f1f1f1", + cytoscapeBackgroundColor: resolvedMode === "dark" ? "#161616" : "#f5f5f5", + distributionColor: resolvedMode === "dark" ? "#40E0D0" : "#008080", + cardBackgroundColor: resolvedMode === "dark" ? "#1e1e1e" : "#eaeaea", + cardHoverColor: resolvedMode === "dark" ? "#323232" : "#F0F0F0", + hoverColor: resolvedMode === "dark" ? "#323232" : "#D6D6D6", + usecaseCardColor: resolvedMode === "dark" ? "#2f2f2f" : "rgba(245, 245, 245, 1)", + usecaseCardHoverColor: resolvedMode === "dark" ? "#2F2F2F" : "rgba(245, 245, 245, 1)", + usecaseDialogFieldColor: resolvedMode === "dark" ? "#2B2B2B" : "#F5F5F5", + accentColor: resolvedMode === "dark" ? "#ff8544" : "#ff8544", + green: resolvedMode === "dark" ? "#5cc879" : "#008000", + defaultBorder: resolvedMode === "dark" ? '1px solid #494949' : '1px solid #CCCCCC', linkColor: brandColor === "#ff8544" ? "#f86a3e" : brandColor, - slateGrayColor: themeMode === "dark" ? "#494949" : "#CCCCCC", - parsedAppPaperColor: themeMode === "dark" ? "#2f2f2f" : "#CCCCCC", - welcomeCardSubtextColor: themeMode === "dark" ? "#C8C8C8" : "#2f2f2f", - deleteColor: themeMode === "dark" ? "#FD4C62" : "#d32f2f", + slateGrayColor: resolvedMode === "dark" ? "#494949" : "#CCCCCC", + parsedAppPaperColor: resolvedMode === "dark" ? "#2f2f2f" : "#CCCCCC", + borderRadius: 10, - loaderColor: themeMode === "dark" ? "#1a1a1a" : "#E0E0E0", + loaderColor: resolvedMode === "dark" ? "#1a1a1a" : "#E0E0E0", jsonIconStyle: "round", - jsonTheme: themeMode === "dark" ? "summerfruit" : { + jsonTheme: resolvedMode === "dark" ? "summerfruit" : { base00: "#ffffff", // background base01: "#f0f0f0", // very light grey base02: "#f5f5f5", // light grey @@ -216,11 +225,11 @@ export const getTheme = (themeMode, brandColor) => }, jsonCollapseStringsAfterLength: 100, drawer: { - backgroundColor: themeMode === "dark" ? "#262626" : "#f9f9f9" + backgroundColor: resolvedMode === "dark" ? "#262626" : "#f9f9f9" }, actionSidebarField: { - backgroundColor: themeMode === "dark" ? "#2F2F2F" : "#F1F1F1", - color: themeMode === "dark" ? "#ffffff" : "#000000", + backgroundColor: resolvedMode === "dark" ? "#2F2F2F" : "#F1F1F1", + color: resolvedMode === "dark" ? "#ffffff" : "#000000", borderRadius: 8, height: 40, border: "none", @@ -229,53 +238,53 @@ export const getTheme = (themeMode, brandColor) => padding: 5, width: "98%", borderRadius: 5, - border: themeMode === "dark" ? "1px solid rgba(255,255,255,0.7)" : "1px solid rgba(0,0,0,0.3)", - backgroundColor: themeMode === "dark" + border: resolvedMode === "dark" ? "1px solid rgba(255,255,255,0.7)" : "1px solid rgba(0,0,0,0.3)", + backgroundColor: resolvedMode === "dark" ? "#1A1A1A" : "#f1f1f1", - color: themeMode === "dark" + color: resolvedMode === "dark" ? "#F1F1F1" : "#1A1A1A", overflowX: "auto", }, textFieldStyle: { - backgroundColor: themeMode === "dark" ? "#212121" : "#FFFFFF", - color: themeMode === "dark" ? "#ffffff" : "#000000", + backgroundColor: resolvedMode === "dark" ? "#212121" : "#FFFFFF", + color: resolvedMode === "dark" ? "#ffffff" : "#000000", borderRadius: "5px", height: 40, - border: themeMode === "dark" ? "1px solid #4D4D4D" : "1px solid #E0E0E0", + border: resolvedMode === "dark" ? "1px solid #4D4D4D" : "1px solid #E0E0E0", }, DialogStyle: { - backgroundColor: themeMode === "dark" ? "#212121" : "#ffffff", + backgroundColor: resolvedMode === "dark" ? "#212121" : "#ffffff", borderRadius: 2, - boxShadow: themeMode === "dark" ? "0px 0px 10px 0px rgba(0,0,0,0.75)" : "0px 0px 10px 0px rgba(0,0,0,0.2)", - border: themeMode === "dark" ? "1px solid #494949" : "1px solid #cccccc", + boxShadow: resolvedMode === "dark" ? "0px 0px 10px 0px rgba(0,0,0,0.75)" : "0px 0px 10px 0px rgba(0,0,0,0.2)", + border: resolvedMode === "dark" ? "1px solid #494949" : "1px solid #cccccc", }, innerTextfieldStyle: { height: 40, fontSize: 16, - backgroundColor: themeMode === "dark" ? "#212121" : "#f5f5f5", + backgroundColor: resolvedMode === "dark" ? "#212121" : "#f5f5f5", }, tooltip: { - backgroundColor: themeMode === "dark" ? "#212121" : "#ffffff", - color: themeMode === "dark" ? "#ffffff" : "#000000", - border: themeMode === "dark" ? "1px solid #494949" : "1px solid #cccccc", + backgroundColor: resolvedMode === "dark" ? "#212121" : "#ffffff", + color: resolvedMode === "dark" ? "#ffffff" : "#000000", + border: resolvedMode === "dark" ? "1px solid #494949" : "1px solid #cccccc", }, chipStyle: { - backgroundColor: themeMode === "dark" ? "#333333" : "#F5F5F5", - borderColor: themeMode === "dark" ? "#444444" : "#E0E0E0", - color: themeMode === "dark" ? "#FFFFFF" : "#333333", + backgroundColor: resolvedMode === "dark" ? "#333333" : "#F5F5F5", + borderColor: resolvedMode === "dark" ? "#444444" : "#E0E0E0", + color: resolvedMode === "dark" ? "#FFFFFF" : "#333333", }, defaultImage: "/images/no_image.png", singulOrange: "/images/singul_orange.png", singulGreen: "/images/singul_green.png", singulBlackWhite: "/icons/workflow-page/shuffle_agent.png", - scrollbarColor: themeMode === "dark" ? "#494949 #2f2f2f": "#c1c1c1 #f1f1f1", - scrollbarColorTransparent: themeMode === "dark" ? '#494949 transparent': "#c1c1c1 transparent", + scrollbarColor: resolvedMode === "dark" ? "#494949 #2f2f2f": "#c1c1c1 #f1f1f1", + scrollbarColorTransparent: resolvedMode === "dark" ? '#494949 transparent': "#c1c1c1 transparent", }, typography: { fontFamily: `"inter", "Roboto", "Helvetica", "Arial", sans-serif`, - color: themeMode === "dark" ? "#ffffff" : "#000000", + color: resolvedMode === "dark" ? "#ffffff" : "#000000", useNextVariants: true, fontWeightLight: 300, fontWeightRegular: 400, @@ -283,36 +292,36 @@ export const getTheme = (themeMode, brandColor) => fontWeightSemiBold: 600, fontWeightBold: 700, allVariants: { - color: themeMode === "dark" ? "#ffffff" : "#1A1A1A", + color: resolvedMode === "dark" ? "#ffffff" : "#1A1A1A", }, h1: { fontSize: 40, - color: themeMode === "dark" ? "#ffffff" : "#1A1A1A" + color: resolvedMode === "dark" ? "#ffffff" : "#1A1A1A" }, h2: { fontSize: 36, - color: themeMode === "dark" ? "#ffffff" : "#1A1A1A" + color: resolvedMode === "dark" ? "#ffffff" : "#1A1A1A" }, h3: { fontSize: 32, - color: themeMode === "dark" ? "#ffffff" : "#1A1A1A" + color: resolvedMode === "dark" ? "#ffffff" : "#1A1A1A" }, h4: { fontSize: 30, fontWeight: 500, - color: themeMode === "dark" ? "#ffffff" : "#1A1A1A" + color: resolvedMode === "dark" ? "#ffffff" : "#1A1A1A" }, h6: { fontSize: 22, - color: themeMode === "dark" ? "#ffffff" : "#1A1A1A" + color: resolvedMode === "dark" ? "#ffffff" : "#1A1A1A" }, body1: { fontSize: 16, - color: themeMode === "dark" ? "#ffffff" : "#1A1A1A" + color: resolvedMode === "dark" ? "#ffffff" : "#1A1A1A" }, body2: { fontSize: 14, - color: themeMode === "dark" ? "#ffffff" : "#1A1A1A" + color: resolvedMode === "dark" ? "#ffffff" : "#1A1A1A" }, }, components: { @@ -327,7 +336,7 @@ export const getTheme = (themeMode, brandColor) => { props: { variant: 'text', color: 'primary' }, style: { - color: themeMode === "dark" ? "#ffffff" : "#1A1A1A", + color: resolvedMode === "dark" ? "#ffffff" : "#1A1A1A", whiteSpace: "nowrap", textWrap: "normal", }, @@ -335,7 +344,7 @@ export const getTheme = (themeMode, brandColor) => { props: { variant: 'text', color: 'secondary' }, style: { - color: themeMode === "dark" ? "#9E9E9E" : "#616161", + color: resolvedMode === "dark" ? "#9E9E9E" : "#616161", whiteSpace: "nowrap", textWrap: "normal", }, @@ -343,24 +352,24 @@ export const getTheme = (themeMode, brandColor) => { props: { variant: 'contained', color: 'primary' }, style: { - backgroundColor: themeMode === "dark" ? brandColor || '#ff8544' : brandColor || '#FF7C35', - color: themeMode === "dark" ? '#1a1a1a': '#FFFFFF', + backgroundColor: resolvedMode === "dark" ? brandColor || '#ff8544' : brandColor || '#FF7C35', + color: resolvedMode === "dark" ? '#1a1a1a': '#FFFFFF', borderRadius: '4px', whiteSpace: "nowrap", textWrap: "normal", transition: 'background-color 0.2s ease-in-out', '&:hover': { fontWeight: 600, - backgroundColor: themeMode === 'dark' ? brandColor || "#ff955c" : brandColor || '#FF8D4F', - color: themeMode === "dark" ? '#1a1a1a': '#FFFFFF', + backgroundColor: resolvedMode === 'dark' ? brandColor || "#ff955c" : brandColor || '#FF8D4F', + color: resolvedMode === "dark" ? '#1a1a1a': '#FFFFFF', }, }, }, { props: { variant: 'contained', color: 'secondary' }, style: { - backgroundColor: themeMode === "dark" ? '#494949' : '#C9C9C9', - color: themeMode === "dark" ? '#ffffff' : '#4C4C4C', + backgroundColor: resolvedMode === "dark" ? '#494949' : '#C9C9C9', + color: resolvedMode === "dark" ? '#ffffff' : '#4C4C4C', borderRadius: '4px', boxShadow: 'none', whiteSpace: "nowrap", @@ -368,23 +377,23 @@ export const getTheme = (themeMode, brandColor) => textWrap: "normal", '&:hover': { fontWeight: 600, - border: themeMode === "dark" ? '1px solid #f1f1f1' : 'none', - backgroundColor: themeMode === "dark" ? '#494949' : '#C9C9C9', - color: themeMode === "dark" ? '#ffffff' : '#4C4C4C', + border: resolvedMode === "dark" ? '1px solid #f1f1f1' : 'none', + backgroundColor: resolvedMode === "dark" ? '#494949' : '#C9C9C9', + color: resolvedMode === "dark" ? '#ffffff' : '#4C4C4C', }, }, }, { props: { variant: 'outlined', color: 'primary' }, style: { - borderColor: themeMode === "dark" ? brandColor || "#ff8544" : brandColor || "#cc5f1f", - color: themeMode === "dark" ? brandColor || "#ff8544" : brandColor || "#cc5f1f", + borderColor: resolvedMode === "dark" ? brandColor || "#ff8544" : brandColor || "#cc5f1f", + color: resolvedMode === "dark" ? brandColor || "#ff8544" : brandColor || "#cc5f1f", whiteSpace: "nowrap", fontWeight: 'normal', textWrap: "normal", '&:hover': { - backgroundColor: themeMode === "dark" ? brandColor || "#ff8544" : "#ffe8dc", - color: themeMode === "dark" ? "#1a1a1a" : "#8a3d00", + backgroundColor: resolvedMode === "dark" ? brandColor || "#ff8544" : "#ffe8dc", + color: resolvedMode === "dark" ? "#1a1a1a" : "#8a3d00", fontWeight: 600, }, }, @@ -393,14 +402,14 @@ export const getTheme = (themeMode, brandColor) => props: { variant: 'outlined', color: 'secondary' }, style: { border: '1px solid #C5C5C5', - color: themeMode === "dark" ? '#C5C5C5' : '#2D2D2D', + color: resolvedMode === "dark" ? '#C5C5C5' : '#2D2D2D', whiteSpace: "nowrap", textWrap: "normal", '&:hover': { - backgroundColor: themeMode === "dark" ? '#C5C5C5' : '#EFEFEF', - borderColor: themeMode === "dark" ? '#C5C5C5' : '#2D2D2D', + backgroundColor: resolvedMode === "dark" ? '#C5C5C5' : '#EFEFEF', + borderColor: resolvedMode === "dark" ? '#C5C5C5' : '#2D2D2D', fontWeight: 600, - color: themeMode === "dark" ? '#1a1a1a' : '#1A1A1A', + color: resolvedMode === "dark" ? '#1a1a1a' : '#1A1A1A', }, }, }, @@ -423,8 +432,8 @@ export const getTheme = (themeMode, brandColor) => background: 'linear-gradient(90deg, #e6743a 0%, #d4456e 50%, #8a4de8 100%)', }, '&:disabled': { - background: themeMode === "dark" ? '#494949' : '#C9C9C9', - color: themeMode === "dark" ? '#9E9E9E' : '#616161', + background: resolvedMode === "dark" ? '#494949' : '#C9C9C9', + color: resolvedMode === "dark" ? '#9E9E9E' : '#616161', }, }, }, @@ -469,9 +478,9 @@ export const getTheme = (themeMode, brandColor) => }, '&:disabled': { background: 'transparent', - color: themeMode === "dark" ? '#9E9E9E' : '#616161', + color: resolvedMode === "dark" ? '#9E9E9E' : '#616161', '&::before': { - background: themeMode === "dark" ? '#494949' : '#C9C9C9', + background: resolvedMode === "dark" ? '#494949' : '#C9C9C9', }, }, }, @@ -481,7 +490,7 @@ export const getTheme = (themeMode, brandColor) => MuiTab: { styleOverrides: { root: { - color: themeMode === "dark" ? "#C5C5C5" : "#1A1A1A", + color: resolvedMode === "dark" ? "#C5C5C5" : "#1A1A1A", }, }, }, @@ -490,7 +499,7 @@ export const getTheme = (themeMode, brandColor) => overrides: { MuiMenu: { list: { - backgroundColor: themeMode === "dark" ? "#27292d" : "#ffffff", + backgroundColor: resolvedMode === "dark" ? "#27292d" : "#ffffff", }, }, MuiCssBaseline: { @@ -536,4 +545,5 @@ export const getTheme = (themeMode, brandColor) => }, }, }); +} diff --git a/frontend/src/views/AgentUI.jsx b/frontend/src/views/AgentUI.jsx index 7e3ccc3f..6b2fbbd9 100644 --- a/frontend/src/views/AgentUI.jsx +++ b/frontend/src/views/AgentUI.jsx @@ -44,7 +44,6 @@ import { Add as AddIcon, Warning as WarningIcon, Pause as PauseIcon, - Chat as ChatIcon, } from '@mui/icons-material' import { @@ -72,7 +71,6 @@ const AgentUI = (props) => { const [newSelectedApp, setNewSelectedApp] = React.useState({}) const [appPickerAnchor, setAppPickerAnchor] = React.useState(null) const [chosenApps, setChosenApps] = useState([]) - const [planningEnabled, setPlanningEnabled] = useState([]) const activateApp = (appId) => { if (appId === undefined || appId === null || appId === "") { @@ -190,11 +188,11 @@ const AgentUI = (props) => { const agentWrapperStyle = { width: "100%", - minHeight: "100vh", + maxHeight: "100vh", margin: "auto", backgroundColor: theme.palette.backgroundColor, - paddingBottom: showAgentStarter ? 0 : 50, + paddingBottom: showAgentStarter ? 0 : 1500, } @@ -250,25 +248,21 @@ const AgentUI = (props) => { return } - // If no node_id provided, look for the AI Agent node if (node_id === undefined || node_id === null || node_id === "") { + // Look for AI agent + /* for (var key in execution_data.results) { const item = execution_data.results[key] - if (item?.action?.app_name === "AI Agent") { - node_id = item?.action?.id - break + if (item?.action?.app_name !== "AI Agent") { + continue } + + node_id = item?.action?.id + break } + */ if (node_id === undefined || node_id === null || node_id === "") { - // Fallback: if only one result, use it - if (execution_data?.results?.length === 1) { - setAgentActionResult(execution_data.results[0]) - const validatedData = validateJson(execution_data.results[0].result) - if (validatedData.valid) { - setData(validatedData.result) - } - } return } } @@ -276,11 +270,6 @@ const AgentUI = (props) => { var found = false for (var key in execution_data.results) { const item = execution_data.results[key] - - if (item?.action?.app_name !== "AI Agent") { - continue - } - if (item?.action?.id !== node_id) { continue } @@ -300,6 +289,16 @@ const AgentUI = (props) => { if (found === false) { toast.warn("Failed to find the relevant AI Agent result") + + if (execution_data?.results?.length === 1) { + setAgentActionResult(execution_data.results[0]) + const validatedData = validateJson(execution_data.results[0].result) + if (validatedData.valid) { + setData(validatedData.result) + } else { + toast.warn("Action output result is not valid JSON!") + } + } } } @@ -476,7 +475,7 @@ const AgentUI = (props) => { getAppAuth() }, []) - const maxTimelineWidth = 275 + const maxTimelineWidth = 375 const submitQuestions = (decisionId, questionAnswers, isContinuation) => { console.log("Submitting questions: ", decisionId, questionAnswers) @@ -522,12 +521,8 @@ const AgentUI = (props) => { const executionId = params.get("execution_id") const nodeId = params.get("node_id") const authorization = params.get("authorization") - const workflowIdParam = params.get("workflow_id") - // workflow_id param > execution.workflow.id > executionId - const foundWorkflowId = workflowIdParam !== undefined && workflowIdParam !== null && workflowIdParam !== "" ? workflowIdParam : execution?.workflow?.id !== undefined && execution?.workflow?.id !== null && execution?.workflow?.id !== "" ? execution.workflow.id : executionId - - const url = `${globalUrl}/api/v1/workflows/${foundWorkflowId}/run?reference_execution=${executionId}&authorization=${authorization}&answer=true¬e=${encodeURIComponent(JSON.stringify(newArgument))}&agentic=true&decision_id=${decisionId}&node_id=${nodeId}` + const url = `${globalUrl}/api/v1/workflows/${executionId}/run?reference_execution=${executionId}&authorization=${authorization}&answer=true¬e=${encodeURIComponent(JSON.stringify(newArgument))}&agentic=true&decision_id=${decisionId}` fetch(url, { method: "GET", credentials: "include", @@ -898,32 +893,15 @@ const AgentUI = (props) => { />
- - {itemLabel} - + {itemLabel}
{
: null} - {item.category !== "agent" && questions?.length > 0 && (item?.status === "RUNNING" || item?.status === "WAITING") ? + {questions?.length > 0 && item?.status === "RUNNING" || item?.status === "WAITING" ?
{questions.map((q, questionIndex) => { return ( @@ -1211,25 +1189,7 @@ const AgentUI = (props) => { const [continuationText, setContinuationText] = useState("") - // Find the AI Agent result specifically, not just results[0] - var actionResult = null - if (execution?.results?.length > 0) { - for (var key in execution.results) { - const item = execution.results[key] - if (item?.action?.app_name === "AI Agent") { - actionResult = item - break - } - } - - // Fallback to first result if no AI Agent found - if (actionResult === null) { - actionResult = execution.results[0] - } - } else { - actionResult = execution - } - + var actionResult = execution?.results?.length > 0 ? execution.results[0] : execution const validate = validateJson(actionResult?.result) if (validate.valid === true) { actionResult.result = validate.result @@ -1286,11 +1246,6 @@ const AgentUI = (props) => { for (var key in agent_data?.decisions) { const item = agent_data.decisions[key] - if (item.run_details === undefined) { - console.log("Skipping item without run_details:", item) - continue - } - if (item.run_details.started_at === undefined || item.run_details.started_at === null) { item.run_details.started_at = originalStartTime } @@ -1540,7 +1495,6 @@ const AgentUI = (props) => { parsedAction = parsedAction.slice(0, -1) // Remove last comma } - /* const data = { "id": uuid, "name":"agent", @@ -1563,23 +1517,9 @@ const AgentUI = (props) => { "name":"action", "value": parsedAction, } - ], - "planning_mode": planningEnabled, - } + ]} + const url = `${globalUrl}/api/v1/apps/agent_starter/run` - */ - - const data = { - "jsonrpc": "2.0", - "method": "tools/call", - "params": { - "tool_name": parsedAction, - "input": { - "text": inputText, - }, - } - } - const url = `${globalUrl}/api/v1/agent` fetch(url, { method: "POST", body: JSON.stringify(data), @@ -1664,8 +1604,8 @@ const AgentUI = (props) => { return ( -
-
+
+
{ agentRequestLoading ? : - - + + @@ -1742,29 +1678,12 @@ const AgentUI = (props) => { }} /> -
+
- {/* - - - } label="Planning Mode" - style={chipStyle} - variant={planningEnabled ? "contained" : "outlined"} - onClick={() => { - setPlanningEnabled(!planningEnabled) - }} - disabled={true} - /> - - - */} } label="Select Apps / MCPs" style={chipStyle} - variant={"outlined"} onClick={() => { setAppPickerAnchor(document.getElementById("add_app_chip")) }} @@ -1875,21 +1794,18 @@ const AgentUI = (props) => { {chosenApps?.map((app, index) => { return( - + { - if (app?.id !== undefined) { - window.open(`/apps/${app.id}`, '_blank', 'noopener,noreferrer'); - } + window.open(`/apps/${app.id}`, '_blank', 'noopener,noreferrer'); }} style={{ cursor: "pointer", width: 30, height: 30, - backgroundColor: theme.palette.backgroundColor, }} /> diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index f995697f..db69e886 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -231,7 +231,7 @@ export const triggers = [ status: "uninitialized", trigger_type: "SCHEDULE", errors: null, - large_image: "data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iNDgiIGhlaWdodD0iNDgiIHZpZXdCb3g9IjAgMCA0OCA0OCIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPHJlY3Qgd2lkdGg9IjQ4IiBoZWlnaHQ9IjQ4IiByeD0iOCIgZmlsbD0iI0UzQTQxQiIvPgo8cmVjdCB3aWR0aD0iMjQiIGhlaWdodD0iMjQiIHRyYW5zZm9ybT0idHJhbnNsYXRlKDEyIDEyKSIgZmlsbD0iI0UzQTQxQiIvPgo8Y2lyY2xlIGN4PSIyNCIgY3k9IjI0IiByPSI4Ljc1IiBzdHJva2U9IndoaXRlIiBzdHJva2Utd2lkdGg9IjEuNSIvPgo8cGF0aCBkPSJNMjguNSAyNEgyNC4yNUMyNC4xMTE5IDI0IDI0IDIzLjg4ODEgMjQgMjMuNzVWMjAuNSIgc3Ryb2tlPSJ3aGl0ZSIgc3Ryb2tlLXdpZHRoPSIxLjUiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIvPgo8L3N2Zz4=", + large_image: "data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iNDgiIGhlaWdodD0iNDgiIHZpZXdCb3g9IjAgMCA0OCA0OCIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPHJlY3Qgd2lkdGg9IjQ4IiBoZWlnaHQ9IjQ4IiByeD0iOCIgZmlsbD0iIzIxQTBCRCIvPgo8Y2lyY2xlIGN4PSIyNCIgY3k9IjI0IiByPSI4Ljc1IiBzdHJva2U9IndoaXRlIiBzdHJva2Utd2lkdGg9IjEuNSIvPgo8cGF0aCBkPSJNMjguNSAyNEgyNC4yNUMyNC4xMTE5IDI0IDI0IDIzLjg4ODEgMjQgMjMuNzVWMjAuNSIgc3Ryb2tlPSJ3aGl0ZSIgc3Ryb2tlLXdpZHRoPSIxLjUiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIvPgo8L3N2Zz4K", label: "Schedule", is_valid: true, environment: "onprem", @@ -300,11 +300,6 @@ export const triggers = [ "name": "subflow", "example": "", "value": "", - }, - { - "name": "subflow_failure", - "example": "", - "value": "", } ], status: "running", @@ -503,31 +498,6 @@ export function setActionState(actionId, updates, workflowId = null) { } } -// Will use this function to remove the action data when the node will get removed from the cytoscape. -export function removeActionState(actionId, workflowId = null) { - if (!actionId) return; - - try { - const stored = localStorage.getItem(ACTION_STATES_STORAGE_KEY); - if (!stored) return; - - const allStates = JSON.parse(stored); - - if (workflowId && allStates[workflowId]) { - delete allStates[workflowId][actionId]; - - // Clean up empty workflow objects - if (Object.keys(allStates[workflowId]).length === 0) { - delete allStates[workflowId]; - } - } - - localStorage.setItem(ACTION_STATES_STORAGE_KEY, JSON.stringify(allStates)); - } catch (e) { - console.error("Failed to remove action state:", e); - } -} - const splitter = "|~|"; const svgSize = 24; const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); @@ -535,7 +505,7 @@ const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); //const referenceUrl = "https://shuffler.io/functions/webhooks/" //const referenceUrl = window.location.origin+"/api/v1/hooks/" -const searchClient = algoliasearch("JNSS5CFDZZ", "33e4e3564f4f060e96e0531957bed552") +const searchClient = algoliasearch("JNSS5CFDZZ", "c8f882473ff42d41158430be09ec2b4e") const AngularWorkflow = (defaultprops) => { const { globalUrl, setCookie, isLoggedIn, isLoaded, userdata, data_id, ReactGA, } = defaultprops; const {themeMode, supportEmail, brandColor} = useContext(Context) @@ -596,8 +566,6 @@ const AngularWorkflow = (defaultprops) => { const [originalWorkflow, setOriginalWorkflow] = React.useState({}); const [originalSelectedEnvironment, setOriginalSelectedEnvironment] = React.useState({}); const [subworkflow, setSubworkflow] = React.useState({}); - const [subworkflowFailure, setSubworkflowFailure] = React.useState({}); - const [subworkflowFailureStartnode, setSubworkflowFailureStartnode] = React.useState(""); const [subworkflowStartnode, setSubworkflowStartnode] = React.useState(""); const [leftViewOpen, setLeftViewOpen] = React.useState(isMobile ? false : true); const [leftBarSize, setLeftBarSize] = React.useState(isMobile ? 0 : 235) @@ -1162,14 +1130,7 @@ const AngularWorkflow = (defaultprops) => { "description": "Translates your JSON data into a standard formats, then stores it in the Shuffle Datastore", "label": "Translate standard", "example": "{\"source_data\": \"{\\\"event\\\": \\\"login\\\", \\\"user\\\": \\\"john_doe\\\", \\\"timestamp\\\": \\\"2023-10-01T12:00:00Z\\\"}\", \"standard\": \"OCSF\"}", - "parameters": [ - { - "name": "app_name", - "value": "", - "required": true, - "multiline": false, - }, - { + "parameters": [{ "name": "source_data", "value": "", "required": true, @@ -1189,14 +1150,7 @@ const AngularWorkflow = (defaultprops) => { "name": "Cases", "description": "Available actions for case management", "label": "Cases", - "parameters": [ - { - "name": "app_name", - "value": "", - "required": true, - "multiline": false, - }, - { + "parameters": [{ "name": "action", "value": "list_tickets", "options": [ @@ -1221,14 +1175,7 @@ const AngularWorkflow = (defaultprops) => { "name": "Communication", "description": "Available actions for communication", "label": "Communication", - "parameters": [ - { - "name": "app_name", - "value": "", - "required": true, - "multiline": false, - }, - { + "parameters": [{ "name": "action", "value": "list_messages", "options": [ @@ -1263,8 +1210,7 @@ const AngularWorkflow = (defaultprops) => { "disable_user", "get_identity", "get_asset", - "search_identity", - "list_users", + "search_identity" ], "required": true, }, @@ -2253,26 +2199,6 @@ const AngularWorkflow = (defaultprops) => { } } - if (param.name === "subflow_failure" && param.value !== undefined && param.value !== null && param.value.length > 0) { - if (param.value === workflow?.id) { - setSubworkflowFailure(workflow); - } else { - const sub = responseJson.find((data) => data?.id === param.value); - if (sub !== undefined) { - setSubworkflowFailure(sub); - - // Populate startnode if set - const startnodeParam = trigger.parameters.find((p) => p.name === "subflow_failure_startnode"); - if (startnodeParam && startnodeParam.value && sub.actions) { - const foundAction = sub.actions.find((a) => a?.id === startnodeParam.value); - if (foundAction) { - setSubworkflowFailureStartnode(foundAction); - } - } - } - } - } - if (param.name === "startnode" && param.value !== undefined && param.value !== null) { if (Object.getOwnPropertyNames(baseSubflow).length > 0) { @@ -2418,7 +2344,7 @@ const AngularWorkflow = (defaultprops) => { return } - setExecutionsLoading(true); + setExecutionsLoading(true); var url = `${globalUrl}/api/v2/workflows/${id}/executions` var method = "GET" @@ -2899,7 +2825,6 @@ const AngularWorkflow = (defaultprops) => { stop() return } - //console.log(responseJson) // Loop nodes and find results // Update on every interval? idk @@ -4046,8 +3971,6 @@ const AngularWorkflow = (defaultprops) => { if (actionAppname === appname) { workflow.actions[actionkey].selectedAuthentication = item; workflow.actions[actionkey].authentication_id = item.id; - selectedAction.selectedAuthentication = item; - selectedAction.authentication_id = item.id; appUpdates = true; } } @@ -5272,8 +5195,7 @@ const AngularWorkflow = (defaultprops) => { if (responseJson.public) { - // Delay setting appAuthentication to prevent race condition with graph setup - setTimeout(() => setAppAuthentication([]), 100) + setAppAuthentication([]) setLeftBarSize(300) if (Object.getOwnPropertyNames(creatorProfile).length === 0) { @@ -5664,7 +5586,7 @@ const AngularWorkflow = (defaultprops) => { } ReactDOM.unstable_batchedUpdates(() => { - // setRightSideBarOpen(true); + setRightSideBarOpen(true); setLastSaved(false); /* @@ -6850,7 +6772,7 @@ const AngularWorkflow = (defaultprops) => { } //event.target.unselect(); - // setRightSideBarOpen(true); + setRightSideBarOpen(true); return } else if (data.buttonType === "copy") { @@ -6942,7 +6864,7 @@ const AngularWorkflow = (defaultprops) => { if (sourcenode !== null && sourcenode !== undefined) { const sourcedata = sourcenode.data() - if (sourcedata?.trigger_type !== "SUBFLOW" && sourcedata?.trigger_type !== "USERINPUT") { + if (sourcedata.trigger_type !== "SUBFLOW" && sourcedata.trigger_type !== "USERINPUT") { continue } @@ -7207,12 +7129,12 @@ const AngularWorkflow = (defaultprops) => { const tmpAuth = JSON.parse(JSON.stringify(newAppAuth)); - const curappName = curapp.name.toLowerCase().replaceAll(" ", "_") + const curappName = curapp.name.toLowerCase() for (let tmpAuthKey in tmpAuth) { var item = tmpAuth[tmpAuthKey]; const newfields = {}; - if (item.app.name.toLowerCase().replaceAll(" ", "_") !== curappName) { + if (item.app.name.toLowerCase() !== curappName) { continue } @@ -7594,6 +7516,7 @@ const AngularWorkflow = (defaultprops) => { setSelectedTriggerIndex(trigger_index) setSelectedTrigger(data) + //setSelectedActionEnvironment(data.env) }, 25) } else if (data.type === "COMMENT") { if (selectedNodes?.length > 1) { @@ -7906,7 +7829,7 @@ const AngularWorkflow = (defaultprops) => { continue } - const paramname = param.name?.toLowerCase()?.trim()?.replaceAll("_", " "); + const paramname = param.name.toLowerCase().trim().replaceAll("_", " "); const foundresult = GetParamMatch(paramname, exampledata, ""); if (foundresult.length > 0) { @@ -7943,7 +7866,10 @@ const AngularWorkflow = (defaultprops) => { continue } - const paramname = param.name?.toLowerCase()?.trim()?.replaceAll("_", " "); + const paramname = param.name + .toLowerCase() + .trim() + .replaceAll("_", " "); const foundresult = GetParamMatch(paramname, exampledata, ""); if (foundresult.length > 0) { @@ -8243,11 +8169,11 @@ const AngularWorkflow = (defaultprops) => { if (workflow.branches[branchkey].destination_id === edge.target && workflow.branches[branchkey].source_id === edge.source) { console.log("That branch already exists: ", workflow.branches[branchkey]) - //const foundbranch = cy.getElementById(workflow.branches[branchkey].id) - const foundbranch = cy.getElementById(edge.id) + const foundbranch = cy.getElementById(workflow.branches[branchkey].id) if (foundbranch !== undefined && foundbranch !== null && foundbranch.data() !== undefined && foundbranch.data() !== null) { console.log("Removing branch: ", foundbranch.data()) - //event.target.remove() + + event.target.remove() found = true break @@ -8619,10 +8545,6 @@ const AngularWorkflow = (defaultprops) => { workflow.actions = workflow.actions.filter((a) => a.id !== data.id); workflow.triggers = workflow.triggers.filter((a) => a.id !== data.id); - - // Clean up action state from localStorage - removeActionState(data.id, workflow.id); - if (workflow.start === data.id && workflow.actions.length > 0) { // FIXME - should check branches connected to startnode, as picking random // is just confusing @@ -8732,7 +8654,7 @@ const AngularWorkflow = (defaultprops) => { if ((event.ctrlKey || event.metaKey) && !event.shiftKey) { // If any modal/sidebar is open, let browser handle normal copy - if (isAnyModalOrSidebarOpen || event.target?.closest('.MuiDialog-root, .MuiModal-root, [role="dialog"]')) { + if (isAnyModalOrSidebarOpen) { return } @@ -10183,8 +10105,8 @@ const AngularWorkflow = (defaultprops) => { // Calculates how a branch should curve (it's still weird~) // https://codepen.io/guillaumethomas/pen/xxbbBKO const calculateEdgeCurve = (sourcenodePosition, destinationnodePosition) => { - const xParsed = destinationnodePosition?.x - sourcenodePosition?.x - const yParsed = destinationnodePosition?.y - sourcenodePosition?.y + const xParsed = destinationnodePosition.x - sourcenodePosition.x + const yParsed = destinationnodePosition.y - sourcenodePosition.y const z = Math.sqrt(xParsed * xParsed + yParsed * yParsed) const costheta = xParsed / z @@ -10351,7 +10273,7 @@ const AngularWorkflow = (defaultprops) => { action.iconBackground = iconInfo.iconBackgroundColor action.fillstyle = "linear-gradient" } - } else if(!action.isStartNode) { + }else if(!action.isStartNode) { // This is to round the corners of the image // If action has no large_image (e.g. imported/synced workflow where it was stripped), // inject it from the available apps in the sidebar @@ -10361,7 +10283,6 @@ const AngularWorkflow = (defaultprops) => { apps.find((a) => a.name === action.app_name) imageSource = (foundApp && foundApp.large_image) ? foundApp.large_image : "" } - const originalBase64 = imageSource !== "" ? imageSource : theme.palette.defaultImage const roundedImage = await roundBase64Image(originalBase64, 16); action = {...action, large_image: roundedImage} @@ -11503,10 +11424,10 @@ const AngularWorkflow = (defaultprops) => { // No matter what, it's being stopped. if (!responseJson.success) { if (responseJson.reason !== undefined) { - toast.warn("Failed to stop schedule: " + responseJson.reason); + toast("Failed to stop schedule: " + responseJson.reason); } } else { - toast.success("Successfully stopped schedule"); + toast("Successfully stopped schedule"); } if (triggerindex !== undefined && triggerindex !== null && triggerindex >= 0) { @@ -13240,7 +13161,7 @@ const AngularWorkflow = (defaultprops) => { if (queryID !== undefined && queryID !== null) { aa('init', { appId: "JNSS5CFDZZ", - apiKey: "33e4e3564f4f060e96e0531957bed552", + apiKey: "c8f882473ff42d41158430be09ec2b4e", }) const timestamp = new Date().getTime() @@ -16111,7 +16032,7 @@ const AngularWorkflow = (defaultprops) => { onClick={() => { // Change Direction of the branch target/source const foundBranch = cy.getElementById(selectedEdge.id) - if (foundBranch !== undefined && foundBranch !== null && foundBranch?.length > 0) { + if (foundBranch !== undefined && foundBranch !== null) { const source = foundBranch.data("source") const target = foundBranch.data("target") @@ -19371,236 +19292,6 @@ const AngularWorkflow = (defaultprops) => { /> ) : null} - {workflow?.triggers && - workflow?.triggers[selectedTriggerIndex] && - workflow?.triggers[selectedTriggerIndex].parameters - ? ( -
- On Decline - - Optionally trigger a workflow when the user declines - - {workflows === undefined || - workflows === null || - workflows.length === 0 ? null : ( - option.id === value.id} - getOptionLabel={(option) => { - if (option === undefined || option === null || option.name === undefined || option.name === null) { - return "No Workflow Selected"; - } - const newname = (option.name.charAt(0).toUpperCase() + option.name.substring(1)).replaceAll("_", " "); - return newname; - }} - options={ - [{ - "id": "", - "name": "No Workflow Selected", - }].concat(workflows) - } - fullWidth - onChange={(event, newValue) => { - if (newValue === null || newValue === undefined || newValue.id === undefined) { - return - } - - var failureParamIndex = workflow.triggers[selectedTriggerIndex].parameters.findIndex((param) => param.name === "subflow_failure") - if (failureParamIndex === -1) { - workflow.triggers[selectedTriggerIndex].parameters.push({ - "name": "subflow_failure", - "value": "", - }) - failureParamIndex = workflow.triggers[selectedTriggerIndex].parameters.length - 1 - } - - workflow.triggers[selectedTriggerIndex].parameters[failureParamIndex].value = newValue.id - setSubworkflowFailureStartnode("") - - // Fetch workflow to get actions for startnode selection - if (newValue.id.length > 0 && (newValue.actions === undefined || newValue.actions === null || newValue.actions.length === 0)) { - fetch(`${globalUrl}/api/v1/workflows/${newValue.id}`, { - method: "GET", - headers: { "Content-Type": "application/json" }, - credentials: "include", - }) - .then((resp) => resp.json()) - .then((responseJson) => { - if (responseJson.id !== undefined) { - setSubworkflowFailure(responseJson) - - // Default startnode - const startAction = responseJson.actions?.find((a) => a.id === responseJson.start) - if (startAction) { - setSubworkflowFailureStartnode(startAction) - } - } - }) - .catch((error) => { - console.log("Failed fetching decline workflow: ", error) - }) - } else { - setSubworkflowFailure(newValue) - const startAction = newValue.actions?.find((a) => a.id === newValue.start) - if (startAction) { - setSubworkflowFailureStartnode(startAction) - } - } - - setWorkflow(workflow) - setUpdate(Math.random()) - setLastSaved(false) - event.target.blur() - }} - renderOption={(props, data, state) => { - return ( - - - {data.name} - - ) - }} - renderInput={(params) => { - return ( -
- - {subworkflowFailure === null || subworkflowFailure === undefined || subworkflowFailure?.id === undefined || subworkflowFailure?.id === null || subworkflowFailure?.id.length === 0 ? null : - - - - - - } -
- ); - }} - /> - )} - - {subworkflowFailure?.actions !== undefined && subworkflowFailure?.actions !== null && subworkflowFailure?.actions?.length > 0 ? ( - option.id === value.id} - getOptionLabel={(option) => { - if (option === undefined || option === null || option.label === undefined || option.label === null) { - return "Default"; - } - const newname = (option.label.charAt(0).toUpperCase() + option.label.substring(1)).replaceAll("_", " "); - return newname; - }} - options={subworkflowFailure.actions} - fullWidth - onChange={(event, newValue) => { - setSubworkflowFailureStartnode(newValue) - - var startnodeParamIndex = workflow.triggers[selectedTriggerIndex].parameters.findIndex((param) => param.name === "subflow_failure_startnode") - if (startnodeParamIndex === -1) { - workflow.triggers[selectedTriggerIndex].parameters.push({ - "name": "subflow_failure_startnode", - "value": "", - }) - startnodeParamIndex = workflow.triggers[selectedTriggerIndex].parameters.length - 1 - } - - workflow.triggers[selectedTriggerIndex].parameters[startnodeParamIndex].value = newValue?.id || "" - setWorkflow(workflow) - setUpdate(Math.random()) - setLastSaved(false) - }} - renderOption={(props, action, state) => { - return ( - - {action.label} - - ) - }} - renderInput={(params) => { - return ( - - ); - }} - /> - ) : null} -
- ) : null} -
Required Input-Questions @@ -20287,7 +19978,6 @@ const AngularWorkflow = (defaultprops) => { "&.Mui-selected": { backgroundColor: themeMode === "dark" ? "#1e1e1e" : "#CCCCCC", color: theme.palette.text.primary, - borderRadius: "6px !important", fontWeight: 600, "&:hover": { backgroundColor: themeMode === "dark" ? "rgba(0,0,0,0.3)" : "rgba(0,0,0,0.1)", @@ -20330,6 +20020,7 @@ const AngularWorkflow = (defaultprops) => { justifyContent: "space-between", width: "100%", position: "relative", + minHeight: 80, }}> {/* Left: Workflow Name Container */}
{ }} > {workflow?.name !== undefined && workflow?.name !== null && workflow?.name?.length > 0 ? - + : null } {workflow.name} - {/* Warning Messages */} - {!distributedFromParent || userdata?.support === true ? - isCorrectOrg ? null : - - Warning: { - toast.info("Changing to correct organisation. Please wait a few seconds.") - changeOrg() - }} - >Change Active Organization to edit this Workflow. - - : - - suborgWorkflows?.length === 0 ? - - Warning: This workflow is controlled by your parent org and may not be editable. - - : - null - } - {parentWorkflows === undefined || parentWorkflows === null || parentWorkflows.length === 0 ? null : - - }
{/* Center: Build/Debug Toggle */} @@ -20602,7 +20237,6 @@ const AngularWorkflow = (defaultprops) => { saveWorkflow(workflow, undefined, undefined, e.target.value) /* Standard re-loads */ - setAllTriggers(undefined) setSelectedTriggerIndex(-1) getEnvironments(e.target.value) @@ -20927,7 +20561,7 @@ const AngularWorkflow = (defaultprops) => { id="execution_location" style={{ color: theme.palette.text.primary }} > - Runtime Location ({selectedActionEnvironment?.Name}) + Runtime Location