From 48505261983338d2845134f553d96fbef1fe7939 Mon Sep 17 00:00:00 2001 From: "trusihin.andrey" Date: Tue, 26 Aug 2025 15:09:39 +0300 Subject: [PATCH 01/57] Worker. Add option to set k8s resources. --- functions/onprem/orborus/orborus.go | 75 +++++++++++++++++++++++------ 1 file changed, 59 insertions(+), 16 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 26f16017..d8870c59 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -51,6 +51,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" ) @@ -734,7 +735,7 @@ func deployServiceWorkers(image string) { var updatedNetworks []swarm.NetworkAttachmentConfig for _, net := range serviceSpec.Networks { if net.Target != "shuffle_shuffle" { - updatedNetworks = append(updatedNetworks, net) + updatedNetworks = append(updatedNetworks, net) } } serviceSpec.Networks = updatedNetworks @@ -786,6 +787,48 @@ func buildEnvVars(envMap map[string]string) []corev1.EnvVar { return envVars } +func buildResourcesFromEnv() corev1.ResourceRequirements { + reqs := corev1.ResourceList{} + lims := corev1.ResourceList{} + + type item struct { + env string + rn corev1.ResourceName + to *corev1.ResourceList + } + + items := []item{ + // kubernetes requests + {env: "KUBERNETES_CPU_REQUEST", rn: corev1.ResourceCPU, to: &reqs}, + {env: "KUBERNETES_MEMORY_REQUEST", rn: corev1.ResourceMemory, to: &reqs}, + {env: "KUBERNETES_EPHEMERAL_STORAGE_REQUEST", rn: corev1.ResourceEphemeralStorage, to: &reqs}, + // kubernetes limits + {env: "KUBERNETES_CPU_LIMIT", rn: corev1.ResourceCPU, to: &lims}, + {env: "KUBERNETES_MEMORY_LIMIT", rn: corev1.ResourceMemory, to: &lims}, + {env: "KUBERNETES_EPHEMERAL_STORAGE_LIMIT", rn: corev1.ResourceEphemeralStorage, to: &lims}, + } + + for _, it := range items { + if v := strings.TrimSpace(os.Getenv(it.env)); v != "" { + if q, err := resource.ParseQuantity(v); err == nil { + (*it.to)[it.rn] = q + } else { + log.Printf("[WARN] Cannot parse %s=%q as resource quantity: %v", it.env, v, err) + } + } + } + + rr := corev1.ResourceRequirements{} + if len(reqs) > 0 { + rr.Requests = reqs + } + if len(lims) > 0 { + rr.Limits = lims + } + + return rr +} + func handleBackendImageDownload(ctx context.Context, images string) error { // Replicate images with lowercase, as the name may be wrong @@ -813,20 +856,20 @@ func handleBackendImageDownload(ctx context.Context, images string) error { newImages = append(newImages, curimage) // Force remove the current image to avoid cached layers - // if swarmConfig == "run" || swarmConfig == "swarm" { - // _, err := dockercli.ImageRemove(ctx, curimage, image.RemoveOptions{ - // Force: true, - // PruneChildren: true, - // }) + // if swarmConfig == "run" || swarmConfig == "swarm" { + // _, err := dockercli.ImageRemove(ctx, curimage, image.RemoveOptions{ + // Force: true, + // PruneChildren: true, + // }) - // if err != nil { - // log.Printf("[ERROR] Failed removing image for re-download: %s", err) - // } else { - // log.Printf("[DEBUG] Removed image: %s", curimage) - // } - // } else { - // //log.Printf("[DEBUG] Skipping image removal for %s as swarmConfig is not set to run or swarm. Value: %#v", curimage, swarmConfig) - // } + // if err != nil { + // log.Printf("[ERROR] Failed removing image for re-download: %s", err) + // } else { + // log.Printf("[DEBUG] Removed image: %s", curimage) + // } + // } else { + // //log.Printf("[DEBUG] Skipping image removal for %s as swarmConfig is not set to run or swarm. Value: %#v", curimage, swarmConfig) + // } err := shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, curimage) if err != nil { @@ -887,7 +930,7 @@ func handleBackendImageDownload(ctx context.Context, images string) error { log.Printf("[ERROR] Failed updating service %s with the new image %s: %s. Resp: %#v", service.Spec.Annotations.Name, image, err, resp) } else { log.Printf("[DEBUG] Updated service %s with the new image %s. Resp: %#v", service.Spec.Annotations.Name, image, resp) - + found = true if !strings.Contains(fmt.Sprintf("%s", resp), "error") { @@ -1173,6 +1216,7 @@ func deployK8sWorker(image string, identifier string, env []string) error { Image: kubernetesImage, Env: buildEnvVars(envMap), SecurityContext: containerSecurityContext, + Resources: buildResourcesFromEnv(), //ImagePullPolicy: "Never", ImagePullPolicy: corev1.PullIfNotPresent, @@ -2214,7 +2258,6 @@ func main() { log.Printf("[INFO] Waiting for executions at %s with Environment %#v", fullUrl, environment) - hasStarted := false for { if req.Method == "POST" { From f26f8728356173370de71465c22907e4b2a0fdbb Mon Sep 17 00:00:00 2001 From: "trusihin.andrey" Date: Tue, 26 Aug 2025 16:24:01 +0300 Subject: [PATCH 02/57] Rename k8s resources envs. Add the same options to shuffle app --- functions/onprem/orborus/orborus.go | 40 +++++++++++++++++---- functions/onprem/worker/worker.go | 55 ++++++++++++++++++++++++++--- 2 files changed, 84 insertions(+), 11 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index d8870c59..3103405a 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -799,13 +799,13 @@ func buildResourcesFromEnv() corev1.ResourceRequirements { items := []item{ // kubernetes requests - {env: "KUBERNETES_CPU_REQUEST", rn: corev1.ResourceCPU, to: &reqs}, - {env: "KUBERNETES_MEMORY_REQUEST", rn: corev1.ResourceMemory, to: &reqs}, - {env: "KUBERNETES_EPHEMERAL_STORAGE_REQUEST", rn: corev1.ResourceEphemeralStorage, to: &reqs}, + {env: "SHUFFLE_WORKER_CPU_REQUEST", rn: corev1.ResourceCPU, to: &reqs}, + {env: "SHUFFLE_WORKER_MEMORY_REQUEST", rn: corev1.ResourceMemory, to: &reqs}, + {env: "SHUFFLE_WORKER_EPHEMERAL_STORAGE_REQUEST", rn: corev1.ResourceEphemeralStorage, to: &reqs}, // kubernetes limits - {env: "KUBERNETES_CPU_LIMIT", rn: corev1.ResourceCPU, to: &lims}, - {env: "KUBERNETES_MEMORY_LIMIT", rn: corev1.ResourceMemory, to: &lims}, - {env: "KUBERNETES_EPHEMERAL_STORAGE_LIMIT", rn: corev1.ResourceEphemeralStorage, to: &lims}, + {env: "SHUFFLE_WORKER_CPU_LIMIT", rn: corev1.ResourceCPU, to: &lims}, + {env: "SHUFFLE_WORKER_MEMORY_LIMIT", rn: corev1.ResourceMemory, to: &lims}, + {env: "SHUFFLE_WORKER_EPHEMERAL_STORAGE_LIMIT", rn: corev1.ResourceEphemeralStorage, to: &lims}, } for _, it := range items { @@ -1072,6 +1072,34 @@ func deployK8sWorker(image string, identifier string, env []string) error { env = append(env, fmt.Sprintf("IS_KUBERNETES=true")) env = append(env, fmt.Sprintf("KUBERNETES_NAMESPACE=%s", os.Getenv("KUBERNETES_NAMESPACE"))) + // worker resource env + for _, k := range []string{ + "SHUFFLE_WORKER_CPU_REQUEST", + "SHUFFLE_WORKER_MEMORY_REQUEST", + "SHUFFLE_WORKER_EPHEMERAL_STORAGE_REQUEST", + "SHUFFLE_WORKER_CPU_LIMIT", + "SHUFFLE_WORKER_MEMORY_LIMIT", + "SHUFFLE_WORKER_EPHEMERAL_STORAGE_LIMIT", + } { + if v := os.Getenv(k); v != "" { + env = append(env, fmt.Sprintf("%s=%s", k, v)) + } + } + + // app resource env + for _, k := range []string{ + "SHUFFLE_APP_CPU_REQUEST", + "SHUFFLE_APP_MEMORY_REQUEST", + "SHUFFLE_APP_EPHEMERAL_STORAGE_REQUEST", + "SHUFFLE_APP_CPU_LIMIT", + "SHUFFLE_APP_MEMORY_LIMIT", + "SHUFFLE_APP_EPHEMERAL_STORAGE_LIMIT", + } { + if v := os.Getenv(k); v != "" { + env = append(env, fmt.Sprintf("%s=%s", k, v)) + } + } + if len(os.Getenv("KUBERNETES_SERVICE_HOST")) > 0 { env = append(env, fmt.Sprintf("KUBERNETES_SERVICE_HOST=%s", os.Getenv("KUBERNETES_SERVICE_HOST"))) } diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 2fbf51e3..133545f3 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -42,6 +42,7 @@ import ( //k8s deps appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes" @@ -639,6 +640,7 @@ func deployk8sApp(image string, identifier string, env []string) error { }, }, SecurityContext: containerSecurityContext, + Resources: buildResourcesFromEnv(), }, }, DNSPolicy: corev1.DNSClusterFirst, @@ -2375,6 +2377,49 @@ func buildEnvVars(envMap map[string]string) []corev1.EnvVar { } return envVars } + +func buildResourcesFromEnv() corev1.ResourceRequirements { + reqs := corev1.ResourceList{} + lims := corev1.ResourceList{} + + type item struct { + env string + rn corev1.ResourceName + to *corev1.ResourceList + } + + items := []item{ + // kubernetes requests + {env: "SHUFFLE_APP_CPU_REQUEST", rn: corev1.ResourceCPU, to: &reqs}, + {env: "SHUFFLE_APP_MEMORY_REQUEST", rn: corev1.ResourceMemory, to: &reqs}, + {env: "SHUFFLE_APP_EPHEMERAL_STORAGE_REQUEST", rn: corev1.ResourceEphemeralStorage, to: &reqs}, + // kubernetes limits + {env: "SHUFFLE_APP_CPU_LIMIT", rn: corev1.ResourceCPU, to: &lims}, + {env: "SHUFFLE_APP_MEMORY_LIMIT", rn: corev1.ResourceMemory, to: &lims}, + {env: "SHUFFLE_APP_EPHEMERAL_STORAGE_LIMIT", rn: corev1.ResourceEphemeralStorage, to: &lims}, + } + + for _, it := range items { + if v := strings.TrimSpace(os.Getenv(it.env)); v != "" { + if q, err := resource.ParseQuantity(v); err == nil { + (*it.to)[it.rn] = q + } else { + log.Printf("[WARN] Cannot parse %s=%q as resource quantity: %v", it.env, v, err) + } + } + } + + rr := corev1.ResourceRequirements{} + if len(reqs) > 0 { + rr.Requests = reqs + } + if len(lims) > 0 { + rr.Limits = lims + } + + return rr +} + func getWorkerBackendExecution(auth string, executionId string) (*shuffle.WorkflowExecution, error) { backendUrl := os.Getenv("BASE_URL") if len(backendUrl) == 0 { @@ -2385,9 +2430,9 @@ func getWorkerBackendExecution(auth string, executionId string) (*shuffle.Workfl streamResultUrl := fmt.Sprintf("%s/api/v1/streams/results", backendUrl) topClient := shuffle.GetExternalClient(backendUrl) - requestData := shuffle.ActionResult { + requestData := shuffle.ActionResult{ Authorization: auth, - ExecutionId: executionId, + ExecutionId: executionId, } data, err := json.Marshal(requestData) @@ -2647,7 +2692,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } if setExecution || workflowExecution.Status == "FINISHED" || workflowExecution.Status == "ABORTED" || workflowExecution.Status == "FAILURE" { - if debug { + if debug { log.Printf("[DEBUG][%s] Running setexec with status %s and %d/%d results", workflowExecution.ExecutionId, workflowExecution.Status, len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)) } @@ -2665,7 +2710,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" || os.Getenv("SHUFFLE_SWARM_CONFIG") == "swarm" { finished := shuffle.ValidateFinished(ctx, -1, *workflowExecution) if !finished { - if debug { + if debug { log.Printf("[DEBUG][%s] Handling next node since it's not finished!", workflowExecution.ExecutionId) } @@ -3630,7 +3675,7 @@ func sendAppRequest(ctx context.Context, incomingUrl, appName string, port int, log.Printf("[ERROR] Failed reading app request body body: %s", err) return err } else { - if debug { + if debug { log.Printf("[DEBUG][%s] NEWRESP (from app): %s", workflowExecution.ExecutionId, string(body)) } } From dd89bbdd33f70625a0314a549c4169927d8b187a Mon Sep 17 00:00:00 2001 From: "trusihin.andrey" Date: Tue, 26 Aug 2025 18:10:12 +0300 Subject: [PATCH 03/57] Remove SHUFFLE_WORKER_* env from orborus.go --- functions/onprem/orborus/orborus.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 3103405a..6d0d73ea 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -1072,20 +1072,6 @@ func deployK8sWorker(image string, identifier string, env []string) error { env = append(env, fmt.Sprintf("IS_KUBERNETES=true")) env = append(env, fmt.Sprintf("KUBERNETES_NAMESPACE=%s", os.Getenv("KUBERNETES_NAMESPACE"))) - // worker resource env - for _, k := range []string{ - "SHUFFLE_WORKER_CPU_REQUEST", - "SHUFFLE_WORKER_MEMORY_REQUEST", - "SHUFFLE_WORKER_EPHEMERAL_STORAGE_REQUEST", - "SHUFFLE_WORKER_CPU_LIMIT", - "SHUFFLE_WORKER_MEMORY_LIMIT", - "SHUFFLE_WORKER_EPHEMERAL_STORAGE_LIMIT", - } { - if v := os.Getenv(k); v != "" { - env = append(env, fmt.Sprintf("%s=%s", k, v)) - } - } - // app resource env for _, k := range []string{ "SHUFFLE_APP_CPU_REQUEST", From b6e90050371ef06b7c8ff95d93ef02f76a737a78 Mon Sep 17 00:00:00 2001 From: "trusihin.andrey" Date: Wed, 27 Aug 2025 09:26:47 +0300 Subject: [PATCH 04/57] Fix variables name --- functions/onprem/orborus/orborus.go | 38 ++++++++++++++--------------- functions/onprem/worker/worker.go | 38 ++++++++++++++--------------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 6d0d73ea..7e212505 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -788,42 +788,42 @@ func buildEnvVars(envMap map[string]string) []corev1.EnvVar { } func buildResourcesFromEnv() corev1.ResourceRequirements { - reqs := corev1.ResourceList{} - lims := corev1.ResourceList{} + requests := corev1.ResourceList{} + limits := corev1.ResourceList{} type item struct { - env string - rn corev1.ResourceName - to *corev1.ResourceList + env string + resourceName corev1.ResourceName + resourceList corev1.ResourceList } items := []item{ // kubernetes requests - {env: "SHUFFLE_WORKER_CPU_REQUEST", rn: corev1.ResourceCPU, to: &reqs}, - {env: "SHUFFLE_WORKER_MEMORY_REQUEST", rn: corev1.ResourceMemory, to: &reqs}, - {env: "SHUFFLE_WORKER_EPHEMERAL_STORAGE_REQUEST", rn: corev1.ResourceEphemeralStorage, to: &reqs}, + {env: "SHUFFLE_WORKER_CPU_REQUEST", resourceName: corev1.ResourceCPU, resourceList: requests}, + {env: "SHUFFLE_WORKER_MEMORY_REQUEST", resourceName: corev1.ResourceMemory, resourceList: requests}, + {env: "SHUFFLE_WORKER_EPHEMERAL_STORAGE_REQUEST", resourceName: corev1.ResourceEphemeralStorage, resourceList: requests}, // kubernetes limits - {env: "SHUFFLE_WORKER_CPU_LIMIT", rn: corev1.ResourceCPU, to: &lims}, - {env: "SHUFFLE_WORKER_MEMORY_LIMIT", rn: corev1.ResourceMemory, to: &lims}, - {env: "SHUFFLE_WORKER_EPHEMERAL_STORAGE_LIMIT", rn: corev1.ResourceEphemeralStorage, to: &lims}, + {env: "SHUFFLE_WORKER_CPU_LIMIT", resourceName: corev1.ResourceCPU, resourceList: limits}, + {env: "SHUFFLE_WORKER_MEMORY_LIMIT", resourceName: corev1.ResourceMemory, resourceList: limits}, + {env: "SHUFFLE_WORKER_EPHEMERAL_STORAGE_LIMIT", resourceName: corev1.ResourceEphemeralStorage, resourceList: limits}, } for _, it := range items { - if v := strings.TrimSpace(os.Getenv(it.env)); v != "" { - if q, err := resource.ParseQuantity(v); err == nil { - (*it.to)[it.rn] = q + if value := strings.TrimSpace(os.Getenv(it.env)); value != "" { + if quantity, err := resource.ParseQuantity(value); err == nil { + it.resourceList[it.resourceName] = quantity } else { - log.Printf("[WARN] Cannot parse %s=%q as resource quantity: %v", it.env, v, err) + log.Printf("[WARNING] Cannot parse %s=%q as resource quantity: %v", it.env, value, err) } } } rr := corev1.ResourceRequirements{} - if len(reqs) > 0 { - rr.Requests = reqs + if len(requests) > 0 { + rr.Requests = requests } - if len(lims) > 0 { - rr.Limits = lims + if len(limits) > 0 { + rr.Limits = limits } return rr diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 133545f3..3489666c 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -2379,42 +2379,42 @@ func buildEnvVars(envMap map[string]string) []corev1.EnvVar { } func buildResourcesFromEnv() corev1.ResourceRequirements { - reqs := corev1.ResourceList{} - lims := corev1.ResourceList{} + requests := corev1.ResourceList{} + limits := corev1.ResourceList{} type item struct { - env string - rn corev1.ResourceName - to *corev1.ResourceList + env string + resourceName corev1.ResourceName + resourceList corev1.ResourceList } items := []item{ // kubernetes requests - {env: "SHUFFLE_APP_CPU_REQUEST", rn: corev1.ResourceCPU, to: &reqs}, - {env: "SHUFFLE_APP_MEMORY_REQUEST", rn: corev1.ResourceMemory, to: &reqs}, - {env: "SHUFFLE_APP_EPHEMERAL_STORAGE_REQUEST", rn: corev1.ResourceEphemeralStorage, to: &reqs}, + {env: "SHUFFLE_APP_CPU_REQUEST", resourceName: corev1.ResourceCPU, resourceList: requests}, + {env: "SHUFFLE_APP_MEMORY_REQUEST", resourceName: corev1.ResourceMemory, resourceList: requests}, + {env: "SHUFFLE_APP_EPHEMERAL_STORAGE_REQUEST", resourceName: corev1.ResourceEphemeralStorage, resourceList: requests}, // kubernetes limits - {env: "SHUFFLE_APP_CPU_LIMIT", rn: corev1.ResourceCPU, to: &lims}, - {env: "SHUFFLE_APP_MEMORY_LIMIT", rn: corev1.ResourceMemory, to: &lims}, - {env: "SHUFFLE_APP_EPHEMERAL_STORAGE_LIMIT", rn: corev1.ResourceEphemeralStorage, to: &lims}, + {env: "SHUFFLE_APP_CPU_LIMIT", resourceName: corev1.ResourceCPU, resourceList: limits}, + {env: "SHUFFLE_APP_MEMORY_LIMIT", resourceName: corev1.ResourceMemory, resourceList: limits}, + {env: "SHUFFLE_APP_EPHEMERAL_STORAGE_LIMIT", resourceName: corev1.ResourceEphemeralStorage, resourceList: limits}, } for _, it := range items { - if v := strings.TrimSpace(os.Getenv(it.env)); v != "" { - if q, err := resource.ParseQuantity(v); err == nil { - (*it.to)[it.rn] = q + if value := strings.TrimSpace(os.Getenv(it.env)); value != "" { + if quantity, err := resource.ParseQuantity(value); err == nil { + it.resourceList[it.resourceName] = quantity } else { - log.Printf("[WARN] Cannot parse %s=%q as resource quantity: %v", it.env, v, err) + log.Printf("[WARNING] Cannot parse %s=%q as resource quantity: %v", it.env, value, err) } } } rr := corev1.ResourceRequirements{} - if len(reqs) > 0 { - rr.Requests = reqs + if len(requests) > 0 { + rr.Requests = requests } - if len(lims) > 0 { - rr.Limits = lims + if len(limits) > 0 { + rr.Limits = limits } return rr From cbe8f908bd0b519fde7b8712183960265ad7e577 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Wed, 3 Sep 2025 19:42:56 +0530 Subject: [PATCH 05/57] fix: enabling ppof endpoint over DEBUG_MEMORY env --- functions/onprem/worker/worker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index c6b82823..ea3b748d 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -4772,7 +4772,7 @@ func runWebserver(listener net.Listener) { if strings.ToLower(os.Getenv("SHUFFLE_SWARM_CONFIG")) == "run" || strings.ToLower(os.Getenv("SHUFFLE_APP_REPLICAS")) == "" { // go AutoScaleApps(ctx, dockercli, maxExecutionsPerMinute) } - if strings.ToLower(os.Getenv("SHUFFLE_DEBUG_MEMORY")) == "true" { + if (strings.ToLower(os.Getenv("SHUFFLE_DEBUG_MEMORY")) == "true" || strings.ToLower(os.Getenv("DEBUG_MEMORY")) == "true") { r.HandleFunc("/debug/pprof/", pprof.Index) r.HandleFunc("/debug/pprof/heap", pprof.Handler("heap").ServeHTTP) r.HandleFunc("/debug/pprof/profile", pprof.Profile) From 7612b4b7bba9ccb1151649b0c718f5bcc071cf7c Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Wed, 17 Sep 2025 17:56:57 +0530 Subject: [PATCH 06/57] docs: a few more ideas about auto scale --- functions/onprem/orborus/orborus.go | 7 ++++++- functions/onprem/worker/worker.go | 6 ++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index a6435b65..312ee7f1 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -2183,7 +2183,6 @@ func main() { fullUrl += "?amount=50" } - if isKubernetes == "true" { log.Printf("[INFO] Finished configuring kubernetes environment. Connecting to %s", fullUrl) } else { @@ -2532,6 +2531,8 @@ func main() { executionRequests.Data = executionRequests.Data[0:allowed] } } else if swarmControlMode && (swarmConfig == "run" || swarmConfig == "swarm") { + // any reason it is not maxConcurrency instead of + // hardcoded 50? if len(executionRequests.Data) > 50 { executionRequests.Data = executionRequests.Data[0:50] } @@ -4060,6 +4061,10 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest, image string, return nil } +// 0x0elliot: +// let's never increase worker replicas. +// in our tests, workers replicas mattered a lot less. +// edge-case: subflows are helped with when worker replicas are higher. func AutoScale(ctx context.Context) { if os.Getenv("SHUFFLE_SCALE_REPLICAS") != "" { return diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index ea3b748d..567b52b8 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -4806,6 +4806,12 @@ func runWebserver(listener net.Listener) { } } +// 0x0elliot: +// IF we had to rewrite this, we will focus on ONLY auto scale for apps. +// i recommend we target executions/minute (?) as a metric. +// edge-case: subflows are helped with when worker replicas are higher. +// i kind of never want to scale down. at least, not now. +// also, algorithm is very broken. executions/worker func AutoScaleApps(ctx context.Context, client *dockerclient.Client, maxExecutionsPerMinute int) { ticker := time.NewTicker(1 * time.Second) From 8ce9c2fdbe57d10418fd660268f10fe1a82e8749 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Fri, 19 Sep 2025 21:59:26 +0530 Subject: [PATCH 07/57] fix: adding memory debugging endpoints --- backend/go-app/main.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 79622f95..66577845 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5,6 +5,8 @@ import ( "github.com/shuffle/shuffle-shared" "github.com/shuffle/singul/pkg" + "net/http/pprof" + "archive/zip" "bufio" "bytes" @@ -5506,6 +5508,14 @@ func initHandlers() { r.HandleFunc("/api/v1/dashboards/{key}/widgets", shuffle.HandleNewWidget).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/dashboards/{key}/widgets/{widget_id}", shuffle.HandleGetWidget).Methods("GET", "OPTIONS") + if (strings.ToLower(os.Getenv("SHUFFLE_DEBUG_MEMORY")) == "true" || strings.ToLower(os.Getenv("DEBUG_MEMORY")) == "true") { + r.HandleFunc("/debug/pprof/", pprof.Index) + r.HandleFunc("/debug/pprof/heap", pprof.Handler("heap").ServeHTTP) + r.HandleFunc("/debug/pprof/profile", pprof.Profile) + r.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + r.HandleFunc("/debug/pprof/trace", pprof.Trace) + } + r.Use(shuffle.RequestMiddleware) http.Handle("/", r) } From 962345af936dce3dff37f483fe509da2c07d77af Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Tue, 23 Sep 2025 22:40:41 +0530 Subject: [PATCH 08/57] fix: disabling health in webhook execution --- backend/go-app/main.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 66577845..d8616564 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -2063,7 +2063,7 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { } if len(hook.Workflows) == 1 { - workflow, err := shuffle.GetWorkflow(ctx, hook.Workflows[0]) + workflow, err := shuffle.GetWorkflow(ctx, hook.Workflows[0], true) if err == nil { for _, branch := range workflow.Branches { if branch.SourceID == hook.Id { @@ -5509,11 +5509,14 @@ func initHandlers() { r.HandleFunc("/api/v1/dashboards/{key}/widgets/{widget_id}", shuffle.HandleGetWidget).Methods("GET", "OPTIONS") if (strings.ToLower(os.Getenv("SHUFFLE_DEBUG_MEMORY")) == "true" || strings.ToLower(os.Getenv("DEBUG_MEMORY")) == "true") { + log.Printf("[DEBUG] Memory debugging is enabled on /debug/pprof") r.HandleFunc("/debug/pprof/", pprof.Index) r.HandleFunc("/debug/pprof/heap", pprof.Handler("heap").ServeHTTP) r.HandleFunc("/debug/pprof/profile", pprof.Profile) r.HandleFunc("/debug/pprof/symbol", pprof.Symbol) r.HandleFunc("/debug/pprof/trace", pprof.Trace) + } else { + log.Printf("[DEBUG] Memory debugging is disabled. To enable, set SHUFFLE_DEBUG_MEMORY or DEBUG_MEMORY to true") } r.Use(shuffle.RequestMiddleware) From 5d9b69a45973fddbd99da6f31a943c0b473f2adb Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Wed, 24 Sep 2025 09:26:03 +0200 Subject: [PATCH 09/57] update apt repo for helm --- .github/workflows/helm-release.yml | 4 ++-- .github/workflows/helm-test.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/helm-release.yml b/.github/workflows/helm-release.yml index 4c3c6b6f..69b17917 100644 --- a/.github/workflows/helm-release.yml +++ b/.github/workflows/helm-release.yml @@ -26,9 +26,9 @@ jobs: - name: Install apt dependencies run: | - curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | sudo tee /usr/share/keyrings/helm.gpg > /dev/null sudo apt-get install apt-transport-https -y --no-install-recommends - echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list + curl -fsSL https://packages.buildkite.com/helm-linux/helm-debian/gpgkey | gpg --dearmor | sudo tee /usr/share/keyrings/helm.gpg > /dev/null + echo "deb [signed-by=/usr/share/keyrings/helm.gpg] https://packages.buildkite.com/helm-linux/helm-debian/any/ any main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list sudo apt-get update sudo apt-get install helm -y --no-install-recommends diff --git a/.github/workflows/helm-test.yml b/.github/workflows/helm-test.yml index fb0e34aa..49c20240 100644 --- a/.github/workflows/helm-test.yml +++ b/.github/workflows/helm-test.yml @@ -18,9 +18,9 @@ jobs: - name: Install apt dependencies run: | - curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | sudo tee /usr/share/keyrings/helm.gpg > /dev/null sudo apt-get install apt-transport-https -y --no-install-recommends - echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list + curl -fsSL https://packages.buildkite.com/helm-linux/helm-debian/gpgkey | gpg --dearmor | sudo tee /usr/share/keyrings/helm.gpg > /dev/null + echo "deb [signed-by=/usr/share/keyrings/helm.gpg] https://packages.buildkite.com/helm-linux/helm-debian/any/ any main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list sudo apt-get update sudo apt-get install helm -y --no-install-recommends From 22d51f621b5dc5143e6c08d7b87d9cabf2e17ea0 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Fri, 26 Sep 2025 09:23:06 +0200 Subject: [PATCH 10/57] fix setting app and worker resources via helm Signed-off-by: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> --- .../templates/orborus/orborus-cm-env.yaml | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-cm-env.yaml b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-cm-env.yaml index 87486de5..70209c21 100644 --- a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-cm-env.yaml +++ b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-cm-env.yaml @@ -26,23 +26,23 @@ data: {{- end }} # Shuffle worker resources - {{- $workerResources := (.Values.worker.resources | default (include "common.resources.preset" (dict "type" .Values.worker.resourcesPreset)) | fromYaml) -}} - {{- if $workerResources.requests.cpu }} + {{- $workerResources := (.Values.worker.resources | default (include "common.resources.preset" (dict "type" .Values.worker.resourcesPreset) | fromYaml)) -}} + {{- if and $workerResources.requests $workerResources.requests.cpu }} SHUFFLE_WORKER_CPU_REQUEST: {{ $workerResources.requests.cpu | quote }} {{- end }} - {{- if $workerResources.requests.memory}} + {{- if and $workerResources.requests $workerResources.requests.memory}} SHUFFLE_WORKER_MEMORY_REQUEST: {{ $workerResources.requests.memory | quote }} {{- end }} - {{- if (index $workerResources.requests "ephemeral-storage") }} + {{- if and $workerResources.requests (index $workerResources.requests "ephemeral-storage") }} SHUFFLE_WORKER_EPHEMERAL_STORAGE_REQUEST: {{ (index $workerResources.requests "ephemeral-storage") | quote }} {{- end }} - {{- if $workerResources.limits.cpu }} + {{- if and $workerResources.limits $workerResources.limits.cpu }} SHUFFLE_WORKER_CPU_LIMIT: {{ $workerResources.limits.cpu | quote }} {{- end }} - {{- if $workerResources.limits.memory}} + {{- if and $workerResources.limits $workerResources.limits.memory}} SHUFFLE_WORKER_MEMORY_LIMIT: {{ $workerResources.limits.memory | quote }} {{- end }} - {{- if (index $workerResources.limits "ephemeral-storage") }} + {{- if and $workerResources.limits (index $workerResources.limits "ephemeral-storage") }} SHUFFLE_WORKER_EPHEMERAL_STORAGE_LIMIT: {{ (index $workerResources.limits "ephemeral-storage") | quote }} {{- end }} @@ -57,22 +57,22 @@ data: {{- end }} # Shuffle app resources - {{- $appResources := (.Values.app.resources | default (include "common.resources.preset" (dict "type" .Values.app.resourcesPreset)) | fromYaml) -}} - {{- if $appResources.requests.cpu }} + {{- $appResources := (.Values.app.resources | default (include "common.resources.preset" (dict "type" .Values.app.resourcesPreset) | fromYaml)) -}} + {{- if and $appResources.requests $appResources.requests.cpu }} SHUFFLE_APP_CPU_REQUEST: {{ $appResources.requests.cpu | quote }} {{- end }} - {{- if $appResources.requests.memory}} + {{- if and $appResources.requests $appResources.requests.memory }} SHUFFLE_APP_MEMORY_REQUEST: {{ $appResources.requests.memory | quote }} {{- end }} - {{- if (index $appResources.requests "ephemeral-storage") }} + {{- if and $appResources.requests (index $appResources.requests "ephemeral-storage") }} SHUFFLE_APP_EPHEMERAL_STORAGE_REQUEST: {{ (index $appResources.requests "ephemeral-storage") | quote }} {{- end }} - {{- if $appResources.limits.cpu }} + {{- if and $appResources.limits $appResources.limits.cpu }} SHUFFLE_APP_CPU_LIMIT: {{ $appResources.limits.cpu | quote }} {{- end }} - {{- if $appResources.limits.memory}} + {{- if and $appResources.limits $appResources.limits.memory }} SHUFFLE_APP_MEMORY_LIMIT: {{ $appResources.limits.memory | quote }} {{- end }} - {{- if (index $appResources.limits "ephemeral-storage") }} + {{- if and $appResources.limits (index $appResources.limits "ephemeral-storage") }} SHUFFLE_APP_EPHEMERAL_STORAGE_LIMIT: {{ (index $appResources.limits "ephemeral-storage") | quote }} {{- end }} From a06c4748e3172a914bb8ef444846281b9c7a4594 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Tue, 30 Sep 2025 14:09:06 +0530 Subject: [PATCH 11/57] feat: adding optional resource monitoring --- docker-compose.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index c1c27efb..8ab7e3b9 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -91,6 +91,27 @@ services: - shuffle restart: unless-stopped + # OPTIONAL: advanced monitoring with cAdvisor + # If you have a seperate orborus config YAML on another server, + # Please add it there as well if you want to monitor that server too. + # FYI: both servers will show up on different dashboards. + # cadvisor: + # image: gcr.io/cadvisor/cadvisor:latest + # volumes: + # - /:/rootfs:ro + # - /var/run:/var/run:ro + # - /sys:/sys:ro + # - /var/lib/docker/:/var/lib/docker:ro + # - /dev/disk/:/dev/disk:ro + # - /var/run/docker.sock:/var/run/docker.sock:ro + # ports: + # - "8080:8080" + # privileged: true + # devices: + # - /dev/kmsg:/dev/kmsg + # networks: + # - shuffle + #memcached: # image: memcached:latest # container_name: shuffle-cache From 0b9d13fb2d15229cf951b8dffd7a8b8f94014adc Mon Sep 17 00:00:00 2001 From: "lalitdeore12@gmail.com" Date: Wed, 1 Oct 2025 16:04:51 +0530 Subject: [PATCH 12/57] add cloud sync data in cache --- backend/go-app/main.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index b10d441d..475357ee 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3818,9 +3818,10 @@ func handleCloudJob(job shuffle.CloudSyncJob) error { // Handles jobs from remote (cloud) func remoteOrgJobController(org shuffle.Org, body []byte) error { type retStruct struct { - Success bool `json:"success"` - Reason string `json:"reason"` - Jobs []shuffle.CloudSyncJob `json:"jobs"` + Success bool `json:"success"` + Reason string `json:"reason"` + Jobs []shuffle.CloudSyncJob `json:"jobs"` + SyncFeatures shuffle.SyncFeatures `json:"sync_features"` } responseData := retStruct{} @@ -3887,6 +3888,14 @@ func remoteOrgJobController(org shuffle.Org, body []byte) error { log.Printf("Got job with reason %s and %d job(s)", responseData.Reason, len(responseData.Jobs)) } + cacheKey := fmt.Sprintf("org_sync_features_%s", org.Id) + featuresBytes, err := json.Marshal(responseData.SyncFeatures) + if err != nil { + log.Printf("[ERROR] Failed to marshal SyncFeatures for cache: %s", err) + } else { + shuffle.SetCache(ctx, cacheKey, featuresBytes, 30) + } + for _, job := range responseData.Jobs { err = handleCloudJob(job) if err != nil { From 160352f5f326c5c7156884664f629729e008438c Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Wed, 1 Oct 2025 19:23:27 +0530 Subject: [PATCH 13/57] fixed auto deploy config --- .../kubernetes/charts/shuffle/Chart.yaml | 4 +- .../kubernetes/charts/shuffle/values.yaml | 42 +++++++++++++++---- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/Chart.yaml b/functions/kubernetes/charts/shuffle/Chart.yaml index c1e36716..e286e11f 100644 --- a/functions/kubernetes/charts/shuffle/Chart.yaml +++ b/functions/kubernetes/charts/shuffle/Chart.yaml @@ -9,6 +9,6 @@ dependencies: version: ^2.23.0 repository: oci://registry-1.docker.io/bitnamicharts - name: opensearch - version: ^1.3.0 - repository: oci://registry-1.docker.io/bitnamicharts + version: 2.0.10 + repository: https://charts.bitnami.com/bitnami condition: opensearch.enabled diff --git a/functions/kubernetes/charts/shuffle/values.yaml b/functions/kubernetes/charts/shuffle/values.yaml index 5937ecdd..d0b39608 100644 --- a/functions/kubernetes/charts/shuffle/values.yaml +++ b/functions/kubernetes/charts/shuffle/values.yaml @@ -19,6 +19,8 @@ global: defaultStorageClass: "" ## Compatibility adaptations for Kubernetes platforms ## + security: + allowInsecureImages: true compatibility: ## Compatibility adaptations for Openshift ## @@ -1197,7 +1199,19 @@ orborus: ## - name: FOO ## value: "bar" ## - extraEnvVars: [] + extraEnvVars: + - name: SHUFFLE_APP_SDK_TIMEOUT + value: "300" + - name: SHUFFLE_ORBORUS_EXCUTION_CONCURRENCY + value: "7" + - name: ENVIRONMENT_NAME + value: "Shuffle" + - name: BASE_URL + value: "http://shuffle-backend:5001" + - name: SHUFFLE_STATS_DISABLED + value: "true" + - name: CLEANUP + value: "false" ## @param orborus.extraEnvVarsCM Name of existing ConfigMap containing extra env vars for orborus containers ## extraEnvVarsCM: "" @@ -1841,7 +1855,7 @@ volumePermissions: ## @param volumePermissions.image.pullSecrets OS Shell + Utility image pull secrets ## image: - registry: docker.io + registry: public.ecr.aws repository: bitnami/os-shell tag: 12-debian-12-r30 pullPolicy: IfNotPresent @@ -1898,17 +1912,31 @@ volumePermissions: ## opensearch: enabled: true - + + sysctlImage: + enabled: false + + image: + registry: public.ecr.aws + repository: bitnami/opensearch + tag: "3.2.0" + master: replicaCount: 1 + extraEnvVars: + - { name: OPENSEARCH_JAVA_OPTS, value: "-Xms1g -Xmx1g" } + - { name: DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI, value: "true" } + - { name: node.store.allow_mmap, value: "false" } + - { name: cluster.routing.allocation.disk.threshold_enabled, value: "false" } + data: - replicaCount: 1 + replicaCount: 0 coordinating: - replicaCount: 1 + replicaCount: 0 ingest: - replicaCount: 1 + replicaCount: 0 dashboards: - enabled: true + enabled: false ## @section Vault Parameters ## From dcc20e9c844f464abe1313211493cf0ded77ff61 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Thu, 2 Oct 2025 01:58:12 +0530 Subject: [PATCH 14/57] fix default k8 deployment --- .../kubernetes/charts/shuffle/values.yaml | 30 ++++++++----------- functions/onprem/orborus/orborus.go | 2 +- functions/onprem/worker/worker.go | 2 +- 3 files changed, 14 insertions(+), 20 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/values.yaml b/functions/kubernetes/charts/shuffle/values.yaml index d0b39608..1e6af05e 100644 --- a/functions/kubernetes/charts/shuffle/values.yaml +++ b/functions/kubernetes/charts/shuffle/values.yaml @@ -76,7 +76,7 @@ diagnosticMode: shuffle: ## @param shuffle.baseUrl The external base URL under which Shuffle is reachable. ## - baseUrl: "" + #baseUrl: "" ## ref: https://shuffler.io/docs/organizations ## This chart only supports single-tenant deployments at the moment @@ -527,6 +527,8 @@ backend: ## @param backend.openSearch.username The username that is used for authenticating with OpenSearch ## username: admin + + password: StrongShufflePassword321! ## @param backend.openSearch.certificateFile The path to a custom OpenSearch certificate file ## certificateFile: "" @@ -1204,14 +1206,12 @@ orborus: value: "300" - name: SHUFFLE_ORBORUS_EXCUTION_CONCURRENCY value: "7" - - name: ENVIRONMENT_NAME - value: "Shuffle" - - name: BASE_URL - value: "http://shuffle-backend:5001" - name: SHUFFLE_STATS_DISABLED value: "true" - - name: CLEANUP - value: "false" + - name: KUBERNETES_NAMESPACE + value: "shuffle" + - name: SHUFFLE_BASE_IMAGE_NAME + value: "frikky/shuffle" ## @param orborus.extraEnvVarsCM Name of existing ConfigMap containing extra env vars for orborus containers ## extraEnvVarsCM: "" @@ -1855,8 +1855,8 @@ volumePermissions: ## @param volumePermissions.image.pullSecrets OS Shell + Utility image pull secrets ## image: - registry: public.ecr.aws - repository: bitnami/os-shell + registry: docker.io + repository: bitnamilegacy/os-shell tag: 12-debian-12-r30 pullPolicy: IfNotPresent ## Optionally specify an array of imagePullSecrets. @@ -1917,20 +1917,14 @@ opensearch: enabled: false image: - registry: public.ecr.aws - repository: bitnami/opensearch + registry: docker.io + repository: bitnamilegacy/opensearch tag: "3.2.0" master: replicaCount: 1 - extraEnvVars: - - { name: OPENSEARCH_JAVA_OPTS, value: "-Xms1g -Xmx1g" } - - { name: DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI, value: "true" } - - { name: node.store.allow_mmap, value: "false" } - - { name: cluster.routing.allocation.disk.threshold_enabled, value: "false" } - data: - replicaCount: 0 + replicaCount: 1 coordinating: replicaCount: 0 ingest: diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index ab9b2230..2fd9cd0f 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -1586,7 +1586,7 @@ func initializeImages() { if len(os.Getenv("REGISTRY_URL")) > 0 { baseimageregistry = os.Getenv("REGISTRY_URL") } else { - os.Setenv("REGISTRY_URL", baseimageregistry) + // os.Setenv("REGISTRY_URL", baseimageregistry) } os.Setenv("SHUFFLE_BASE_IMAGE_REGISTRY", baseimageregistry) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 6dd41de4..10c1fc8c 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -3893,7 +3893,7 @@ func sendAppRequest(ctx context.Context, incomingUrl, appName string, port int, if attempts < 2 { // Check the service and fix it. if isKubernetes == "true" { - log.Printf("[WARNING] App Redeployment in K8s isn't fully supported yet, but should be done for app %s with image %s.", appName, image) + log.Printf("[WARNING] App Redeployment in K8s isn't fully supported yet, but should be done for app %s with image %s.", appName, image) } else { _, err = findAppInfo(image, appName, true) if err != nil { From 7947c6dcaa9fc19454438b8d0f8d54b2cda63a3c Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Thu, 2 Oct 2025 02:01:36 +0530 Subject: [PATCH 15/57] don't allow insecure images --- functions/kubernetes/charts/shuffle/values.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/values.yaml b/functions/kubernetes/charts/shuffle/values.yaml index 1e6af05e..4f46e482 100644 --- a/functions/kubernetes/charts/shuffle/values.yaml +++ b/functions/kubernetes/charts/shuffle/values.yaml @@ -19,8 +19,6 @@ global: defaultStorageClass: "" ## Compatibility adaptations for Kubernetes platforms ## - security: - allowInsecureImages: true compatibility: ## Compatibility adaptations for Openshift ## From 2fb54766e03a4178ed9c91efabb29fc46b9daca4 Mon Sep 17 00:00:00 2001 From: monilprajapati Date: Thu, 2 Oct 2025 18:05:24 +0530 Subject: [PATCH 16/57] nunito font issue --- frontend/src/codeeditor-index.css | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/frontend/src/codeeditor-index.css b/frontend/src/codeeditor-index.css index c87341f5..9ccc0acd 100644 --- a/frontend/src/codeeditor-index.css +++ b/frontend/src/codeeditor-index.css @@ -1,9 +1,7 @@ -@import url('https://fonts.googleapis.com/css?family=Nunito+Sans'); - body { margin: 0; padding: 0; - font-family: "Nunito Sans", sans-serif; + font-family: "Segoe UI", Roboto, "Noto Sans", "Liberation Sans", Arial, "Helvetica Neue", -apple-system, BlinkMacSystemFont, "Nunito Sans", sans-serif; -webkit-font-smoothing: antialiased; -moz-osx-font-smoothing: grayscale; } From c13137ce8c8826b20712e203ac44d4d10d923660 Mon Sep 17 00:00:00 2001 From: "lalitdeore12@gmail.com" Date: Thu, 2 Oct 2025 18:55:53 +0530 Subject: [PATCH 17/57] Add tenant restriction --- backend/go-app/main.go | 41 ++++++++++++++++++++++++++++++++++++++++- frontend/src/App.jsx | 7 +++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 475357ee..55bc89c2 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -935,6 +935,45 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { log.Printf("[DEBUG] Failed to get org during getinfo: %s", err) } + childOrgs := []shuffle.Org{} + if len(org.CreatorOrg) > 0 { + childOrgs, err = shuffle.GetAllChildOrgs(ctx, org.CreatorOrg) + if err != nil { + log.Printf("[ERROR] Failed to get child orgs during getinfo: %s", err) + childOrgs = []shuffle.Org{} + } + } + // Change this deadline date as release date while pushing to production + deadline := time.Date(2025, 10, 5, 0, 0, 0, 0, time.UTC).Unix() + if len(org.CreatorOrg) > 0 && len(childOrgs) > 3 && org.Created >= deadline { + parentOrg, err := shuffle.GetOrg(ctx, org.CreatorOrg) + if err != nil { + log.Printf("[ERROR] Failed to get parent org during getinfo: %s", err) + } else { + parent := shuffle.HandleCheckLicense(ctx, *parentOrg) + parentOrg := &parent + if !parentOrg.SyncFeatures.MultiTenant.Active { + userInfo.ActiveOrg = shuffle.OrgMini{ + Id: parentOrg.Id, + Name: parentOrg.Name, + Role: userInfo.Role, + Branding: parentOrg.Branding, + Image: parentOrg.Image, + } + log.Printf("[INFO] Parent org %s has more than 3 child orgs and is not licensed. Moving user %s to parent org %s", parentOrg.Name, userInfo.Username, parentOrg.Name) + + err = shuffle.SetUser(ctx, &userInfo, false) + if err != nil { + log.Printf("[WARNING] Failed setting user to parent org: %s", err) + } + + resp.WriteHeader(200) + resp.Write([]byte(`{"success": true, "reason": "Parent org has more than 3 child orgs and is not licensed. Moving to parent org. Contact support@shuffler.io for more information", "switch_parent": true}`)) + return + } + } + } + //if err == nil { if len(org.Id) > 0 { if userInfo.Role == "" { @@ -3893,7 +3932,7 @@ func remoteOrgJobController(org shuffle.Org, body []byte) error { if err != nil { log.Printf("[ERROR] Failed to marshal SyncFeatures for cache: %s", err) } else { - shuffle.SetCache(ctx, cacheKey, featuresBytes, 30) + shuffle.SetCache(ctx, cacheKey, featuresBytes, 1800) } for _, job := range responseData.Jobs { diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 9425a1c6..75363d20 100755 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -180,6 +180,13 @@ const App = (message, props) => { var userInfo = {}; if (responseJson.success === true) { //console.log("USER: ", responseJson); + if (responseJson?.switch_parent === true) { + toast.info(responseJson.reason) + setTimeout(() => { + window.location.reload(); + }, 3000); + return + } userInfo = responseJson; setIsLoggedIn(true); From c3ae26b429d7418036ebf5b369e73b2373d1a1b6 Mon Sep 17 00:00:00 2001 From: "lalitdeore12@gmail.com" Date: Fri, 3 Oct 2025 16:52:25 +0530 Subject: [PATCH 18/57] Fix cloud sync issue when setup first time --- backend/go-app/main.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 55bc89c2..b00a1628 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -4940,7 +4940,15 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { // 2. Add iterative sync schedule for interval seconds // 3. Add another environment for the org's users org.CloudSync = true - org.SyncFeatures = responseData.SyncFeatures + + // set cache here for 30 min + cacheKey := fmt.Sprintf("org_sync_features_%s", org.Id) + featuresBytes, err := json.Marshal(responseData.SyncFeatures) + if err != nil { + log.Printf("[ERROR] Failed to marshal SyncFeatures for cache: %s", err) + } else { + shuffle.SetCache(ctx, cacheKey, featuresBytes, 1800) + } org.SyncConfig = shuffle.SyncConfig{ Apikey: responseData.SessionKey, From 847fd5bffbcfe6f632153a782a6aa039c3853389 Mon Sep 17 00:00:00 2001 From: monilprajapati Date: Fri, 3 Oct 2025 16:33:59 +0530 Subject: [PATCH 19/57] Cloud sync for subscription --- backend/go-app/main.go | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index b10d441d..f742b937 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -69,11 +69,12 @@ var debug = false //var syncUrl = "http://localhost:5002" type retStruct struct { - Success bool `json:"success"` - SyncFeatures shuffle.SyncFeatures `json:"sync_features"` - SessionKey string `json:"session_key"` - IntervalSeconds int64 `json:"interval_seconds"` - Reason string `json:"reason"` + Success bool `json:"success"` + SyncFeatures shuffle.SyncFeatures `json:"sync_features"` + SessionKey string `json:"session_key"` + IntervalSeconds int64 `json:"interval_seconds"` + Reason string `json:"reason"` + Subscriptions []shuffle.PaymentSubscription `json:"subscriptions"` } type Contact struct { @@ -3821,6 +3822,8 @@ func remoteOrgJobController(org shuffle.Org, body []byte) error { Success bool `json:"success"` Reason string `json:"reason"` Jobs []shuffle.CloudSyncJob `json:"jobs"` + SyncFeatures shuffle.SyncFeatures `json:"sync_features"` + Subscriptions []shuffle.PaymentSubscription `json:"subscriptions"` } responseData := retStruct{} @@ -3887,6 +3890,22 @@ func remoteOrgJobController(org shuffle.Org, body []byte) error { log.Printf("Got job with reason %s and %d job(s)", responseData.Reason, len(responseData.Jobs)) } + cacheKey := fmt.Sprintf("org_sync_features_%s", org.Id) + featuresBytes, err := json.Marshal(responseData.SyncFeatures) + if err != nil { + log.Printf("[ERROR] Failed to marshal SyncFeatures for cache: %s", err) + } else { + shuffle.SetCache(ctx, cacheKey, featuresBytes, 30) + } + + subscriptionCacheKey := fmt.Sprintf("org_subscriptions_%s", org.Id) + subscriptionsBytes, err := json.Marshal(responseData.Subscriptions) + if err != nil { + log.Printf("[ERROR] Failed to marshal Subscriptions for cache: %s", err) + } else { + shuffle.SetCache(ctx, subscriptionCacheKey, subscriptionsBytes, 30) + } + for _, job := range responseData.Jobs { err = handleCloudJob(job) if err != nil { @@ -4650,6 +4669,7 @@ func handleStopCloudSync(syncUrl string, org shuffle.Org) (*shuffle.Org, error) org.CloudSync = false org.SyncFeatures = shuffle.SyncFeatures{} org.SyncConfig = shuffle.SyncConfig{} + org.Subscriptions = []shuffle.PaymentSubscription{} err = shuffle.SetOrg(ctx, org, org.Id) if err != nil { @@ -4865,8 +4885,6 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("[DEBUG] Respbody from sync: %s", string(respBody)) - responseData := retStruct{} err = json.Unmarshal(respBody, &responseData) if err != nil { @@ -4893,7 +4911,7 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { // 3. Add another environment for the org's users org.CloudSync = true org.SyncFeatures = responseData.SyncFeatures - + org.Subscriptions = responseData.Subscriptions org.SyncConfig = shuffle.SyncConfig{ Apikey: responseData.SessionKey, Interval: responseData.IntervalSeconds, From d56b631b07b018a5f32a146169a4934d94720ace Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 3 Oct 2025 13:42:11 +0200 Subject: [PATCH 20/57] Nightly orborus rebuild --- backend/go-app/walkoff.go | 3 +-- functions/onprem/orborus/go.mod | 2 +- functions/onprem/orborus/go.sum | 4 ++-- functions/onprem/orborus/orborus.go | 34 +++++++++++++++++++---------- 4 files changed, 27 insertions(+), 16 deletions(-) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 3cf355ab..090d524e 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -1847,13 +1847,12 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000, 0) if err != nil { log.Printf("{WARNING] Failed getting apps (getworkflowapps): %s", err) - resp.WriteHeader(401) + resp.WriteHeader(400) resp.Write([]byte(`{"success": false}`)) return } newapps := workflowapps - if len(user.PrivateApps) > 0 { found := false for _, item := range user.PrivateApps { diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index ed7e5add..4f3177e8 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -10,7 +10,7 @@ require ( github.com/docker/docker v28.3.3+incompatible github.com/docker/go-connections v0.5.0 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.15 + github.com/shuffle/shuffle-shared v0.9.26 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 ) diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index 3818c38f..f92997a8 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -328,8 +328,8 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.9.15 h1:Gc7c0pbWG6nHWSTkcfAnKgQgWCWfc6aDQ/BIu20z6bM= -github.com/shuffle/shuffle-shared v0.9.15/go.mod h1:PhDEizuz4SmJaSmy0+yrFWwD1mXVUsy8/knKlrqF1qw= +github.com/shuffle/shuffle-shared v0.9.26 h1:D7ZSnRGROtEP7eNprmYuc1wcxa+kV7bY5M+HFrOo990= +github.com/shuffle/shuffle-shared v0.9.26/go.mod h1:PhDEizuz4SmJaSmy0+yrFWwD1mXVUsy8/knKlrqF1qw= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index fb93ad89..17b7781c 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -1571,12 +1571,12 @@ func initializeImages() { if appSdkVersion == "" { appSdkVersion = "latest" - log.Printf("[WARNING] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %#v", appSdkVersion) + log.Printf("[INFO] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %#v", appSdkVersion) } if workerVersion == "" { workerVersion = "latest" - log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %#v", workerVersion) + log.Printf("[INFO] SHUFFLE_WORKER_VERSION not defined. Defaulting to %#v", workerVersion) } if baseimageregistry == "" { @@ -1591,7 +1591,7 @@ func initializeImages() { os.Setenv("SHUFFLE_BASE_IMAGE_REGISTRY", baseimageregistry) - log.Printf("[WARNING] Setting baseimageregistry to %#v", baseimageregistry) + log.Printf("[INFO] Setting baseimageregistry to %#v", baseimageregistry) } if baseimagename == "" { @@ -1600,7 +1600,7 @@ func initializeImages() { baseimagename = "frikky/shuffle" // Dockerhub os.Setenv("SHUFFLE_BASE_IMAGE_NAME", baseimagename) - log.Printf("[WARNING] Setting baseimagename to %#v", baseimagename) + log.Printf("[INFO] Setting baseimagename to %#v", baseimagename) } // Old sane default overrides: @@ -2860,15 +2860,20 @@ func deployTenzirNode() error { ctx := context.Background() cacheKey := "tenzir-key" - - imageName := "frikky/shuffle:tenzir" - containerName := "tenzir-node" - containerStartOptions := container.StartOptions{} _, err = shuffle.GetCache(ctx, cacheKey) if err == nil { return nil } + imageName := "frikky/shuffle:tenzir" + if os.Getenv("TENZIR_IMAGE_NAME") != "" { + imageName = os.Getenv("TENZIR_IMAGE_NAME") + log.Printf("[INFO] Using custom Tenzir image name: %s", imageName) + } + + containerName := "tenzir-node" + containerStartOptions := container.StartOptions{} + containerInfo, err := dockercli.ContainerInspect(ctx, containerName) if err != nil { if dockerclient.IsErrNotFound(err) { @@ -2909,7 +2914,7 @@ func deployTenzirNode() error { } } else { if !containerInfo.State.Running { - log.Printf("[DEBUG] Tenzir Node exists but is not running. Restarting it.") + log.Printf("[DEBUG] Tenzir Node exists, but is not running. Restarting it.") err := dockercli.ContainerStart(ctx, containerName, containerStartOptions) if err != nil { log.Printf("[ERROR] Failed to start Tenzir Node container: %v", err) @@ -3144,7 +3149,7 @@ func createNetworkIfNotExists(ctx context.Context, networkName, subnet, gateway } func checkTenzirNode() error { - if os.Getenv("SHUFFLE_SKIP_PIPELINES") == "true" { + if os.Getenv("SHUFFLE_SKIP_PIPELINES") == "true" && os.Getenv("SHUFFLE_PIPELINE_ENABLED") == "false" { return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES") } @@ -3480,7 +3485,14 @@ func handleFileCategoryChange() error { } if len(pipelineApikey) == 0 { - return errors.New("Shuffle API-key not set for Pipelines: SHUFFLE_PIPELINE_AUTH=") + //var auth = os.Getenv("AUTH") + //var org = os.Getenv("ORG") + + if len(auth) > 0 && len(org) > 0 { + pipelineApikey = auth + } else { + return errors.New("Shuffle API-key not set for Pipelines: SHUFFLE_PIPELINE_AUTH=") + } } req.Header.Add("Authorization", "Bearer "+pipelineApikey) From 7cbd9456a2cb4b3f1d63e4e1e9dc0016452d12e5 Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 3 Oct 2025 14:15:45 +0200 Subject: [PATCH 21/57] More Orborus tweaks for Tenzir --- functions/onprem/orborus/orborus.go | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index dad6be9b..c113c340 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -2121,6 +2121,12 @@ func main() { } } + // Auto enables pipelines IF they are not mentioned + if len(os.Getenv("SHUFFLE_SKIP_PIPELINES")) == 0 { + os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") + os.Setenv("SHUFFLE_PIPELINE_ENABLED", "true") + } + log.Println("[INFO] Setting up execution environment") // //FIXME @@ -2490,8 +2496,7 @@ func main() { // Looking for specific jobs if incRequest.Type == "PIPELINE_CREATE" || incRequest.Type == "PIPELINE_START" || incRequest.Type == "PIPELINE_STOP" || incRequest.Type == "PIPELINE_DELETE" { - - os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") + //os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") tenzirDisabled = false // Running NEW or editing pipelines @@ -2558,7 +2563,7 @@ func main() { log.Printf("[INFO] Got job to start tenzir") // Manual command = overrides to allow starting of Tenzir from the frontend anyway. - os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") + //os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") tenzirDisabled = false // Removed either way @@ -2568,7 +2573,7 @@ func main() { if err != nil { if strings.Contains(fmt.Sprintf("%s", err), "node available") { // Disabling until UI is updated - os.Setenv("SHUFFLE_SKIP_PIPELINES", "true") + //os.Setenv("SHUFFLE_SKIP_PIPELINES", "true") tenzirDisabled = true log.Printf("[ERROR] Failed to start tenzir, reason: %s", err) @@ -3284,8 +3289,9 @@ func createPipeline(command, identifier string) (string, error) { "definition": command, "name": identifier, "hidden": false, + "retry_delay": "500.0ms", "autostart": map[string]bool{ - "created": true, + //"created": true, "completed": false, "failed": false, }, @@ -3294,7 +3300,6 @@ func createPipeline(command, identifier string) (string, error) { "failed": false, "stopped": false, }, - "retry_delay": "500.0ms", } requestBodyJSON, err := json.Marshal(requestBody) @@ -3304,18 +3309,18 @@ func createPipeline(command, identifier string) (string, error) { } forwardData := bytes.NewBuffer(requestBodyJSON) - req, err := http.NewRequest( forwardMethod, url, forwardData, ) + if err != nil { log.Printf("[ERROR] Failed to create HTTP request: %s", err) return "", err } - req.Header.Set("Content-Type", "application/json") + req.Header.Set("Content-Type", "application/json") client := &http.Client{Timeout: 10 * time.Second} resp, err := client.Do(req) if err != nil { @@ -3700,6 +3705,7 @@ func removeFileCategory() error { return nil } +// curl https://get.tenzir.app | sh func removeFile(fileName string) error { containerName := "tenzir-node" srcPath := fmt.Sprintf("/var/lib/tenzir/sigma_rules/%s", fileName) From 2d780192c380b317002145855ba01d72a639493e Mon Sep 17 00:00:00 2001 From: "lalitdeore12@gmail.com" Date: Fri, 3 Oct 2025 18:38:37 +0530 Subject: [PATCH 22/57] Fail org loading base on the timestamp --- backend/go-app/main.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index b00a1628..17817cd5 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -943,9 +943,22 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { childOrgs = []shuffle.Org{} } } - // Change this deadline date as release date while pushing to production - deadline := time.Date(2025, 10, 5, 0, 0, 0, 0, time.UTC).Unix() - if len(org.CreatorOrg) > 0 && len(childOrgs) > 3 && org.Created >= deadline { + + failToLoadOrgs := []string{} + sort.Slice(childOrgs, func(i, j int) bool { + return childOrgs[i].Created < childOrgs[j].Created + }) + + for index, org := range childOrgs { + + if index < 3 { + continue + } + + failToLoadOrgs = append(failToLoadOrgs, org.Id) + } + + if len(org.CreatorOrg) > 0 && len(childOrgs) > 3 && shuffle.ArrayContains(failToLoadOrgs, userInfo.ActiveOrg.Id) { parentOrg, err := shuffle.GetOrg(ctx, org.CreatorOrg) if err != nil { log.Printf("[ERROR] Failed to get parent org during getinfo: %s", err) From eefe0f792c4b68d2d61bcf1bca657ddd4dbbc5fa Mon Sep 17 00:00:00 2001 From: monilprajapati Date: Fri, 3 Oct 2025 18:58:39 +0530 Subject: [PATCH 23/57] Increased the time for cache --- backend/go-app/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index f742b937..57887c9a 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3903,7 +3903,7 @@ func remoteOrgJobController(org shuffle.Org, body []byte) error { if err != nil { log.Printf("[ERROR] Failed to marshal Subscriptions for cache: %s", err) } else { - shuffle.SetCache(ctx, subscriptionCacheKey, subscriptionsBytes, 30) + shuffle.SetCache(ctx, subscriptionCacheKey, subscriptionsBytes, 1800) } for _, job := range responseData.Jobs { From 7dfbe3803da0618f670762fcef58b1befb181302 Mon Sep 17 00:00:00 2001 From: Frikky Date: Sun, 5 Oct 2025 21:27:37 +0200 Subject: [PATCH 24/57] Minor orborus fix --- functions/onprem/orborus/orborus.go | 44 ++++++++++++++++------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index c113c340..08c1557b 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -12,7 +12,6 @@ import ( "fmt" "io" "io/ioutil" - "regexp" "log" "math" "net" @@ -20,6 +19,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "runtime" "strconv" "strings" @@ -90,6 +90,7 @@ var debug = os.Getenv("DEBUG") == "true" // var baseimagename = "shuffle/shuffle" var baseimageregistry = os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY") var baseimagename = os.Getenv("SHUFFLE_BASE_IMAGE_NAME") + //var baseimagetagsuffix = os.Getenv("SHUFFLE_BASE_IMAGE_TAG_SUFFIX") // Used for cloud with auth. Onprem in certain cases too. @@ -344,7 +345,7 @@ func deployServiceWorkers(image string) { if len(dockerSwarmBridgeMTU) == 0 { mtu, err = strconv.Atoi(dockerSwarmBridgeMTU) // by default if err != nil { - if debug { + if debug { log.Printf("[DEBUG] Failed to convert the default MTU to int: %s. Using 1500 instead. Input: %s", err, dockerSwarmBridgeMTU) } @@ -531,7 +532,6 @@ func deployServiceWorkers(image string) { nodeCount = uint64(cnt) } - appReplicas := os.Getenv("SHUFFLE_APP_REPLICAS") appReplicaCnt := 2 if len(appReplicas) > 0 { @@ -2496,6 +2496,8 @@ func main() { // Looking for specific jobs if incRequest.Type == "PIPELINE_CREATE" || incRequest.Type == "PIPELINE_START" || incRequest.Type == "PIPELINE_STOP" || incRequest.Type == "PIPELINE_DELETE" { + log.Printf("[INFO] Handling pipeline request from backend: '%s' with argument '%s'", incRequest.Type, incRequest.ExecutionArgument) + //os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") tenzirDisabled = false @@ -2640,7 +2642,7 @@ func main() { executionRequests.Data = executionRequests.Data[0:allowed] } } else if swarmControlMode && (swarmConfig == "run" || swarmConfig == "swarm") { - // any reason it is not maxConcurrency instead of + // any reason it is not maxConcurrency instead of // hardcoded 50? if len(executionRequests.Data) > 50 { executionRequests.Data = executionRequests.Data[0:50] @@ -2842,19 +2844,16 @@ func handlePipeline(incRequest shuffle.ExecutionRequest) error { return err } } else if incRequest.Type == "PIPELINE_DELETE" || incRequest.Type == "PIPELINE_STOP" { - { - log.Printf("[INFO] Should delete pipeline %#v", identifier) - pipelineId, err := searchPipeline(identifier) - if err != nil { - log.Printf("[ERROR] Failed searching for Pipeline with name %s reason:%s ", identifier, err) - return err - } + pipelineId := incRequest.ExecutionId + log.Printf("[INFO] Should delete pipeline %#v. PipelineID: %s", identifier, pipelineId) + //pipelineId, err := searchPipeline(identifier) + //if err != nil { + //} - err = deletePipeline(pipelineId) - if err != nil { - log.Printf("[ERROR] Failed Deleting Pipeline %s", err) - return err - } + err = deletePipeline(pipelineId) + if err != nil { + log.Printf("[ERROR] Failed Deleting Pipeline %s", err) + return err } /* @@ -2888,6 +2887,7 @@ func handlePipeline(incRequest shuffle.ExecutionRequest) error { log.Printf("[ERROR] Failed searching for Pipeline with name %s reason:%s ", identifier, err) return err } + _, err = updatePipelineState(command, pipelineId, "start") if err != nil { log.Printf("[ERROR] Failed to start Pipeline: %s reason:%s ", pipelineId, err) @@ -3027,7 +3027,9 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri ExposedPorts: nat.PortSet{ "5160/tcp": struct{}{}, "514/udp": struct{}{}, + "1514/udp": struct{}{}, "514/tcp": struct{}{}, + "1514/tcp": struct{}{}, }, Entrypoint: []string{containerName}, Env: []string{}, @@ -3075,6 +3077,8 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri PortBindings: nat.PortMap{ "514/tcp": []nat.PortBinding{{HostPort: "514"}}, "514/udp": []nat.PortBinding{{HostPort: "514"}}, + "1514/tcp": []nat.PortBinding{{HostPort: "1514"}}, + "1514/udp": []nat.PortBinding{{HostPort: "1514"}}, "5160/tcp": []nat.PortBinding{{HostPort: "5160"}}, }, Mounts: []mount.Mount{ @@ -3286,9 +3290,9 @@ func createPipeline(command, identifier string) (string, error) { //command = "from file /var/lib/tenzir/sysmon_logs.ndjson read json | import" requestBody := map[string]interface{}{ - "definition": command, - "name": identifier, - "hidden": false, + "definition": command, + "name": identifier, + "hidden": false, "retry_delay": "500.0ms", "autostart": map[string]bool{ //"created": true, @@ -4072,7 +4076,7 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest, image string, // Specific to debugging if len(workerServerUrl) == 0 { - if debug { + if debug { log.Printf("[INFO] Using default worker server url as previous is invalid: %s. Swapping to shuffle-workers:33333", streamUrl) } } From 186e9d14e0685debc71d16a8dbb2ec922238cf72 Mon Sep 17 00:00:00 2001 From: Frikky Date: Sun, 5 Oct 2025 23:57:32 +0200 Subject: [PATCH 25/57] Rebuilding with /tmp:/tmp mapping in mind --- functions/onprem/orborus/go.mod | 2 +- functions/onprem/orborus/orborus.go | 95 +++++++++++++++-------------- 2 files changed, 51 insertions(+), 46 deletions(-) diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 4f3177e8..1c6a7a5a 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -4,7 +4,7 @@ go 1.24.0 toolchain go1.24.4 -//replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared +replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared require ( github.com/docker/docker v28.3.3+incompatible diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 08c1557b..457b36bc 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -118,7 +118,7 @@ var pipelineApikey = os.Getenv("SHUFFLE_PIPELINE_AUTH") var pipelineUrl = os.Getenv("SHUFFLE_PIPELINE_URL") var executionIds = []string{} -var pipelines = []shuffle.PipelineInfoMini{} +var pipelines = []shuffle.PipelineInfo{} var namespacemade = false // For K8s var skipPipelineMount = false var tenzirDisabled = false @@ -2495,7 +2495,7 @@ func main() { for _, incRequest := range executionRequests.Data { // Looking for specific jobs - if incRequest.Type == "PIPELINE_CREATE" || incRequest.Type == "PIPELINE_START" || incRequest.Type == "PIPELINE_STOP" || incRequest.Type == "PIPELINE_DELETE" { + if incRequest.Type == "PIPELINE_CREATE" || incRequest.Type == "PIPELINE_START" || incRequest.Type == "PIPELINE_STOP" || incRequest.Type == "PIPELINE_DELETE" || incRequest.Type == "PIPELINE_UPDATE" { log.Printf("[INFO] Handling pipeline request from backend: '%s' with argument '%s'", incRequest.Type, incRequest.ExecutionArgument) //os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") @@ -2521,8 +2521,8 @@ func main() { } else if incRequest.Type == "CATEGORY_UPDATE" { os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") - tenzirDisabled = false + tenzirDisabled = false err = handleFileCategoryChange() if err != nil { log.Printf("[ERROR] Failed to download the file category: %s", err) @@ -2576,7 +2576,7 @@ func main() { if strings.Contains(fmt.Sprintf("%s", err), "node available") { // Disabling until UI is updated //os.Setenv("SHUFFLE_SKIP_PIPELINES", "true") - tenzirDisabled = true + //tenzirDisabled = true log.Printf("[ERROR] Failed to start tenzir, reason: %s", err) err = shuffle.CreateOrgNotification( @@ -2824,7 +2824,7 @@ func handlePipeline(incRequest shuffle.ExecutionRequest) error { // no need of execution arguments for STOP and DELETE if (incRequest.Type != "PIPELINE_STOP" && incRequest.Type != "PIPELINE_DELETE") && len(incRequest.ExecutionArgument) == 0 { - log.Printf("[ERROR] No execution argument found for pipeline create. Skipping") + log.Printf("[ERROR] No execution argument found for pipeline type %s. Skipping", incRequest.Type) return errors.New("no execution argument found for pipeline create. Skipping") } @@ -2835,6 +2835,7 @@ func handlePipeline(incRequest shuffle.ExecutionRequest) error { } command := incRequest.ExecutionArgument + pipelines = []shuffle.PipelineInfo{} if incRequest.Type == "PIPELINE_CREATE" { log.Printf("[INFO] Should delete -> recreate new pipeline with id %#v", identifier) //err := deployPipeline(image, identifier, command) @@ -2880,20 +2881,24 @@ func handlePipeline(incRequest shuffle.ExecutionRequest) error { if err != nil { if err.Error() == "no existing pipeline found with name" { log.Printf("[INFO] Starting a new pipeline with command '%s' and identifier '%s'", command, identifier) - _, CreateErr := createPipeline(command, identifier) - return CreateErr + var createErr error + pipelineId, createErr = createPipeline(command, identifier) + if createErr != nil { + return createErr + } + } else { + log.Printf("[ERROR] Failed searching for Pipeline with name %s reason:%s ", identifier, err) + return err } - - log.Printf("[ERROR] Failed searching for Pipeline with name %s reason:%s ", identifier, err) - return err } + log.Printf("[INFO] Starting existing pipeline with ID %s", pipelineId) _, err = updatePipelineState(command, pipelineId, "start") if err != nil { log.Printf("[ERROR] Failed to start Pipeline: %s reason:%s ", pipelineId, err) return err } else { - log.Printf("[INFO] Successfully started the Pipeline: %s", pipelineId) + log.Printf("[INFO] Successfully started pipeline: %s", pipelineId) } } else { @@ -3085,8 +3090,9 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri { Type: "bind", Source: tenzirStorageFolder, - Target: "/var/lib/tenzir/", + Target: "/tmp", }, + /* { Type: "bind", Source: tenzirStorageFolder, @@ -3097,6 +3103,7 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri Source: tenzirStorageFolder, Target: "/var/cache/tenzir/", }, + */ }, VolumeDriver: "local", RestartPolicy: container.RestartPolicy{ @@ -3104,6 +3111,12 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri }, } + if os.Getenv("SHUFFLE_DISABLE_SYSLOG") == "true" { + hostConfig.PortBindings = nat.PortMap{ + "5160/tcp": []nat.PortBinding{{HostPort: "5160"}}, + } + } + if skipPipelineMount { hostConfig.Mounts = []mount.Mount{} } @@ -3289,21 +3302,18 @@ func createPipeline(command, identifier string) (string, error) { //command = "from file /var/lib/tenzir/sysmon_logs.ndjson read json | sigma /var/lib/tenzir/rule.yaml" //command = "from file /var/lib/tenzir/sysmon_logs.ndjson read json | import" + // Make sure to escape them + //if strings.Contains(command, "/") { + // command = strings.ReplaceAll("\\\"", "", command) + // command = strings.ReplaceAll(command, "\"", "") + //} + requestBody := map[string]interface{}{ "definition": command, "name": identifier, "hidden": false, "retry_delay": "500.0ms", - "autostart": map[string]bool{ - //"created": true, - "completed": false, - "failed": false, - }, - "autodelete": map[string]bool{ - "completed": false, - "failed": false, - "stopped": false, - }, + "unstoppable": true, } requestBodyJSON, err := json.Marshal(requestBody) @@ -3339,7 +3349,9 @@ func createPipeline(command, identifier string) (string, error) { } if strings.Contains(string(body), "error") { - log.Printf("[ERROR] Pipeline creation response (%d): %s", resp.StatusCode, string(body)) + log.Printf("[ERROR] Pipeline creation error resp (%d): %s", resp.StatusCode, string(body)) + } else { + log.Printf("[DEBUG] Pipeline creation debug (%d): %s", resp.StatusCode, string(body)) } defer resp.Body.Close() @@ -3365,37 +3377,39 @@ func createPipeline(command, identifier string) (string, error) { return "", errors.New("Pipeline ID not found or empty in the response. See error logs.") } - id := response.ID - return id, nil + return response.ID, nil } func updatePipelineState(command, pipelineId, action string) (string, error) { url := fmt.Sprintf("%s/api/v0/pipeline/update", pipelineUrl) forwardMethod := "POST" - requestBody := map[string]interface{}{ "id": pipelineId, - "definition": command, "action": action, + + /* "autostart": map[string]bool{ "created": true, - "completed": true, - "failed": true, + "completed": false, + "failed": false, }, "autodelete": map[string]bool{ "completed": false, "failed": false, "stopped": false, }, + */ } requestBodyJSON, err := json.Marshal(requestBody) if err != nil { return "", err } - forwardData := bytes.NewBuffer(requestBodyJSON) + log.Printf("[INFO] Updating pipeline %s with action %s to ensure it starts. Body: %s", pipelineId, action, string(requestBodyJSON)) + + forwardData := bytes.NewBuffer(requestBodyJSON) req, err := http.NewRequest( forwardMethod, url, @@ -3478,7 +3492,7 @@ func deletePipeline(pipelineId string) error { log.Printf("[INFO] Pipeline with ID: %s deleted successfully", pipelineId) - pipelines = []shuffle.PipelineInfoMini{} + pipelines = []shuffle.PipelineInfo{} return nil } @@ -3739,7 +3753,7 @@ func removePath(containerName, path string) error { func sendPipelineHealthStatus() (shuffle.LakeConfig, error) { pipelinePayload := shuffle.LakeConfig{ Enabled: false, - Pipelines: []shuffle.PipelineInfoMini{}, + Pipelines: []shuffle.PipelineInfo{}, } if tenzirDisabled { @@ -3751,18 +3765,9 @@ func sendPipelineHealthStatus() (shuffle.LakeConfig, error) { if len(pipelines) == 0 || randint == 0 { pipelineDef, err := listPipelines() - if err == nil { - for _, pipeline := range pipelineDef { - pipelinePayload.Pipelines = append(pipelinePayload.Pipelines, shuffle.PipelineInfoMini{ - ID: pipeline.ID, - Name: pipeline.Name, - Definition: pipeline.Definition, - TotalRuns: pipeline.TotalRuns, - CreatedAt: pipeline.CreatedAt, - }) - } - - pipelines = pipelinePayload.Pipelines + if err == nil || len(pipelines) > 0 { + pipelines = pipelineDef + pipelinePayload.Pipelines = pipelines } } else { pipelinePayload.Pipelines = pipelines @@ -3775,7 +3780,7 @@ func sendPipelineHealthStatus() (shuffle.LakeConfig, error) { log.Printf("[ERROR] Tenzir node connection problem: %s", err) } else { - tenzirDisabled = true + //tenzirDisabled = true log.Printf("[WARNING] Disabling pipelines: %s. You will need to restart the Orborus to fix this.", err) } From 8821bed47f94c672b9cfcb86f19b530ee7f8bb95 Mon Sep 17 00:00:00 2001 From: Frikky Date: Mon, 6 Oct 2025 00:20:54 +0200 Subject: [PATCH 26/57] Bumped to 0.9.27 shared for orborus autobuild --- functions/onprem/orborus/go.mod | 4 ++-- functions/onprem/orborus/go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 1c6a7a5a..b6ccae82 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -4,13 +4,13 @@ go 1.24.0 toolchain go1.24.4 -replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared +//replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared require ( github.com/docker/docker v28.3.3+incompatible github.com/docker/go-connections v0.5.0 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.26 + github.com/shuffle/shuffle-shared v0.9.27 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 ) diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index f92997a8..a1ec5127 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -328,8 +328,8 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.9.26 h1:D7ZSnRGROtEP7eNprmYuc1wcxa+kV7bY5M+HFrOo990= -github.com/shuffle/shuffle-shared v0.9.26/go.mod h1:PhDEizuz4SmJaSmy0+yrFWwD1mXVUsy8/knKlrqF1qw= +github.com/shuffle/shuffle-shared v0.9.27 h1:YwyWXsp4fCOAPmc1DD+NNf9sVa4RHzp26SvWKxH4ytc= +github.com/shuffle/shuffle-shared v0.9.27/go.mod h1:PhDEizuz4SmJaSmy0+yrFWwD1mXVUsy8/knKlrqF1qw= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= From 92ab86f09220255e29aa3702d40f6866a8d49bcb Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Mon, 6 Oct 2025 16:12:45 +0530 Subject: [PATCH 27/57] fixed helm release ci/cd --- .github/workflows/helm-release.yml | 14 +++++++------- functions/kubernetes/charts/shuffle/values.yaml | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/helm-release.yml b/.github/workflows/helm-release.yml index 69b17917..0063be1a 100644 --- a/.github/workflows/helm-release.yml +++ b/.github/workflows/helm-release.yml @@ -24,13 +24,13 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Install apt dependencies - run: | - sudo apt-get install apt-transport-https -y --no-install-recommends - curl -fsSL https://packages.buildkite.com/helm-linux/helm-debian/gpgkey | gpg --dearmor | sudo tee /usr/share/keyrings/helm.gpg > /dev/null - echo "deb [signed-by=/usr/share/keyrings/helm.gpg] https://packages.buildkite.com/helm-linux/helm-debian/any/ any main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list - sudo apt-get update - sudo apt-get install helm -y --no-install-recommends + - name: Setup Helm + uses: azure/setup-helm@4 + with: + version: v3.18.4 # I run it locally so I know it works :b + + - name: Verify Helm + run: helm version - name: Set versions run: | diff --git a/functions/kubernetes/charts/shuffle/values.yaml b/functions/kubernetes/charts/shuffle/values.yaml index 4f46e482..f8c8aeff 100644 --- a/functions/kubernetes/charts/shuffle/values.yaml +++ b/functions/kubernetes/charts/shuffle/values.yaml @@ -968,7 +968,7 @@ orborus: image: registry: ghcr.io repository: shuffle/shuffle-orborus - tag: "" + tag: "nightly" digest: "" ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' From bb8a5642145b16af8750a09632386aa8737c2fb8 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Mon, 6 Oct 2025 16:14:50 +0530 Subject: [PATCH 28/57] fix: small typo in version --- .github/workflows/helm-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/helm-release.yml b/.github/workflows/helm-release.yml index 0063be1a..153bb9ee 100644 --- a/.github/workflows/helm-release.yml +++ b/.github/workflows/helm-release.yml @@ -25,7 +25,7 @@ jobs: uses: actions/checkout@v4 - name: Setup Helm - uses: azure/setup-helm@4 + uses: azure/setup-helm@v4 with: version: v3.18.4 # I run it locally so I know it works :b From 758fa9ebe15ddb7568a4c87b40c16791543a150b Mon Sep 17 00:00:00 2001 From: Frikky Date: Mon, 6 Oct 2025 12:46:19 +0200 Subject: [PATCH 29/57] More network attachments for Orborus <-> tenzir connectivity --- functions/onprem/orborus/orborus.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 457b36bc..47ec5c37 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -2947,13 +2947,19 @@ func deployTenzirNode() error { if dockerclient.IsErrNotFound(err) { // Create network if it doesn't exist networkName := "tenzir-network" - networkSubnet := "192.168.1.0/24" - networkGateway := "192.168.1.1" + networkSubnet := "192.168.102.0/24" + networkGateway := "192.168.102.1" err = createNetworkIfNotExists(ctx, networkName, networkSubnet, networkGateway) if err != nil { - log.Printf("[ERROR] Failed to create network: %s", err) - return err + log.Printf("[ERROR] Failed to create network %s: %s", networkName, err) + //return err + } + + // Trying to connect orborus to the tenzir network as well + err = dockercli.NetworkConnect(ctx, networkName, containerId, nil) + if err != nil { + log.Printf("[ERROR] Error connecting tenzir container to network: %s", err) } // Check if image exists @@ -3008,6 +3014,7 @@ func deployTenzirNode() error { if err != nil { log.Printf("[WARNING] Failed marshalling execution: %s", err) } + err = shuffle.SetCache(ctx, cacheKey, cacheData, 1) if err != nil { log.Printf("[WARNING] Failed updating cache for tenzir: %s", err) @@ -3125,7 +3132,7 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri EndpointsConfig: map[string]*network.EndpointSettings{ "tenzir-network": { IPAMConfig: &network.EndpointIPAMConfig{ - IPv4Address: "192.168.1.100", + IPv4Address: "192.168.102.100", }, }, }, From f850265c1e821bdd0dbf2333074f965564dc0098 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Mon, 6 Oct 2025 16:39:55 +0530 Subject: [PATCH 30/57] add a random sleep so no two worker check cache at same time --- functions/onprem/worker/worker.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 6797ff3c..752eca62 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -3874,6 +3874,12 @@ func sendAppRequest(ctx context.Context, incomingUrl, appName string, port int, // Checking as LATE as possible, ensuring we don't rerun what's already ran // ctx = context.Background() + + // Sleep between 0 and 250 ms for randomness so no same worker check at same time (same as cloud) + rand.Seed(time.Now().UnixNano()) + randMs := rand.Intn(250) + time.Sleep(time.Duration(randMs) * time.Millisecond) + newExecId := fmt.Sprintf("%s_%s", workflowExecution.ExecutionId, action.ID) _, err = shuffle.GetCache(ctx, newExecId) if err == nil { From 35a86c2e47aa111280c5253990dd5ea9b78a41e2 Mon Sep 17 00:00:00 2001 From: Frikky Date: Mon, 6 Oct 2025 14:20:45 +0200 Subject: [PATCH 31/57] More verbose details --- functions/onprem/orborus/orborus.go | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 47ec5c37..eb372e8a 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -3038,9 +3038,7 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri Healthcheck: healthconfig, ExposedPorts: nat.PortSet{ "5160/tcp": struct{}{}, - "514/udp": struct{}{}, "1514/udp": struct{}{}, - "514/tcp": struct{}{}, "1514/tcp": struct{}{}, }, Entrypoint: []string{containerName}, @@ -3087,8 +3085,6 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri hostConfig := &container.HostConfig{ PortBindings: nat.PortMap{ - "514/tcp": []nat.PortBinding{{HostPort: "514"}}, - "514/udp": []nat.PortBinding{{HostPort: "514"}}, "1514/tcp": []nat.PortBinding{{HostPort: "1514"}}, "1514/udp": []nat.PortBinding{{HostPort: "1514"}}, "5160/tcp": []nat.PortBinding{{HostPort: "5160"}}, @@ -3186,12 +3182,19 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri return err } - log.Printf("[INFO] Successfully deployed Tenzir Node! Setting up default syslog listener on UDP 514") + log.Printf("[INFO] Successfully deployed Tenzir Node! Setting up default syslog listener on TCP/1514 AND UDP/1514") - command := "from udp://0.0.0.0:514 read syslog | import" - _, err = createPipeline(command, "default-syslog-514") + command := `from "tcp://0.0.0.0:1514" { read_syslog } | import` + _, err = createPipeline(command, "default-syslog-tcp-514") if err != nil { - log.Printf("[ERROR] Failed to create default syslog pipeline: %s", err) + log.Printf("[ERROR] Failed to create tcp syslog pipeline: %s", err) + return nil + } + + command = `load_udp "0.0.0.0:1514", insert_newlines=true | read_syslog | import` + _, err = createPipeline(command, "default-syslog-udp-514") + if err != nil { + log.Printf("[ERROR] Failed to create udp syslog pipeline: %s", err) return nil } @@ -3645,12 +3648,12 @@ func extractZIP(zipFile, destDir string) error { } log.Printf("[DEBUG] Total size of the ZIP file: %d bytes", totalSize) - defer r.Close() if err := os.MkdirAll(destDir, 0755); err != nil { return err } + log.Printf("[DEBUG] Total files to extract: %d", len(r.File)) for _, f := range r.File { // Fix path traversal if strings.Contains(f.Name, "..") { @@ -3671,16 +3674,15 @@ func extractFile(f *zip.File, destDir string) error { if err != nil { return err } + defer rc.Close() - path := filepath.Join(destDir, f.Name) - out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) if err != nil { return err } - defer out.Close() + defer out.Close() _, err = io.Copy(out, rc) return err } From 79fd8fe3549b3e10ad990f79774587f6de9ee842 Mon Sep 17 00:00:00 2001 From: monilprajapati Date: Mon, 6 Oct 2025 18:48:50 +0530 Subject: [PATCH 32/57] File sync with cloud --- frontend/src/components/Billing.jsx | 178 ++ frontend/src/components/LicencePopup.jsx | 3118 +++++++++++----------- 2 files changed, 1728 insertions(+), 1568 deletions(-) diff --git a/frontend/src/components/Billing.jsx b/frontend/src/components/Billing.jsx index 8b14eaf5..3ed76123 100644 --- a/frontend/src/components/Billing.jsx +++ b/frontend/src/components/Billing.jsx @@ -87,6 +87,8 @@ const Billing = memo((props) => { const [currentIndex, setCurrentIndex] = useState(0); const [deleteAlertIndex, setDeleteAlertIndex] = useState(-1); const [deleteAlertVerification, setDeleteAlertVerification] = useState(false); + const [supportAppRunLimit, setSupportAppRunLimit] = useState(selectedOrganization?.billing?.internal_app_runs_hard_limit || ''); + const [supportLimitDialogOpen, setSupportLimitDialogOpen] = useState(false); const [isScale, setIsScale] = useState(false); const [currentTab, setCurrentTab] = useState(0) const [allChildOrgs, setAllChildOrgs] = useState([]) @@ -145,6 +147,9 @@ const Billing = memo((props) => { const findCurrentIndex = sortedAlertThresholds.some(threshold => threshold.Email_send === false); setCurrentIndex(findCurrentIndex ? sortedAlertThresholds.findIndex(threshold => threshold.Email_send === false) : - 1); + if (selectedOrganization?.Billing?.internal_app_runs_hard_limit !== undefined && selectedOrganization?.Billing?.internal_app_runs_hard_limit !== null && selectedOrganization?.Billing?.internal_app_runs_hard_limit > 0) { + setSupportAppRunLimit(selectedOrganization?.Billing?.internal_app_runs_hard_limit) + } }, [selectedOrganization]); @@ -1903,6 +1908,52 @@ const Billing = memo((props) => { setAlertThresholds([...alertThresholds, { percentage: '', count: '', Email_send: false }]); }; + + const handleUpdateSupportAppRunLimit = () => { + if (!supportAppRunLimit || isNaN(supportAppRunLimit) || supportAppRunLimit < 0) { + toast.error("Please enter a valid app run limit"); + return; + } + + toast("Updating app run limit. Please wait..."); + + const data = { + org_id: selectedOrganization.id, + editing: "internal_appruns_hard_limit", + billing: { + internal_app_runs_hard_limit: parseInt(supportAppRunLimit) || 0, + } + }; + + const url = globalUrl + "/api/v1/orgs/" + selectedOrganization.id; + fetch(url, { + mode: "cors", + method: "POST", + body: JSON.stringify(data), + credentials: "include", + crossDomain: true, + withCredentials: true, + headers: { + "Content-Type": "application/json; charset=utf-8", + }, + }) + .then((response) => { + if (response.status === 200) { + toast.success("Successfully updated app run limit"); + setSupportLimitDialogOpen(false); + if (handleGetOrg !== undefined) { + handleGetOrg(selectedOrganization.id); + } + } else { + toast.error("Failed to update app run limit. Please try again."); + } + }) + .catch((error) => { + console.log("Error updating app run limit:", error); + toast.error("Failed to update app run limit. Please try again."); + }); + }; + const updateAlertThreshold = (index, field, value) => { const totalValue = userdata.app_execution_limit; @@ -2025,6 +2076,13 @@ const Billing = memo((props) => { } }, [isChildOrg, currentTab]); + // Update supportAppRunLimit when selectedOrganization changes + useEffect(() => { + if (selectedOrganization?.billing?.internal_app_runs_hard_limit !== undefined) { + setSupportAppRunLimit(selectedOrganization.billing.internal_app_runs_hard_limit); + } + }, [selectedOrganization?.billing?.internal_app_runs_hard_limit]); + return (
@@ -2118,6 +2176,8 @@ const Billing = memo((props) => { globalUrl={globalUrl} selectedOrganization={selectedOrganization} billingInfo={billingInfo} + monthlyAppRunsParent={monthlyAppRunsParent} + monthlyAllSuborgExecutions={monthlyAllSuborgExecutions} isCloud={isCloud} userdata={userdata} stripeKey={stripeKey} @@ -2154,6 +2214,8 @@ const Billing = memo((props) => { isLoggedIn={isLoggedIn} globalUrl={globalUrl} selectedOrganization={selectedOrganization} + monthlyAppRunsParent={monthlyAppRunsParent} + monthlyAllSuborgExecutions={monthlyAllSuborgExecutions} billingInfo={billingInfo} isCloud={isCloud} userdata={userdata} @@ -2639,6 +2701,122 @@ const Billing = memo((props) => { Save + {userdata.support === true && ( +
+ + ⚠️ Support Only - App Run Limit Control + + + Note: Setting an app run hard limit below current usage will immediately stop all workflow executions for this organization. + + + Current app runs this month: {Number(monthlyAppRunsParent ?? 0) + Number(monthlyAllSuborgExecutions ?? 0)} / {userdata.app_execution_limit} + + + + + {/* Support Limit Dialog */} + setSupportLimitDialogOpen(false)} + maxWidth="sm" + fullWidth + PaperProps={{ + sx: { + borderRadius: theme?.palette?.DialogStyle?.borderRadius, + border: theme?.palette?.DialogStyle?.border, + minWidth: '440px', + fontFamily: theme?.typography?.fontFamily, + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + zIndex: 1000, + '& .MuiDialogContent-root': { + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + }, + '& .MuiDialogTitle-root': { + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + }, + '& .MuiDialogActions-root': { + backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, + } + } + }} + > + + ⚠️ Set App Run Limit + + + + Note: This will set a hard limit on app executions. If the organization reaches this limit, all workflow executions will be stopped. + + + Current usage: {Number(monthlyAppRunsParent ?? 0) + Number(monthlyAllSuborgExecutions ?? 0)} app runs this month + + + Current hard limit: {selectedOrganization?.billing?.internal_app_runs_hard_limit || 'Not set'} + + setSupportAppRunLimit(e.target.value)} + inputProps={{ min: 0 }} + style={{ marginTop: 10 }} + helperText="Set to 0 to completely disable app run hard limit" + /> + + + + + + +
+ )} +
): null} diff --git a/frontend/src/components/LicencePopup.jsx b/frontend/src/components/LicencePopup.jsx index 70e4cc6e..9d1dc0c0 100644 --- a/frontend/src/components/LicencePopup.jsx +++ b/frontend/src/components/LicencePopup.jsx @@ -1,133 +1,135 @@ -import React, { useState, useEffect, useContext } from "react"; -import ReactGA from 'react-ga4'; +import React, { useState, useEffect, useContext, useRef } from "react"; +import ReactGA from "react-ga4"; -import {getTheme} from "../theme.jsx"; -import countries from "../components/Countries.jsx"; +import { getTheme } from "../theme.jsx"; import { - Box, - Paper, - Typography, - Divider, - Button, - Grid, - Card, - Dialog, - DialogTitle, - DialogContent, - TextField, - InputAdornment, - IconButton, - Chip, - Checkbox, - Tooltip, - Slider, - DialogActions, - CardContent, - ButtonGroup, - DialogContentText, - ToggleButton, - ToggleButtonGroup, + Box, + Typography, + Divider, + Button, + Grid, + LinearProgress, + Dialog, + DialogTitle, + DialogContent, + TextField, + InputAdornment, + IconButton, + Chip, + Checkbox, + Tooltip, + DialogActions, + FormControlLabel, + Switch, + CircularProgress, + Skeleton, } from "@mui/material"; import { useNavigate, Link } from "react-router-dom"; -import { Autocomplete } from "@mui/material"; -import { toast } from "react-toastify" +import { toast } from "react-toastify"; import { Context } from "../context/ContextApi.jsx"; import { - Cached as CachedIcon, - ContentCopy as ContentCopyIcon, - Draw as DrawIcon, - Close as CloseIcon, - Done as DoneIcon, - Clear as ClearIcon, - AddTask as AddTaskIcon, + ContentCopy as ContentCopyIcon, + Draw as DrawIcon, + Close as CloseIcon, + Done as DoneIcon, } from "@mui/icons-material"; -import { typecost, typecost_single, } from "../views/HandlePaymentNew.jsx"; -import { handlePayasyougo } from "../views/HandlePaymentNew.jsx" -import Billing from "./Billing.jsx"; - +// This is the main component which shows the cards on Billing & Stats tab const LicencePopup = (props) => { - const { globalUrl, userdata, serverside, billingInfo, stripeKey, setModalOpen, isScale, isLoggedIn, isMobile, selectedOrganization, isCloud, features, licensePopup = false } = props; - //const alert = useAlert(); - let navigate = useNavigate(); - const [selectedDealModalOpen, setSelectedDealModalOpen] = React.useState(false); - const [dealList, setDealList] = React.useState([]); - const [dealName, setDealName] = React.useState(""); - const [dealAddress, setDealAddress] = React.useState(""); - const [dealType, setDealType] = React.useState("MSSP"); - const [dealCountry, setDealCountry] = React.useState("United States"); - const [dealCurrency, setDealCurrency] = React.useState("USD"); - const [dealStatus, setDealStatus] = React.useState("initiated"); - const [dealValue, setDealValue] = React.useState(""); - const [dealDiscount, setDealDiscount] = React.useState(""); - const [dealerror, setDealerror] = React.useState(""); - const [variant, setVariant] = useState(0) - const [shuffleVariant, setShuffleVariant] = useState(isCloud ? 0 : 1) - const [BillingEmail, setBillingEmail] = useState(selectedOrganization?.Billing?.Email); - const [openChangeEmailBox, setOpenChangeEmailBox] = useState(false); - // const parsedFields = maxFields === undefined ? 300 : maxFields - const initialShuffleVariant = isCloud ? 0 : 1; - const [paymentType, setPaymentType] = useState(0) - const [currentPrice, setCurrentPrice] = useState(129) - const [isLoaded, setIsLoaded] = useState(false) - const [errorMessage, setErrorMessage] = useState("") - const [highlight, setHighlight] = useState(false) + const { + globalUrl, + userdata, + serverside, + billingInfo, + stripeKey, + setModalOpen, + isScale, + isLoggedIn, + isMobile, + monthlyAppRunsParent, + monthlyAllSuborgExecutions, + selectedOrganization, + setSelectedOrganization, + isCloud, + features, + handleGetOrg, + licensePopup = false, + } = props; + //const alert = useAlert(); + let navigate = useNavigate(); + const [shuffleVariant, setShuffleVariant] = useState(isCloud ? 0 : 1); + const [BillingEmail, setBillingEmail] = useState( + selectedOrganization?.Billing?.Email + ); - const { themeMode } = useContext(Context); - const theme = getTheme(themeMode); + const { themeMode } = useContext(Context); + const theme = getTheme(themeMode); - // Cloud - const [calculatedApps, setCalculatedApps] = useState(600) - const [calculatedCost, setCalculatedCost] = useState("$600") - const [selectedValue, setSelectedValue] = useState(100) - useEffect(() => { - if(selectedOrganization?.Billing?.Email !== BillingEmail) { - setBillingEmail(selectedOrganization?.Billing?.Email); - } - }, [selectedOrganization]) + useEffect(() => { + if (selectedOrganization?.Billing?.Email !== BillingEmail) { + setBillingEmail(selectedOrganization?.Billing?.Email); + } + }, [selectedOrganization]); - // Onprem - const [calculatedCores, setCalculatedCores] = useState('600') - const [onpremSelectedValue, setOnpremSelectedValue] = useState(8) - const [billingCycle, setBillingCycle] = useState("annual") - const [scaleValue, setScaleValue] = useState( - new URLSearchParams(window.location.search).get("app_runs") || - (userdata?.app_execution_limit / 1000) + 50 || 10 - ); + const [billingCycle, setBillingCycle] = useState("annual"); + const [scaleValue, setScaleValue] = useState( + new URLSearchParams(window.location.search).get("app_runs") || + userdata?.app_execution_limit / 1000 + 50 || + 10 + ); + const [isLoading, setIsLoading] = useState(true); - useEffect(() => { - setScaleValue((userdata?.app_execution_limit / 1000) + 50 || 10) - }, [userdata]) + useEffect(() => { + setScaleValue(userdata?.app_execution_limit / 1000 + 50 || 10); + }, [userdata]); - const getPrice = (basePrice) => { - return Math.round(billingCycle === "annual" ? basePrice * 0.9 : basePrice); // 10% discount for annual - }; - - const stripe = typeof window === 'undefined' || window.location === undefined ? "" : props.stripeKey === undefined ? "" : window.Stripe ? window.Stripe(props.stripeKey) : "" + // Set loading state based on data availability + useEffect(() => { + if (selectedOrganization && userdata) { + // Add a small delay to show skeleton briefly for better UX + const timer = setTimeout(() => { + setIsLoading(false); + }, 500); + return () => clearTimeout(timer); + } + }, [selectedOrganization, userdata]); + + const getPrice = (basePrice) => { + return Math.round(billingCycle === "annual" ? basePrice * 0.9 : basePrice); // 10% discount for annual + }; + + const stripe = + typeof window === "undefined" || window.location === undefined + ? "" + : props.stripeKey === undefined + ? "" + : window.Stripe + ? window.Stripe(props.stripeKey) + : ""; // Handle slider change for Scale plan - const handleScaleChange = (event, newValue) => { - setScaleValue(newValue); + const handleScaleChange = (event, newValue) => { + setScaleValue(newValue); - // Add app runs to URL query params - const urlSearchParams = new URLSearchParams(window.location.search); - urlSearchParams.set("app_runs", newValue); // Convert to actual app runs (k to actual number) - const newUrl = `${window.location.pathname}?${urlSearchParams.toString()}`; - window.history.replaceState({}, "", newUrl); - }; + // Add app runs to URL query params + const urlSearchParams = new URLSearchParams(window.location.search); + urlSearchParams.set("app_runs", newValue); // Convert to actual app runs (k to actual number) + const newUrl = `${window.location.pathname}?${urlSearchParams.toString()}`; + window.history.replaceState({}, "", newUrl); + }; - // Handle billing cycle change - const handleBillingCycleChange = (event, newValue) => { - if (newValue !== null) { - setBillingCycle(newValue); + // Handle billing cycle change + const handleBillingCycleChange = (event, newValue) => { + if (newValue !== null) { + setBillingCycle(newValue); - if(isCloud){ + if (isCloud) { ReactGA.event({ - category: 'Billingpage', - action: 'Billing Cycle Changed', + category: "Billingpage", + action: "Billing Cycle Changed", label: `${billingCycle} -> ${newValue}`, }); } @@ -139,1479 +141,1459 @@ const LicencePopup = (props) => { window.location.pathname }?${urlSearchParams.toString()}`; window.history.replaceState({}, "", newUrl); - } + } + }; + + const payasyougo = "Pay as you go"; + + const paperStyle = { + padding: 20, + paddingBottom: 30, + borderRadius: theme.palette?.borderRadius, + height: "100%", + }; + + const userInScalePlan = userdata?.app_execution_limit > 2000; + + // These functions are being used for the dynamic features from the orgSyncFeatures + // Add this function to format the limit value + const formatLimit = (limit) => { + if (limit === null || limit === undefined || limit === 0) + return "Unlimited"; + if (typeof limit === "string" && limit.toLowerCase() === "unlimited") + return "Unlimited"; + if (typeof limit === "number") return limit.toLocaleString(); + return limit.toString(); + }; + + // Add this function to format the feature text with proper unlimited handling + const formatFeatureText = (feature, limit) => { + if (!feature) return ""; + + // Dynamic features that use limits + const featureMapping = { + app_executions: (limit) => { + const formattedLimit = formatLimit(limit); + return `Includes ${formattedLimit} App Executions per month`; + }, + multi_env: (limit) => { + const formattedLimit = formatLimit(limit); + return formattedLimit === "Unlimited" + ? "Unlimited Environments" + : `${formattedLimit} Environment${ + parseInt(formattedLimit) > 1 ? "s" : "" + }`; + }, + multi_tenant: (limit) => { + const formattedLimit = formatLimit(limit); + return formattedLimit === "Unlimited" + ? "Unlimited Tenants" + : `${formattedLimit} Tenant${ + parseInt(formattedLimit) > 1 ? "s" : "" + }`; + }, + multi_region: (limit) => { + const formattedLimit = formatLimit(limit); + return formattedLimit === "Unlimited" + ? "Unlimited Regions" + : `${formattedLimit} Region${ + parseInt(formattedLimit) > 1 ? "s" : "" + }`; + }, + webhook: (limit) => { + const formattedLimit = formatLimit(limit); + return formattedLimit === "Unlimited" + ? "Unlimited Webhooks" + : `${formattedLimit} Webhook${ + parseInt(formattedLimit) > 1 ? "s" : "" + }`; + }, + schedules: (limit) => { + const formattedLimit = formatLimit(limit); + return formattedLimit === "Unlimited" + ? "Unlimited Schedules" + : `${formattedLimit} Schedule${ + parseInt(formattedLimit) > 1 ? "s" : "" + }`; + }, + user_input: (limit) => { + const formattedLimit = formatLimit(limit); + return formattedLimit === "Unlimited" + ? "Unlimited User Inputs" + : `${formattedLimit} User Input${ + parseInt(formattedLimit) > 1 ? "s" : "" + }`; + }, + send_mail: (limit) => { + const formattedLimit = formatLimit(limit); + return formattedLimit === "Unlimited" + ? "Unlimited Emails per month" + : `${formattedLimit} Email${ + parseInt(formattedLimit) > 1 ? "s" : "" + } per month`; + }, + send_sms: (limit) => { + const formattedLimit = formatLimit(limit); + return formattedLimit === "Unlimited" + ? "Unlimited SMS per month" + : `${formattedLimit} SMS${ + parseInt(formattedLimit) > 1 ? "s" : "" + } per month`; + }, + email_trigger: (limit) => { + const formattedLimit = formatLimit(limit); + return formattedLimit === "Unlimited" + ? "Unlimited Email Triggers" + : `${formattedLimit} Email Trigger${ + parseInt(formattedLimit) > 1 ? "s" : "" + }`; + }, + notifications: (limit) => { + const formattedLimit = formatLimit(limit); + return formattedLimit === "Unlimited" + ? "Unlimited Notifications" + : `${formattedLimit} Notification${ + parseInt(formattedLimit) > 1 ? "s" : "" + }`; + }, + workflows: (limit) => { + const formattedLimit = formatLimit(limit); + return formattedLimit === "Unlimited" + ? "Unlimited Workflows" + : `${formattedLimit} Workflow${ + parseInt(formattedLimit) > 1 ? "s" : "" + }`; + }, + autocomplete: (limit) => { + const formattedLimit = formatLimit(limit); + return formattedLimit === "Unlimited" + ? "Unlimited Autocomplete" + : `${formattedLimit} Autocomplete${ + parseInt(formattedLimit) > 1 ? "s" : "" + }`; + }, + workflow_executions: (limit) => { + const formattedLimit = formatLimit(limit); + return formattedLimit === "Unlimited" + ? "Unlimited Workflow Executions" + : `${formattedLimit} Workflow Execution${ + parseInt(formattedLimit) > 1 ? "s" : "" + }`; + }, + authentication: (limit) => { + const formattedLimit = formatLimit(limit); + return formattedLimit === "Unlimited" + ? "Unlimited Authentication" + : `${formattedLimit} Authentication${ + parseInt(formattedLimit) > 1 ? "s" : "" + }`; + }, + shuffle_gpt: (limit) => { + const formattedLimit = formatLimit(limit); + return formattedLimit === "Unlimited" + ? "Unlimited Shuffle GPT" + : `${formattedLimit} Shuffle GPT${ + parseInt(formattedLimit) > 1 ? "s" : "" + }`; + }, }; - const payasyougo = "Pay as you go" - - const paperStyle = { - padding: 20, - paddingBottom: 30, - borderRadius: theme.palette?.borderRadius, - height: "100%", - - } - - const userInScalePlan = userdata?.app_execution_limit > 2000 - const appRuns = (userdata?.app_execution_limit / 1000) + "K App Runs" - - // Add this function to format the limit value - const formatLimit = (limit) => { - if (limit === null || limit === undefined || limit === 0) return "Unlimited"; - if (typeof limit === "string" && limit.toLowerCase() === "unlimited") return "Unlimited"; - if (typeof limit === "number") return limit.toLocaleString(); - return limit.toString(); - }; - - // Add this function to format the feature text with proper unlimited handling - const formatFeatureText = (feature, limit) => { - if (!feature) return ""; - - // Dynamic features that use limits - const featureMapping = { - app_executions: (limit) => { - const formattedLimit = formatLimit(limit); - return `Includes ${formattedLimit} App Executions per month`; - }, - multi_env: (limit) => { - const formattedLimit = formatLimit(limit); - return formattedLimit === "Unlimited" - ? "Unlimited Environments" - : `${formattedLimit} Environment${parseInt(formattedLimit) > 1 ? 's' : ''}`; - }, - multi_tenant: (limit) => { - const formattedLimit = formatLimit(limit); - return formattedLimit === "Unlimited" - ? "Unlimited Tenants" - : `${formattedLimit} Tenant${parseInt(formattedLimit) > 1 ? 's' : ''}`; - }, - multi_region: (limit) => { - const formattedLimit = formatLimit(limit); - return formattedLimit === "Unlimited" - ? "Unlimited Regions" - : `${formattedLimit} Region${parseInt(formattedLimit) > 1 ? 's' : ''}`; - }, - webhook: (limit) => { - const formattedLimit = formatLimit(limit); - return formattedLimit === "Unlimited" - ? "Unlimited Webhooks" - : `${formattedLimit} Webhook${parseInt(formattedLimit) > 1 ? 's' : ''}`; - }, - schedules: (limit) => { - const formattedLimit = formatLimit(limit); - return formattedLimit === "Unlimited" - ? "Unlimited Schedules" - : `${formattedLimit} Schedule${parseInt(formattedLimit) > 1 ? 's' : ''}`; - }, - user_input: (limit) => { - const formattedLimit = formatLimit(limit); - return formattedLimit === "Unlimited" - ? "Unlimited User Inputs" - : `${formattedLimit} User Input${parseInt(formattedLimit) > 1 ? 's' : ''}`; - }, - send_mail: (limit) => { - const formattedLimit = formatLimit(limit); - return formattedLimit === "Unlimited" - ? "Unlimited Emails per month" - : `${formattedLimit} Email${parseInt(formattedLimit) > 1 ? 's' : ''} per month`; - }, - send_sms: (limit) => { - const formattedLimit = formatLimit(limit); - return formattedLimit === "Unlimited" - ? "Unlimited SMS per month" - : `${formattedLimit} SMS${parseInt(formattedLimit) > 1 ? 's' : ''} per month`; - }, - email_trigger: (limit) => { - const formattedLimit = formatLimit(limit); - return formattedLimit === "Unlimited" - ? "Unlimited Email Triggers" - : `${formattedLimit} Email Trigger${parseInt(formattedLimit) > 1 ? 's' : ''}`; - }, - notifications: (limit) => { - const formattedLimit = formatLimit(limit); - return formattedLimit === "Unlimited" - ? "Unlimited Notifications" - : `${formattedLimit} Notification${parseInt(formattedLimit) > 1 ? 's' : ''}`; - }, - workflows: (limit) => { - const formattedLimit = formatLimit(limit); - return formattedLimit === "Unlimited" - ? "Unlimited Workflows" - : `${formattedLimit} Workflow${parseInt(formattedLimit) > 1 ? 's' : ''}`; - }, - autocomplete: (limit) => { - const formattedLimit = formatLimit(limit); - return formattedLimit === "Unlimited" - ? "Unlimited Autocomplete" - : `${formattedLimit} Autocomplete${parseInt(formattedLimit) > 1 ? 's' : ''}`; - }, - workflow_executions: (limit) => { - const formattedLimit = formatLimit(limit); - return formattedLimit === "Unlimited" - ? "Unlimited Workflow Executions" - : `${formattedLimit} Workflow Execution${parseInt(formattedLimit) > 1 ? 's' : ''}`; - }, - authentication: (limit) => { - const formattedLimit = formatLimit(limit); - return formattedLimit === "Unlimited" - ? "Unlimited Authentication" - : `${formattedLimit} Authentication${parseInt(formattedLimit) > 1 ? 's' : ''}`; - }, - shuffle_gpt: (limit) => { - const formattedLimit = formatLimit(limit); - return formattedLimit === "Unlimited" - ? "Unlimited Shuffle GPT" - : `${formattedLimit} Shuffle GPT${parseInt(formattedLimit) > 1 ? 's' : ''}`; - }, - }; - - try { - // Check if we have a mapping for this feature - const formatter = featureMapping[feature]; - if (formatter) { - return formatter(limit); - } - - // Default format for unknown features - const formattedLimit = formatLimit(limit); - return `${feature}: ${formattedLimit}`; - } catch (error) { - console.warn(`Error formatting feature ${feature}:`, error); - return `${feature}: ${formatLimit(limit)}`; - } - }; - - - // Update the subscription features section - billingInfo.subscription = { - "active": true, - "name": appRuns, - "price": userInScalePlan ? "" : "Free", - "currency": userInScalePlan ? "" : "Free", - "currency_text": "", - "interval": "", - "description": "", - "features": userInScalePlan ? [ - // Add static features first - ...(userInScalePlan ? ["Standard Email Support"] : []), - - // Then add dynamic features from the database - ...Object.entries(features || {}) - .filter(([_, featureData]) => { - return featureData && - typeof featureData === 'object' && - featureData.active === true; - }) - .map(([featureName, featureData]) => { - try { - return formatFeatureText(featureName, featureData?.limit); - } catch (error) { - console.warn(`Error processing feature ${featureName}:`, error); - return ""; - } - }) - .filter(feature => - feature.length > 0 && - !feature.toLowerCase().includes('unlimited') // Add this filter to remove "unlimited" features - ) - ] : [ - userInScalePlan ? "Standard Email Support" : "Community Support", - userInScalePlan ? `Includes ${appRuns}. ` : `Includes ${appRuns} for free. `, - userInScalePlan ? "Multi-Tenant & Multi-Region" : "Get all 2500+ Apps and 10 Workflows", - userInScalePlan ? "All features included in the Scale plan" : "Invite up to 5 users" - ], - "limit": userInScalePlan ? userdata?.app_execution_limit : 10000, - } - - const sendSignatureRequest = (subscription) => { - const url = `${globalUrl}/api/v1/orgs/${selectedOrganization.id}`; - - fetch(url, { - body: JSON.stringify({ - org_id: selectedOrganization.id, - subscription: subscription, - }), - mode: "cors", - method: "POST", - credentials: "include", - crossDomain: true, - withCredentials: true, - headers: { - "Content-Type": "application/json; charset=utf-8", - }, - }) - .then((response) => { - if (response.status !== 200) { - console.log("Error in response"); - } - return response.json(); - }) - .then((responseJson) => { - console.log("Response from signature request: ", responseJson); - }) - .catch((error) => { - console.log("Error: ", error); - }) - } - - // Create a function to remove duplicates and merge features - const mergeUniqueFeatures = (existingFeatures, newFeatures) => { - // Convert arrays to Sets to remove duplicates - const uniqueFeatures = new Set([ - ...(existingFeatures || []), - ...(newFeatures || []) - ]); - return Array.from(uniqueFeatures); - }; - - const SubscriptionObject = (props) => { - const { globalUrl, index, userdata, serverside, billingInfo, stripeKey, selectedOrganization, handleGetOrg, subscription, highlight, } = props; - - const [signatureOpen, setSignatureOpen] = React.useState(false); - const [tosChecked, setTosChecked] = React.useState(subscription.eula_signed) - const [hovered, setHovered] = React.useState(false) - const [newBillingEmail, setNewBillingEmail] = useState(''); - var top_text = userInScalePlan ? "Scale Plan" : "Starter Plan" - // if (subscription.limit === undefined && subscription.level === undefined || subscription.level === null || subscription.level === 0) { - // subscription.name = "Enterprise" - // subscription.currency_text = "$" - // subscription.price = subscription.level * 180 - // subscription.limit = subscription.level * 100000 - // subscription.interval = subscription.recurrence - // subscription.features = [ - // "Includes " + subscription.limit + " app runs/month. ", - // "Multi-Tenancy and Region-Selection", - // "And all other features from /pricing", - // ] - // } - - // if (userdata?.app_execution_limit >= 300000) { - // subscription.name = "Enterprise" - // subscription.currency_text = "$" - // subscription.price = typecost_single - // subscription.limit = userdata?.app_execution_limit - // subscription.interval = "app run / month" - // subscription.features = [ - // "Includes " + (userdata?.app_execution_limit/1000) + "K app runs/month. ", - // "Multi-Tenancy and Region-Selection", - // "And all other features from /pricing", - // ] - // } - - - var newPaperstyle = JSON.parse(JSON.stringify(paperStyle)) - if (subscription.name === "Enterprise" && subscription.active === true) { - top_text = "Enterprise Plan" - - // newPaperstyle.border = "1px solid #f85a3e" - } - - var showSupport = false - if (subscription.name.includes("default")) { - top_text = "Custom Contract" - // newPaperstyle.border = "1px solid #f85a3e" - showSupport = true - } - - if (subscription.name.includes("App Run Units")) { - top_text = "Scale Plan" - showSupport = true - } - - if (userdata?.app_execution_limit >= 300000) { - top_text = "Enterprise Plan" - } - - if (subscription.name.includes("Open Source")) { - top_text = "Open Source" - showSupport = true - } - - if (subscription.name.includes("Scale")) { - top_text = "Scale access" - } - - if (highlight === true) { - // Add an "Upgrade now" button - // newPaperstyle.border = "1px solid #f85a3e" - } - - const handleClickOpen = () => { - setOpenChangeEmailBox(true); - }; - - const HandleChangeBillingEmail = (orgId) => { - const email = newBillingEmail; - const emailPattern = /^[a-zA-Z0-9._-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}$/; - if (!emailPattern.test(email)) { - toast("Please enter a valid email address"); - return; - } else { - setNewBillingEmail(email); - } - - toast("Updating billing email. Please Wait") - - const data = { - org_id: orgId, - email: newBillingEmail, - billing: { - email: newBillingEmail, - }, - }; - - const url = `${globalUrl}/api/v1/orgs/${orgId}/billing`; - fetch(url, { - method: "POST", - body: JSON.stringify(data), - credentials: "include", - headers: { - "Content-Type": "application/json", - }, - }) - .then((response) => { - if (response.status !== 200) { - console.log("Bad status code in get org:", response.status); - } - return response.json(); - }).then((responseJson) => { - if (responseJson.success === true) { - toast.success("Successfully updated billing email"); - setBillingEmail(newBillingEmail); - setOpenChangeEmailBox(false); - } else { - toast.error("Failed to update billing email. Please try again."); - } - }) - .catch((error) => { - console.log("Error getting org:", error); - }); - } - - - const extraFeatures = Object.entries(features || {}) - .filter(([_, featureData]) => { - return featureData && - typeof featureData === 'object' && - featureData.active === true; - }) - .map(([featureName, featureData]) => { - return formatFeatureText(featureName, featureData?.limit); - }) - .filter(feature => - feature.length > 0 && - !feature.toLowerCase().includes('unlimited') // Add this filter to remove "unlimited" features - ) - - subscription.features = mergeUniqueFeatures(subscription.features, extraFeatures); - - return ( - -
-
setHovered(true)} - // onMouseLeave={() => setHovered(false)} - > - - - { - e.preventDefault(); - setSignatureOpen(false); - setTosChecked(false) - }} - > - - - - Read and Accept the EULA - - - { - setTosChecked(e.target.checked) - }} - inputProps={{ 'aria-label': 'primary checkbox' }} - /> - { - setTosChecked(!tosChecked) - }}> - Accept - - - By clicking the “accept” button, you are signing the document, electronically agreeing that it has the same legal validity and effects as a handwritten signature, and that you have the competent authority to represent and sign on behalf an entity. Need support or have questions? Contact us at support@shuffler.io. - - -
- -
-
-
- {subscription.active === true && !isScale && } -
- {top_text === "Base Cloud Access" && userdata.has_card_available === true && !isScale ? - { - console.log("Clicked chip") - }} - variant="outlined" - color="primary" - /> - : null} - - {top_text} - - - {top_text === "Base Cloud Access" && userdata.has_card_available === false ? - - : null} - {isCloud && highlight === true && top_text !== "Starter Plan" ? - - { - setSignatureOpen(true) - }} - > - - - - : null} -
- -
- - {subscription.name} - - - {subscription.currency_text !== undefined ? -
- - {subscription.currency_text}{subscription.price} - - - {subscription.interval.length > 0 ? `/ ${subscription.interval}` : ""} - -
- : null} - - - Features - -
    - {subscription.features !== undefined && subscription.features !== null ? - subscription.features.map((feature, index) => { - var parsedFeature = feature - if (feature.includes("Documentation: ")) { - parsedFeature = - - Documentation to get started - - } - - if (feature.includes("Worker License: ")) { - const fieldId = "webhook_uri_field_" + index - parsedFeature = - - - Use the {feature.split("Worker License: ")[0]} Worker - - { }} - InputProps={{ - endAdornment: - - { - var copyText = document.getElementById(fieldId); - if (copyText !== undefined && copyText !== null) { - console.log("NAVIGATOR: ", navigator); - const clipboard = navigator.clipboard; - if (clipboard === undefined) { - toast("Can only copy over HTTPS (port 3443)"); - return; - } - - navigator.clipboard.writeText(copyText.value); - copyText.select(); - copyText.setSelectionRange( - 0, - 99999 - ); /* For mobile devices */ - - /* Copy the text inside the text field */ - document.execCommand("copy"); - toast("Copied Webhook URL"); - } else { - console.log("Couldn't find webhook URI field: ", copyText); - } - }} - edge="end" - > - - - - }} - fullWidth - /> - - } - - return ( -
  • - - {parsedFeature} - -
  • - ) - }) - : null} -
- - { - isCloud ? - userdata?.app_execution_limit && userdata?.app_execution_limit !== 10000 ? - `You have already subscribed to the ${top_text}, which includes ${userdata?.app_execution_limit/1000}K app runs/month. You can increase the limit by upgrading current plan. Contact support@shuffler.io for more information.` : - `You are using free Starter plan with max ${userdata?.app_execution_limit === 10000 ? "10,000" : "2,000"} runs per month. Upgrade to increase this limit.` - - : - `You are not subscribed to any plan and are using the free, open source plan. This plan has no enforced limits, but scale issues may occur due to CPU congestion.` - } - - {/* {isCloud && (userdata.has_card_available === true || selectedOrganization?.Billing?.Email?.length > 0 )? -
- Billing email: {BillingEmail} - - {setOpenChangeEmailBox(false)}} - PaperProps={{ - sx: { - borderRadius: theme?.palette?.DialogStyle?.borderRadius, - border: theme?.palette?.DialogStyle?.border, - minWidth: '440px', - fontFamily: theme?.typography?.fontFamily, - backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, - zIndex: 1000, - '& .MuiDialogContent-root': { - backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, - }, - '& .MuiDialogTitle-root': { - backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, - }, - '& .MuiDialogActions-root': { - backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, - }, - } - }} - > - Change Billing Email - - - Enter the new billing email address. - - { if (event.key === 'Enter') HandleChangeBillingEmail(selectedOrganization.id) }} - onChange={(e) => setNewBillingEmail(e.target.value)} - /> - - - - - - -
- : null} */} -
- {isCloud ? ( - ) : null} - - -
- {/* -
- - Schedule Call Now -
- */} - -
-
- ) - } - - // useEffect(() => { - // console.log("New variant: ", shuffleVariant) - - // if (shuffleVariant === 1) { - // setCalculatedCost("$960") - // setSelectedValue(8) - // } else { - // if (userdata && userdata?.app_execution_limit) { - // if (userdata.app_execution_limit >= 30000 && userdata.app_execution_limit < 40000) { - // setSelectedValue(400) - // setCalculatedCost("$1280") - // }else if (userdata?.app_execution_limit >= 40000 && userdata?.app_execution_limit < 50000) { - // setSelectedValue(500) - // setCalculatedCost("$1600") - // } else if (userdata?.app_execution_limit >= 500000 && userdata?.app_execution_limit < 600000) { - // setSelectedValue(600) - // setCalculatedCost("$1920") - // } else if (userdata?.app_execution_limit >= 60000 && userdata?.app_execution_limit < 70000) { - // setSelectedValue(700) - // setCalculatedCost("$2240") - // } else if (userdata?.app_execution_limit >= 70000 && userdata?.app_execution_limit < 80000) { - // setSelectedValue(800) - // setCalculatedCost("$2560") - // } else if (userdata?.app_execution_limit >= 80000 && userdata?.app_execution_limit < 90000) { - // setSelectedValue(900) - // setCalculatedCost("$2880") - // }else { - // setCalculatedCost("$960") - // setSelectedValue(300) - // } - // }else { - // setCalculatedCost("$960") - // setSelectedValue(300) - // } - // } - // }, [userdata]) - - if (typeof window === 'undefined' || window.location === undefined) { - return null - } - - const setMonthlyCost = (variant, paymentType) => { - setErrorMessage("") - if (variant === 0 && paymentType === 0) { - setCurrentPrice(129) - } else if (variant === 0 && paymentType === 1) { - setCurrentPrice(155) - } else if (variant === 1 && paymentType === 0) { - setCurrentPrice(999) - } else if (variant === 1 && paymentType === 1) { - setCurrentPrice(1199) - } else if (variant === 2 && paymentType === 0) { - setCurrentPrice(15) - } else if (variant === 2 && paymentType === 1) { - setCurrentPrice(18) - } - } - - const handleChange = (event, newValue) => { - - if (shuffleVariant === 1) { - setSelectedValue(newValue) - if (newValue === 32) { - setCalculatedCost(`Get A Quote`) - } else { - setCalculatedCost(`$${newValue * 120}`) - } - } else { - setSelectedValue(newValue) - if (newValue < 300) { - setCalculatedCost(`Pay as you go`) - } else if (newValue === 1000) { - setCalculatedCost(`Get A Quote`) - } else { - setCalculatedCost(`$${newValue * 1000 * typecost}`) - } - } - } - - if (!isLoaded) { - setIsLoaded(true) - - const tmpsearch = typeof window === 'undefined' || window.location === undefined ? "" : window.location.search - const tmpVar = new URLSearchParams(tmpsearch).get("variant") - if (tmpVar !== undefined && tmpVar !== null && tmpVar < 3) { - setVariant(parseInt(tmpVar)) - } - - const tmpType = new URLSearchParams(tmpsearch).get("payment_type") - if (tmpType !== undefined && tmpType !== null && tmpType < 2) { - setPaymentType(parseInt(tmpType)) - } - - const modal = new URLSearchParams(tmpsearch).get("payment_modal") - if (modal !== undefined && modal !== null && modal === "open") { - setModalOpen(true) - } - - const tmpView = new URLSearchParams(tmpsearch).get("view") - if (tmpView !== undefined && tmpView !== null && tmpView === "failure") { - setErrorMessage("Something went wrong with your payment. Please try again.") - } - - const urlSearchParams = new URLSearchParams(window.location.search); - const params = Object.fromEntries(urlSearchParams.entries()); - const foundTab = params["tab"]; - if (foundTab !== null && foundTab !== undefined) { - if (foundTab === "onprem") { - setShuffleVariant(1); - } else if (foundTab === "cloud") { - setShuffleVariant(0); - } - } - - const foundHighlight = params["highlight"]; - if (foundHighlight !== null && foundHighlight !== undefined) { - setHighlight(true) - } - } - - //const skipFreemode = window.location.pathname.startsWith("/admin") - const skipFreemode = false - const maxwidth = isMobile ? "91%" : skipFreemode ? 1100 : 1200 - const activeIcon = - const inActiveIcon = - const defaultTaskIcon = - - const buttonBackground = "linear-gradient(to right, #f86a3e, #f34079)" - const level1Button = - - - const level2Button = - - - const level3Button = skipFreemode ? null : - - - - const cardStyle = { - // height: "100%", - // width: "100%", - // textAlign: "center", - color: "white", - } - - // const isLoggedInHandler = () => { - // if (calculatedCost === payasyougo) { - // handlePayasyougo(props.userdata) - // return - // } - - // const priceItem = - // window.location.origin === "https://shuffler.io/" || "https://sandbox.shuffler.io/" - // ? shuffleVariant === 0 - // ? "price_1PWI3uDzMUgUjxHSffUBwWCy" - // : "price_1PWI8EDzMUgUjxHSfEhUB7oL" - - // : shuffleVariant === 0 - // ? "price_1PZPSSEJjT17t98NLJoTMYja" - // : "price_1PZPQuEJjT17t98N3yORUtd9"; - - // const successUrl = `${window.location.origin}/admin?admin_tab=billingstats&payment=success` - // const failUrl = `${window.location.origin}/admin?admin_tab=billingstats&payment=failure` - // const quantity = shuffleVariant === 0 ? selectedValue / 100 : selectedValue - - // console.log("Priceitem: ", priceItem, quantity, shuffleVariant) - // var checkoutObject = { - // lineItems: [ - // { - // price: priceItem, - // quantity: quantity, - // }, - // ], - // mode: "subscription", - // billingAddressCollection: "auto", - // successUrl: successUrl, - // cancelUrl: failUrl, - // clientReferenceId: props.userdata.active_org.id, - // } - - // if (stripe === undefined || stripe === null || stripe.redirectToCheckout === undefined) { - // window.open("https://shuffler.io/admin?admin_tab=billingstats&payment=stripe_error", "_self") - // } - - // stripe.redirectToCheckout(checkoutObject) - // .then(function (result) { - // console.log("SUCCESS STRIPE?: ", result) - - // ReactGA.event({ - // category: "pricing", - // action: "add_card_success", - // label: "", - // }) - // }) - // .catch(function (error) { - // console.error("STRIPE ERROR: ", error) - - // ReactGA.event({ - // category: "pricing", - // action: "add_card_error", - // label: "", - // }) - // }) - // } - - const isLoggedInHandler = () => { - var priceItem; - if (window.location.origin === "https://shuffler.io" || window.location.origin === "https://sandbox.shuffler.io") { - priceItem = billingCycle === "monthly" ? "price_1R66rbEJjT17t98NHIQ78nrz" : "price_1R671UEJjT17t98NzfqWvSG7" - } else if (window.location.origin === "http://localhost:3002") { - priceItem = billingCycle === "monthly" ? "price_1R678hEJjT17t98Nai5J50gs" : "price_1R6c84EJjT17t98NR68gUfT7" + try { + // Check if we have a mapping for this feature + const formatter = featureMapping[feature]; + if (formatter) { + return formatter(limit); } - - const successUrl = `${window.location.origin}/admin?admin_tab=billingstats&payment=success`; - const failUrl = `${window.location.origin}/pricing?admin_tab=billingstats&payment=failure`; - - let quantity; - - if (billingCycle === "monthly") { - quantity = scaleValue / 10 - } else { - quantity = (scaleValue / 10) * 12 + + // Default format for unknown features + const formattedLimit = formatLimit(limit); + return `${feature}: ${formattedLimit}`; + } catch (error) { + console.warn(`Error formatting feature ${feature}:`, error); + return `${feature}: ${formatLimit(limit)}`; + } + }; + + // Send signature request to backend + const sendSignatureRequest = (subscription) => { + const url = `${globalUrl}/api/v1/orgs/${selectedOrganization.id}`; + + fetch(url, { + body: JSON.stringify({ + org_id: selectedOrganization.id, + editing: "subscription_update", + subscription_index: 0, + subscription: subscription, + }), + mode: "cors", + method: "POST", + credentials: "include", + crossDomain: true, + withCredentials: true, + headers: { + "Content-Type": "application/json; charset=utf-8", + }, + }) + .then((response) => { + if (response.status !== 200) { + console.log("Error in response"); } - - redirectToCheckout(priceItem, quantity, successUrl, failUrl); - }; - - const redirectToCheckout = (priceItem, quantity, successUrl, failUrl) => { - const checkoutObject = { - lineItems: [ - { - price: priceItem, - quantity: quantity, - }, - ], - mode: "subscription", - billingAddressCollection: "auto", - successUrl: successUrl, - cancelUrl: failUrl, - clientReferenceId: userdata.active_org.id, + return response.json(); + }) + .then((responseJson) => { + console.log("Response from signature request: ", responseJson); + if (typeof handleGetOrg === "function") { + handleGetOrg(selectedOrganization.id); + } + }) + .catch((error) => { + console.log("Error: ", error); + }); + }; + + // Create a function to remove duplicates and merge features + const mergeUniqueFeatures = (existingFeatures, newFeatures) => { + // Convert arrays to Sets to remove duplicates + const uniqueFeatures = new Set([ + ...(existingFeatures || []), + ...(newFeatures || []), + ]); + return Array.from(uniqueFeatures); + }; + + // This is the dialog for editing subscription with better UX + const EditSubscriptionDialog = ({ + open, + onClose, + subscription, + globalUrl, + selectedOrganization, + onSaved, + }) => { + const initialForm = { + name: subscription?.name || "", + active: !!subscription?.active, + support_level: subscription?.support_level || "", + recurrence: subscription?.recurrence || "month", + amount: subscription?.amount || "", + currency: subscription?.currency || "USD", + level: subscription?.level || "", + limit: subscription?.limit || 0, + startdate: subscription?.startdate || 0, + enddate: subscription?.enddate || 0, + cancellationdate: subscription?.cancellationdate || 0, + features: Array.isArray(subscription?.features) + ? subscription.features + : [], + eula: subscription?.eula, + eula_signed: subscription?.eula_signed, + eula_signed_by: subscription?.eula_signed_by, + reference: subscription?.reference, + }; + const [form, setForm] = useState(initialForm); + + useEffect(() => { + if (!open) return; + setForm(initialForm); + setFeaturesMarkdown(featuresToMarkdown(subscription?.features)); + setErrors({}); + }, [open, subscription]); + + const handleCancel = () => { + setForm(initialForm); + setFeaturesMarkdown(featuresToMarkdown(subscription?.features)); + setErrors({}); + onClose?.(); + }; + + const toInputDate = (epoch) => { + if (!epoch || isNaN(epoch)) return ""; + try { + return new Date(epoch * 1000).toISOString().slice(0, 10); + } catch (e) { + return ""; + } + }; + const toEpoch = (dateStr) => { + if (!dateStr) return 0; + const ms = Date.parse(dateStr); + return isNaN(ms) ? 0 : Math.floor(ms / 1000); + }; + + const [errors, setErrors] = useState({}); + const [saving, setSaving] = useState(false); + + // Simple helper: Convert features array <-> markdown list + const featuresToMarkdown = (arr) => { + const list = Array.isArray(arr) ? arr : []; + return list + .map((line) => { + const text = String(line || ""); + // If already looks like a list item, keep as-is + if (/^\s*-\s+/.test(text)) return text; + return `- ${text}`; + }) + .join("\n"); + }; + + const markdownToFeatures = (markdown) => { + if (!markdown) return []; + return String(markdown) + .split("\n") + .map((raw) => raw.replace(/\s+$/, "")) + .filter(Boolean) + .map((line) => { + // Keep indentation depth of multiples of two spaces before dash + const m = line.match(/^(\s*)-\s+(.*)$/); + if (!m) { + return line.trim(); + } + const indent = m[1] || ""; + const text = m[2] || ""; + return `${indent}- ${text}`.trimEnd(); + }); + }; + + const [featuresMarkdown, setFeaturesMarkdown] = useState( + featuresToMarkdown(subscription?.features) + ); + const featuresInputRef = useRef(null); + + // Handle tab indentation for markdown textarea + const handleFeaturesKeyDown = (e) => { + if (e.key !== "Tab") return; + + const textarea = featuresInputRef.current; + if (!textarea) return; + + e.preventDefault(); + + const { selectionStart, selectionEnd } = textarea; + const text = featuresMarkdown; + + // Find the start and end of the current line(s) + const lineStart = text.lastIndexOf("\n", selectionStart - 1) + 1; + const lineEnd = text.indexOf("\n", selectionEnd); + const actualLineEnd = lineEnd === -1 ? text.length : lineEnd; + + // Get the selected lines + const selectedText = text.slice(lineStart, actualLineEnd); + const lines = selectedText.split("\n"); + + // Apply indentation + const indent = " "; // 2 spaces + const newLines = lines.map(line => { + if (e.shiftKey) { + // Shift+Tab: remove indentation + if (line.startsWith(indent)) { + return line.slice(indent.length); + } + if (line.startsWith(" ")) { + return line.slice(1); + } + return line; + } else { + // Tab: add indentation + return `${indent}${line}`; + } + }); + + // Update the text + const newText = + text.slice(0, lineStart) + + newLines.join("\n") + + text.slice(actualLineEnd); + + setFeaturesMarkdown(newText); + + // Update cursor position + const indentChange = e.shiftKey ? -indent.length : indent.length; + const newSelectionStart = Math.max(lineStart, selectionStart + indentChange); + const newSelectionEnd = Math.max(lineStart, selectionEnd + (indentChange * lines.length)); + + // Use requestAnimationFrame for better performance than setTimeout + requestAnimationFrame(() => { + try { + textarea.selectionStart = newSelectionStart; + textarea.selectionEnd = newSelectionEnd; + } catch (error) { + // Ignore selection errors + } + }); + }; + + const validate = () => { + const next = {}; + if (!form.name || form.name.trim().length === 0) + next.name = "Name is required"; + if (form.amount !== "" && Number.isNaN(Number(form.amount))) + next.amount = "Amount must be a number"; + if (form.limit !== "" && Number.isNaN(Number(form.limit))) + next.limit = "Limit must be a number"; + if (form.startdate && form.enddate && form.enddate < form.startdate) + next.enddate = "End date must be after start date"; + if (!form.recurrence || String(form.recurrence).trim().length === 0) + next.recurrence = "Recurrence is required"; + setErrors(next); + return Object.keys(next).length === 0; + }; + + const save = async () => { + if (!validate()) return; + setSaving(true); + try { + const url = `${globalUrl}/api/v1/orgs/${selectedOrganization.id}`; + const payload = { + org_id: selectedOrganization.id, + editing: "subscription_update", + subscription_index: 0, + subscription: { + ...form, + // Ensure backend gets array of features + features: markdownToFeatures(featuresMarkdown), + }, }; - - console.log("OBJECT: ", priceItem, checkoutObject); - - stripe - .redirectToCheckout(checkoutObject) - .then(function (result) { - console.log("SUCCESS STRIPE?: ", result); - }) - .catch(function (error) { - console.error("STRIPE ERROR: ", error); + const res = await fetch(url, { + method: "POST", + credentials: "include", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(payload), + }); + const data = await res.json().catch(() => ({})); + if (res.ok && data && data.success !== false) { + toast.success("Subscription updated"); + onClose?.(); + onSaved?.({ + ...form, + features: markdownToFeatures(featuresMarkdown), }); - }; + } else { + toast.error("Failed to update subscription"); + } + } catch (e) { + toast.error("Failed to update subscription"); + } finally { + setSaving(false); + } + }; return ( -
- - - {(selectedOrganization.subscriptions === undefined || selectedOrganization.subscriptions === null || selectedOrganization.subscriptions.length === 0) && isCloud ? - - : - selectedOrganization.subscriptions !== undefined && - selectedOrganization.subscriptions !== null && - selectedOrganization.subscriptions.length > 0 ? - selectedOrganization.subscriptions - .reverse() - .map((sub, index) => { - return ( - - ) - }) - : null} - {!isCloud ? - - - {/* - - */} - - : null} + + + Edit subscription + + +
+ setForm({ ...form, name: e.target.value })} + fullWidth + error={!!errors.name} + helperText={errors.name} + /> + + setForm((prev) => { + const active = e.target.checked; + const todayEpoch = toEpoch( + new Date().toISOString().slice(0, 10) + ); + return { + ...prev, + active, + cancellationdate: active + ? 0 + : prev.cancellationdate && prev.cancellationdate !== 0 + ? prev.cancellationdate + : todayEpoch, + }; + }) + } + /> + } + label="Active" + /> - {/* {isCloud && - selectedOrganization.subscriptions !== undefined && - selectedOrganization.subscriptions !== null && - selectedOrganization.subscriptions.length > 0 ? - selectedOrganization.subscriptions - .reverse() - .map((sub, index) => { - return ( - - ) - }) - : null} */} - - { - licensePopup && - ( - - {errorMessage.length > 0 ? Error: {errorMessage} : null} - -
- - { - billingCycle === "annual" && - ( - - - 10% OFF - - - ) - } -
-
- - {scaleValue > 300 ? "Enterprise Plan" : "Scale Plan"} - - - - - Monthly - - - Annual - - - -
- - - App Runs Units - + + setForm({ ...form, support_level: e.target.value }) + } + fullWidth + /> + setForm({ ...form, recurrence: e.target.value })} + fullWidth + error={!!errors.recurrence} + helperText={errors.recurrence} + /> -
- - {scaleValue > 300 ? "Let's Talk" : `$${getPrice(32) * (scaleValue / 10)}`} - - 300 ? 1 : 0, - }} - > - {scaleValue > 300 ? `for ${scaleValue > 500 ? "500k+" : `${scaleValue}k`} App Runs` : `/month for ${scaleValue}k App Runs`} - -
- - { - if(value === 510){ - return "500k+" - } - return `${value}k` - }} - step={10} - min={10} - max={510} - marks - sx={{ - color: "#ff8544", - "& .MuiSlider-thumb": { - width: 15, - height: 15, - }, - "& .MuiSlider-valueLabel": { - backgroundColor: "rgba(33, 33, 33, 1)", - color: "rgba(241, 241, 241, 1)", - fontSize: 14, - borderRadius: "4px", - border: "1px solid rgba(73, 73, 73, 1)", - fontFamily: theme?.typography?.fontFamily, - }, - }} - /> - + $ + ), + }} + onChange={(e) => setForm({ ...form, amount: e.target.value })} + fullWidth + error={!!errors.amount} + helperText={errors.amount || "0 for Free"} + /> -
-
- {defaultTaskIcon} - Standard Email Support -
- -
- {defaultTaskIcon} - - {shuffleVariant === 0 ? "Multi-Tenant" : "Lightning-Fast Workflows"} - -
- -
- {defaultTaskIcon} - - {shuffleVariant === 0 ? "Multi-Region Tenants" : "High Availability"} - -
- -
- {defaultTaskIcon} - 30 Days workflow run history -
-
-
- -
- - - -
-
- - - ) + + setForm({ ...form, startdate: toEpoch(e.target.value) }) + } + InputLabelProps={{ shrink: true }} + fullWidth + /> + + setForm({ ...form, enddate: toEpoch(e.target.value) }) + } + InputLabelProps={{ shrink: true }} + fullWidth + error={!!errors.enddate} + helperText={errors.enddate} + /> + {form.active ? null : ( + + setForm({ + ...form, + cancellationdate: toEpoch(e.target.value), + }) } - + InputLabelProps={{ shrink: true }} + fullWidth + /> + )} + +
+ + Features + + setFeaturesMarkdown(e.target.value)} + placeholder={"- Feature\n - Sub feature"} + multiline + minRows={8} + fullWidth + inputRef={featuresInputRef} + onKeyDown={handleFeaturesKeyDown} + /> +
+ + Preview + +
+ {markdownToFeatures(featuresMarkdown).map((feat, idx) => { + const depth = (feat.match(/^(\s+)-\s+/) || [])[1] + ? Math.min( + 3, + Math.floor( + (feat.match(/^(\s+)-\s+/) || [])[1].length / 2 + ) + ) + : 0; + const label = String(feat).replace(/^\s*-\s+/, ""); + return ( +
+ {depth === 0 ? ( + + ) : ( + + )} + {label} +
+ ); + })} +
+
+
+
+ + + + + +
+ ); + }; + + // Skeleton loading component for subscription cards + const SubscriptionSkeleton = () => ( +
+
+ {/* Header skeleton */} +
+
+ + +
+
- ) -} + + {/* Price skeleton */} +
+ +
+ + {/* Divider */} + + + {/* App runs section skeleton */} +
+ + + +
+ + {/* Features section skeleton */} +
+ +
+ {[1, 2, 3, 4, 5].map((i) => ( +
+ + +
+ ))} +
+
+ + {/* Buttons skeleton */} +
+ + +
+
+
+ ); + + // Actual Subscription Object + const SubscriptionObject = (props) => { + const { + globalUrl, + userdata, + selectedOrganization, + handleGetOrg, + subscription, + isLoading = false, + } = props; + + const [signatureOpen, setSignatureOpen] = React.useState(false); + const [tosChecked, setTosChecked] = React.useState( + subscription?.eula_signed + ); + // Edit subscription dialog state + const [editOpen, setEditOpen] = React.useState(false); + const [localSub, setLocalSub] = React.useState(subscription); + // Keep local subscription state in sync with latest DB data + React.useEffect(() => { + setLocalSub(subscription); + }, [subscription]); + // Keep tosChecked in sync with local subscription + React.useEffect(() => { + setTosChecked(!!(localSub && localSub.eula_signed)); + }, [localSub && localSub.eula_signed]); + const [newBillingEmail, setNewBillingEmail] = useState(""); + + // Old function for changing billing email -> Not in use anymore + const HandleChangeBillingEmail = (orgId) => { + const email = newBillingEmail; + const emailPattern = /^[a-zA-Z0-9._-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}$/; + if (!emailPattern.test(email)) { + toast("Please enter a valid email address"); + return; + } else { + setNewBillingEmail(email); + } + + toast("Updating billing email. Please Wait"); + + const data = { + org_id: orgId, + email: newBillingEmail, + billing: { + email: newBillingEmail, + }, + }; + + const url = `${globalUrl}/api/v1/orgs/${orgId}/billing`; + fetch(url, { + method: "POST", + body: JSON.stringify(data), + credentials: "include", + headers: { + "Content-Type": "application/json", + }, + }) + .then((response) => { + if (response.status !== 200) { + console.log("Bad status code in get org:", response.status); + } + return response.json(); + }) + .then((responseJson) => { + if (responseJson.success === true) { + toast.success("Successfully updated billing email"); + setBillingEmail(newBillingEmail); + } else { + toast.error("Failed to update billing email. Please try again."); + } + }) + .catch((error) => { + console.log("Error getting org:", error); + }); + }; + + // Get extra features from features object + const extraFeatures = Object.entries(features || {}) + .filter(([_, featureData]) => { + return ( + featureData && + typeof featureData === "object" && + featureData.active === true + ); + }) + .map(([featureName, featureData]) => { + return formatFeatureText(featureName, featureData?.limit); + }) + .filter( + (feature) => + feature.length > 0 && + !feature.toLowerCase().includes("unlimited") && // Add this filter to remove "unlimited" features + !feature.includes("App Executions per month") + ); + + const finalFeatures = mergeUniqueFeatures(localSub.features, extraFeatures); + + const usedAppRuns = Number(monthlyAppRunsParent) + Number(monthlyAllSuborgExecutions); + const appRunsLimit = userdata?.app_execution_limit || selectedOrganization?.sync_features?.app_executions?.limit; + const appRunsPct = + appRunsLimit > 0 + ? Math.min(100, Math.round((usedAppRuns / appRunsLimit) * 100)) + : 0; + + const [showAllFeatures, setShowAllFeatures] = useState(false); + + // Render new Current Subscription Card UI if this is the active plan + const visibleFeatures = (finalFeatures || []).filter(Boolean); + const collapsed = showAllFeatures + ? visibleFeatures + : visibleFeatures.slice(0, 6); + + const getFeatureIndent = (text) => { + // Count leading spaces in patterns like " - sub item" + const match = String(text).match(/^(\s+)-\s+/); + if (!match) return 0; + const spaces = match[1].length; + return Math.min(3, Math.floor(spaces / 2)); + }; + + const stripPrefix = (text) => { + return String(text) + .replace(/^\s*-\s+/, "") + .trim(); + }; + + const isCancelled = localSub.cancellationdate !== 0; + const isPaidPlan = localSub.amount !== "0"; + const amountToshow = isPaidPlan + ? String(localSub.currency || "").toLowerCase() === "usd" + ? "$" + localSub?.amount + : localSub?.currency + localSub?.amount + : "Free"; + + if (typeof window === "undefined" || window.location === undefined) { + return null; + } + + // Show skeleton if loading + if (isLoading) { + return ; + } + + return ( + <> + setEditOpen(false)} + subscription={localSub} + globalUrl={globalUrl} + selectedOrganization={selectedOrganization} + onSaved={(updated) => { + // Update local card immediately for responsive UI + setLocalSub((prev) => ({ ...prev, ...updated })); + // Refresh organization data from server + if (typeof handleGetOrg === "function") { + handleGetOrg(selectedOrganization.id); + } + }} + /> + + {/* EULA Signature Dialog */} + + + { + e.preventDefault(); + setSignatureOpen(false); + setTosChecked(false); + }} + > + + + + + Read and Accept the EULA + + + + { + setTosChecked(e.target.checked); + }} + inputProps={{ "aria-label": "primary checkbox" }} + /> + { + setTosChecked(!tosChecked); + }} + > + Accept + + + By clicking the “accept” button, you are signing the document, + electronically agreeing that it has the same legal validity and + effects as a handwritten signature, and that you have the + competent authority to represent and sign on behalf an entity. + Need support or have questions? Contact us at support@shuffler.io. + + +
+ +
+
+
+ +
+
+
+
+ + {localSub.name} + + + {localSub.support_level} + +
+
+ {/* EULA Signature Button */} + {isCloud && localSub.eula && appRunsLimit >= 12000 && ( + + { + if (localSub.eula_signed && !userdata.support) { + return; + } + setSignatureOpen(true); + }} + style={{ + padding: "6px", + color: localSub.eula_signed ? "#545454" : "#ff8544", + backgroundColor: localSub.eula_signed + ? "rgba(241, 241, 241, 0.1)" + : "rgba(255, 133, 68, 0.1)", + borderRadius: "50%", + }} + > + + + + )} + {isPaidPlan ? ( +
+ + + {!isCancelled ? "Active" : "Inactive"} + +
+ ) : null} +
+
+ +
+ + {amountToshow} + + {isPaidPlan && ( + + /{" "} + {localSub.recurrence === "month" + ? "Monthly" + : localSub.recurrence === "year" + ? "Annual" + : localSub.recurrence} + + )} +
+ {(localSub.enddate || localSub.Enddate) && localSub.active ? ( + + {`${ + isPaidPlan ? "Next billing: " : "App runs resets on " + }${new Date( + (localSub.enddate || localSub.Enddate) * 1000 + ).toLocaleDateString(undefined, { + day: "2-digit", + month: "short", + year: "numeric", + })}`} + + ) : null} + + {localSub.cancellationdate !== 0 ? ( + + {`Cancelled on ${new Date( + (localSub.cancellationdate || localSub.CancellationDate) * + 1000 + ).toLocaleDateString(undefined, { + day: "2-digit", + month: "short", + year: "numeric", + })}`} + + ) : null} + + + + { + (isCloud || (!isCloud && selectedOrganization.cloud_sync)) && ( +
+ + App Runs + +
+ + {usedAppRuns?.toLocaleString?.() || usedAppRuns} of{" "} + {appRunsLimit?.toLocaleString?.() || appRunsLimit} + + + + +
+
+ ) + } + +
+ + Included: + +
+ {collapsed.map((feat, idx) => { + const depth = getFeatureIndent(feat); + const label = stripPrefix(feat); + return ( +
+ {depth === 0 ? ( + + ) : ( + + )} + {label} +
+ ); + })} +
+ {visibleFeatures.length > 6 && ( + + )} +
+ +
+ {isCloud && + localSub.name.toLowerCase().includes("scale") && + localSub?.reference && + localSub.reference.length > 0 ? ( + + ) : null} + {subscription.amount === "0" && ( + + )} + + {userdata.support && ( + + )} +
+
+
+ + ); + }; + + + return ( +
+ + + {isLoading ? ( + + ) : ( + <> + {selectedOrganization.subscriptions !== undefined && + selectedOrganization.subscriptions !== null && + selectedOrganization.subscriptions.length > 0 + ? (selectedOrganization.subscriptions || []) + .slice() + .map((sub, index) => { + return ( + + ); + }) + : null} + + )} + + +
+ ); +}; export default LicencePopup; From 303188efec7bb341e40fd35d189cbec70646442c Mon Sep 17 00:00:00 2001 From: monilprajapati Date: Mon, 6 Oct 2025 18:50:32 +0530 Subject: [PATCH 33/57] New shuffle shared version --- backend/go-app/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 4896209c..5bf206d9 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -24,7 +24,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.25 + github.com/shuffle/shuffle-shared v0.9.28 github.com/shuffle/singul v0.0.16 golang.org/x/crypto v0.40.0 google.golang.org/api v0.236.0 From d6b32a23eda8aa7fc66fb5022490b2d89b37bbed Mon Sep 17 00:00:00 2001 From: monilprajapati Date: Mon, 6 Oct 2025 19:05:57 +0530 Subject: [PATCH 34/57] sub logic in backend --- backend/go-app/main.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index f1d53d63..debb8390 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3950,6 +3950,14 @@ func remoteOrgJobController(org shuffle.Org, body []byte) error { shuffle.SetCache(ctx, cacheKey, featuresBytes, 1800) } + subscriptionCacheKey := fmt.Sprintf("org_subscriptions_%s", org.Id) + subscriptionsBytes, err := json.Marshal(responseData.Subscriptions) + if err != nil { + log.Printf("[ERROR] Failed to marshal Subscriptions for cache: %s", err) + } else { + shuffle.SetCache(ctx, subscriptionCacheKey, subscriptionsBytes, 1800) + } + for _, job := range responseData.Jobs { err = handleCloudJob(job) if err != nil { From d066587234c82102d2d9d7d25211932b4abdce18 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Mon, 6 Oct 2025 20:40:45 +0530 Subject: [PATCH 35/57] fix: missing import --- backend/go-app/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index debb8390..6845056e 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -48,6 +48,7 @@ import ( newscheduler "github.com/carlescere/scheduler" "golang.org/x/crypto/bcrypt" "gopkg.in/yaml.v3" + "sort" // Web "github.com/gorilla/mux" From 5bfb17c9b09ac331bd3a8bf1e22900f0dc06d6f4 Mon Sep 17 00:00:00 2001 From: Frikky Date: Tue, 7 Oct 2025 17:13:08 +0200 Subject: [PATCH 36/57] Added basic revisions for datastore keys --- functions/onprem/orborus/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index b6ccae82..804823dc 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -4,7 +4,7 @@ go 1.24.0 toolchain go1.24.4 -//replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared +replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared require ( github.com/docker/docker v28.3.3+incompatible From ded032d93ed9d03adedbad7e46675388086dfc49 Mon Sep 17 00:00:00 2001 From: "lalitdeore12@gmail.com" Date: Wed, 8 Oct 2025 18:06:25 +0530 Subject: [PATCH 37/57] Fix - branding isues --- backend/go-app/main.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 6845056e..7dc3016e 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -1138,7 +1138,8 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { // Check for licensing/branding of parent and override parentOrg, err := shuffle.GetOrg(ctx, parentOrgId) if err == nil { - if parentOrg.LeadInfo.IntegrationPartner { + parent := shuffle.HandleCheckLicense(ctx, *parentOrg) + if parentOrg.LeadInfo.IntegrationPartner || parent.SyncFeatures.Branding.Active { parsedStatus = append(parsedStatus, "integration_partner") // except theme take from parent org @@ -1180,7 +1181,9 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { } } else { // for parent org branding - if org.LeadInfo.IntegrationPartner { + licenseOrg := shuffle.HandleCheckLicense(ctx, *org) + org = &licenseOrg + if org.LeadInfo.IntegrationPartner || org.SyncFeatures.Branding.Active { userInfo.ActiveOrg.Branding.Theme = org.Branding.Theme userInfo.ActiveOrg.Branding.DocumentationLink = org.Defaults.DocumentationReference userInfo.ActiveOrg.Branding.SupportEmail = org.Branding.SupportEmail From 96f91d0e9e6409aa273cc129fe98ed9918e57ef0 Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 9 Oct 2025 16:05:41 +0200 Subject: [PATCH 38/57] Orborus rebuild --- functions/onprem/orborus/go.mod | 4 ++-- functions/onprem/orborus/go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 804823dc..4277613e 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -4,13 +4,13 @@ go 1.24.0 toolchain go1.24.4 -replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared +//replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared require ( github.com/docker/docker v28.3.3+incompatible github.com/docker/go-connections v0.5.0 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.27 + github.com/shuffle/shuffle-shared v0.9.29 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 ) diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index a1ec5127..f4575157 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -328,8 +328,8 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.9.27 h1:YwyWXsp4fCOAPmc1DD+NNf9sVa4RHzp26SvWKxH4ytc= -github.com/shuffle/shuffle-shared v0.9.27/go.mod h1:PhDEizuz4SmJaSmy0+yrFWwD1mXVUsy8/knKlrqF1qw= +github.com/shuffle/shuffle-shared v0.9.29 h1:6f0liFf1a566FjX3d6eQjgYHhVfznHb8RyxM4QhfnUA= +github.com/shuffle/shuffle-shared v0.9.29/go.mod h1:PhDEizuz4SmJaSmy0+yrFWwD1mXVUsy8/knKlrqF1qw= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= From a137368eee07b405eef75a5a17bb4c0ac0ceedc6 Mon Sep 17 00:00:00 2001 From: monilprajapati Date: Fri, 10 Oct 2025 17:40:06 +0530 Subject: [PATCH 39/57] Sync UI files from cloud --- frontend/src/components/Billing.jsx | 5 +- frontend/src/components/LeftSideBar.jsx | 98 ++++++++++++++++++++- frontend/src/components/LicencePopup.jsx | 22 +++-- frontend/src/components/OrganizationTab.jsx | 78 +++++++++++++--- 4 files changed, 181 insertions(+), 22 deletions(-) diff --git a/frontend/src/components/Billing.jsx b/frontend/src/components/Billing.jsx index 3ed76123..b93848b1 100644 --- a/frontend/src/components/Billing.jsx +++ b/frontend/src/components/Billing.jsx @@ -2099,13 +2099,13 @@ const Billing = memo((props) => { {isCloud ? "Get more out of Shuffle by adding your credit card, such as no App Run limitations, and priority support from our team. We use Stripe to manage subscriptions and do not store any of your billing information. You can manage your subscription and billing information below." : - "Shuffle is an Open Source automation platform, and no license is required. We do however offer a Scale license with HA guarantees, along with support hours. By buying a license on https://shuffler.io, you can get access to the license immediately, and if Cloud Syncronisation is enabled, the UI in your local instance will also update." + !(selectedOrganization?.subscriptions !== undefined && selectedOrganization?.subscriptions.length > 0 && selectedOrganization?.subscriptions[0]?.name?.toLowerCase().includes("enterprise") && selectedOrganization?.subscriptions[0]?.active) ? "Shuffle is an Enterprise automation platform, and a license is required. We do however offer a Scale license with HA guarantees, along with support hours. By buying a license on https://shuffler.io, you can get access to the license immediately, and if Cloud Syncronisation is enabled, the UI in your local instance will also update." : "Here you can check your license and billing information." } : {isCloud ? "Get more out of Shuffle by adding your credit card, such as no App Run limitations, and priority support from our team. We use Stripe to manage subscriptions and do not store any of your billing information. You can manage your subscription and billing information below." : - "Shuffle is an Open Source automation platform, and no license is required. We do however offer a Scale license with HA guarantees, along with support hours. By buying a license on https://shuffler.io, you can get access to the license immediately, and if Cloud Syncronisation is enabled, the UI in your local instance will also update." + !(selectedOrganization?.subscriptions !== undefined && selectedOrganization?.subscriptions.length > 0 && selectedOrganization?.subscriptions[0]?.name?.toLowerCase().includes("enterprise") && selectedOrganization?.subscriptions[0]?.active) ? "Shuffle is an Enterprise automation platform, and a license is required. We do however offer a Scale license with HA guarantees, along with support hours. By buying a license on https://shuffler.io, you can get access to the license immediately, and if Cloud Syncronisation is enabled, the UI in your local instance will also update." : "Here you can check your license and billing information." } } } @@ -2924,6 +2924,7 @@ const Billing = memo((props) => { userdata={userdata} currentTab={currentTab} syncStats={true} + statistics={statistics} /> }
diff --git a/frontend/src/components/LeftSideBar.jsx b/frontend/src/components/LeftSideBar.jsx index cb851d43..8bc3b734 100644 --- a/frontend/src/components/LeftSideBar.jsx +++ b/frontend/src/components/LeftSideBar.jsx @@ -92,6 +92,8 @@ const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, }) => { region_url: org.region_url, })) || [] ); + const [activeOrgData, setActiveOrgData] = useState(null); + const [isProdStatusOn, setIsProdStatusOn] = useState(false); const userOrgs = React.useMemo(() => { return orgOptions.find((option) => option.name === selectedOrg); }, [selectedOrg, orgOptions]); @@ -682,7 +684,7 @@ const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, }) => { Version: - 2.1.0 + 2.1.1 @@ -936,6 +938,40 @@ const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, }) => { const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); const showPartnerLogo = userdata?.org_status?.includes("integration_partner") && userdata?.active_org?.image !== undefined && userdata?.active_org?.image !== null && userdata?.active_org?.image.length > 0 + useEffect(() => { + const orgId = userdata?.active_org?.id; + if (!orgId) { + return; + } + + let fetched = false; + fetch(`${globalUrl}/api/v1/orgs/${orgId}`, { + method: "GET", + credentials: "include", + headers: { "Content-Type": "application/json" }, + }) + .then((response) => (response.ok ? response.json() : null)) + .then((org) => { + if (!fetched && org) { + setActiveOrgData(org); + if (!isCloud) { + if (org?.cloud_sync && org?.subscriptions[0]?.name?.toLowerCase().includes("enterprise") && org?.subscriptions[0]?.active) { + setIsProdStatusOn(true); + } else if (org?.subscriptions[0]?.name?.toLowerCase().includes("enterprise") && org?.subscriptions[0]?.active) { + setIsProdStatusOn(true); + } else { + setIsProdStatusOn(false); + } + } + } + }) + .catch(() => {}); + + return () => { + fetched = true; + }; + }, [userdata?.active_org?.id, globalUrl]); + return (
{ } }} > + { style={{ width: showPartnerLogo ? 30 : 24, height: showPartnerLogo ? 30 : 24 }} /> - + + { + !isCloud && expandLeftNav && ( + + {isProdStatusOn ? "Enterprise" : "Open Source"} + + ) + } + { }} > + {!isCloud ? ( +
{ + navigate("/admin?admin_tab=prodstatus") + }} + > + + + {expandLeftNav ? isProdStatusOn ? "Prod. Status ON" : "Prod. Status OFF" : isProdStatusOn ? "ON" : "OFF"} + +
+ ) : null} + {userdata?.licensed !== true && !userdata?.org_status?.includes("integration_partner") && expandLeftNav &&
-
{ )}
+ )} {(localSub.enddate || localSub.Enddate) && localSub.active ? ( {`${ isPaidPlan ? "Next billing: " : "App runs resets on " @@ -1494,7 +1504,7 @@ const LicencePopup = (props) => { Manage subscription ) : null} - {subscription.amount === "0" && ( + {!isPaidPlan && ( +
+ + Drag & drop your OpenAPI (YAML/JSON) anywhere + + + or click to browse files + +
+ + { Continue + {/* Generate App Modal */} diff --git a/frontend/src/components/AppGrid.jsx b/frontend/src/components/AppGrid.jsx index e647085b..33998bfb 100644 --- a/frontend/src/components/AppGrid.jsx +++ b/frontend/src/components/AppGrid.jsx @@ -33,6 +33,7 @@ import { ClearRefinements, connectStateResults } from "react-instantsearch-dom"; +import { useDebouncedCallback } from "../utils/useDebouncedCallback"; import aa from "search-insights"; import { useLocation } from 'react-router-dom'; @@ -160,6 +161,8 @@ const AppGrid = (props) => { refine(searchQuery.trim()); }; + const debouncedRefine = useDebouncedCallback((value) => refine(value), 300); + return (
{ placeholder="Search more than 2500 Apps" id="shuffle_search_field" onChange={(event) => { - setSearchQuery(event.currentTarget.value); + const value = event.currentTarget.value; + setSearchQuery(value); removeQuery("q"); - refine(event.currentTarget.value); + debouncedRefine(value); }} onKeyDown={(event) => { if(event.key === "Enter") { diff --git a/frontend/src/components/BillingStats.jsx b/frontend/src/components/BillingStats.jsx index 8e011d42..0cd2a38e 100644 --- a/frontend/src/components/BillingStats.jsx +++ b/frontend/src/components/BillingStats.jsx @@ -1,4 +1,4 @@ -import React, { useState, useEffect, useContext, memo, useMemo } from 'react'; +import React, { useState, useEffect, useContext, useCallback } from 'react'; import {getTheme} from '../theme.jsx'; import classNames from "classnames"; @@ -73,6 +73,7 @@ const AppStats = (defaultprops) => { const [resultRows, setResultRows] = useState([]) const [resultLoading, setResultLoading] = useState(true) const { themeMode, brandColor } = useContext(Context); + const [onpremAppRuns, setOnpremAppRuns] = useState(0) const theme = getTheme(themeMode, brandColor) const includedExecutions = selectedOrganization?.sync_features?.app_executions !== undefined ? selectedOrganization?.sync_features?.app_executions?.limit : 0 @@ -83,12 +84,156 @@ const AppStats = (defaultprops) => { } }, []) + const handleDataSetting = useCallback((inputdata, grouping) => { + if (inputdata === undefined || inputdata === null) { + return + } + + const statKey = syncStats === true ? "onprem_stats" : "daily_statistics" + const dailyStats = inputdata[statKey] + if (dailyStats === undefined || dailyStats === null) { + return + } + + var appRuns = { + "key": "App Runs", + "data": [] + } + + var childorgappRuns = { + "key": "Child Org App Runs", + "data": [] + } + + var workflowRuns = { + "key": "Workflow Runs (includes subflows)", + "data": [] + } + + var subflowRuns = { + "key": "Subflow Runs", + "data": [] + } + + var appcostRuns = { + "key": "Cost of App Runs", + "data": [] + } + + for (let key in dailyStats) { + // Always skips first one as it has accumulated data in it + if (key === 0) { + continue + } + + const item = dailyStats[key] + if (item["date"] === undefined) { + console.log("No date: ", item) + continue + } + + // Check if app_executions key in item + if (item["app_executions"] !== undefined && item["app_executions"] !== null) { + appRuns["data"].push({ + key: new Date(item["date"]).toISOString(), + data: item["app_executions"] + }) + + // Add number + appcostRuns["data"].push({ + key: new Date(item["date"]).toISOString(), + data: (item["app_executions"] * invocationCost).toFixed(2) + }) + } + + if (item["child_app_executions"] !== undefined && item["child_app_executions"] !== null) { + childorgappRuns["data"].push({ + key: new Date(item["date"]).toISOString(), + data: item["child_app_executions"] + }) + } + + // Check if workflow_executions key in item + if (item["workflow_executions"] !== undefined && item["workflow_executions"] !== null) { + workflowRuns["data"].push({ + key: new Date(item["date"]).toISOString(), + data: item["workflow_executions"] + }) + } + + if (item["subflow_executions"] !== undefined && item["subflow_executions"] !== null) { + subflowRuns["data"].push({ + key: new Date(item["date"]).toISOString(), + data: item["subflow_executions"] + }) + } + } + + // Only add today's data if endTime is not set or if today falls within the selected date range + const today = new Date() + const todayStartOfDay = new Date(today) + todayStartOfDay.setHours(0, 0, 0, 0) + const shouldAddTodayData = endTime === "" || endTime === undefined || endTime === null || + (new Date(endTime) >= todayStartOfDay) + + if (!syncStats && shouldAddTodayData) { + // Adds data for today + if (inputdata["daily_app_executions"] !== undefined && inputdata["daily_app_executions"] !== null) { + appRuns["data"].push({ + key: new Date().toISOString(), + data: inputdata["daily_app_executions"] + }) + + appcostRuns["data"].push({ + key: new Date().toISOString(), + data: (inputdata["daily_app_executions"] * invocationCost).toFixed(2) + }) + } + + if (inputdata["daily_child_app_executions"] !== undefined && inputdata["daily_app_executions"] !== null) { + childorgappRuns["data"].push({ + key: new Date().toISOString(), + data: inputdata["daily_child_app_executions"] + }) + } + + if (inputdata["daily_workflow_executions"] !== undefined && inputdata["daily_workflow_executions"] !== null) { + workflowRuns["data"].push({ + key: new Date().toISOString(), + data: inputdata["daily_workflow_executions"] + }) + } + + if (inputdata["daily_subflow_executions"] !== undefined && inputdata["daily_subflow_executions"] !== null) { + subflowRuns["data"].push({ + key: new Date().toISOString(), + data: inputdata["daily_subflow_executions"] + }) + } + } + + // Only for parent orgs + if (childorgappRuns["data"].length > 0) { + setChildOrgsAppRuns(childorgappRuns) + } + + setSubflowRuns(subflowRuns) + setWorkflowRuns(workflowRuns) + setAppruns(appRuns) + setApprunCosts(appcostRuns) + }, [syncStats, endTime, startTime]) + useEffect(() => { if (statistics && statistics?.org_id?.length > 0) { handleDataSetting(statistics, "day") } }, [statistics]) + useEffect(() => { + setStartTime("") + setEndTime("") + }, [currentTab]) + const getWorkflowStats = async (workflow, startTime, endTime) => { if (workflow.id === undefined || workflow.id === null || workflow.id === "") { @@ -227,7 +372,7 @@ const AppStats = (defaultprops) => { } const statKey = syncStats === true ? "onprem_stats" : "daily_statistics" - if (statistics[statKey] === undefined || statistics[statKey] === null) { + if (!syncStats && (statistics[statKey] === undefined || statistics[statKey] === null)) { setFilteredStatistics(statistics) setMonthlyAppRunsParent(statistics["monthly_app_executions"] ?? 0) return @@ -356,17 +501,55 @@ const AppStats = (defaultprops) => { workflowexecutions += item["workflow_executions"] appexecutions += item["app_executions"] - if (currentTab === 0) { + if (currentTab === 0 || currentTab === 3) { appexecutions += (item["child_app_executions"] ?? 0) } estimatedcost += (item["app_executions"] * invocationCost) } + const today = new Date(); + const isCurrentMonthSelected = + (startTime === "" && endTime === "") || + ( + new Date(foundstarttime).getMonth() === today.getMonth() && + new Date(foundstarttime).getFullYear() === today.getFullYear() && + new Date(foundendtime).getMonth() === today.getMonth() && + new Date(foundendtime).getFullYear() === today.getFullYear() + ); + + if (!syncStats && isCurrentMonthSelected) { + if (statistics["daily_app_executions"] !== undefined && statistics["daily_app_executions"] !== null) { + appexecutions += statistics["daily_app_executions"] + (statistics["daily_child_app_executions"] ?? 0) + } + } + tmpstats["monthly_workflow_executions"] = workflowexecutions tmpstats["monthly_app_executions"] = appexecutions + if (syncStats) { + setOnpremAppRuns(appexecutions) + } + } else { + const today = new Date(); + const isCurrentMonthSelected = + (startTime === "" && endTime === "") || + ( + new Date(foundstarttime).getMonth() === today.getMonth() && + new Date(foundstarttime).getFullYear() === today.getFullYear() && + new Date(foundendtime).getMonth() === today.getMonth() && + new Date(foundendtime).getFullYear() === today.getFullYear() + ); + + if (!syncStats && isCurrentMonthSelected) { + if (statistics["daily_app_executions"] !== undefined && statistics["daily_app_executions"] !== null) { + appexecutions += statistics["daily_app_executions"] + (statistics["daily_child_app_executions"] ?? 0) + } + } + + tmpstats["monthly_app_executions"] = appexecutions } + // Make estimatedcost have max 2 decimals if (isCloud) { // Exclude includedExecutions*month @@ -380,11 +563,11 @@ const AppStats = (defaultprops) => { handleDataSetting(tmpstats, "day") // if we have done monthly reset than only show monthly app runs as current month app run const currentMonth = new Date().getMonth() + 1 - if (!monthlyAppRunsParent && statistics["monthly_app_executions"] > 0 && currentMonth === statistics["last_monthly_reset_month"]) { + if (!syncStats && !monthlyAppRunsParent && statistics["monthly_app_executions"] > 0 && currentMonth === statistics["last_monthly_reset_month"]) { setMonthlyAppRunsParent(statistics["monthly_app_executions"]) } - if (!monthlyAllSuborgExecutions && statistics["monthly_child_app_executions"]> 0 && currentMonth === statistics["last_monthly_reset_month"]) { + if (!syncStats && !monthlyAllSuborgExecutions && statistics["monthly_child_app_executions"]> 0 && currentMonth === statistics["last_monthly_reset_month"]) { setMonthlyAllSuborgExecutions(statistics["monthly_child_app_executions"]) } @@ -397,7 +580,7 @@ const AppStats = (defaultprops) => { loadWorkflowStats(foundWorkflows, startTime, endTime) } - }, [statistics, startTime, endTime]) + }, [statistics, startTime, endTime, syncStats, currentTab, handleDataSetting]) const handleStartTimeChange = (date) => { setStartTime(date) @@ -407,142 +590,7 @@ const AppStats = (defaultprops) => { setEndTime(date) } - const handleDataSetting = (inputdata, grouping) => { - if (inputdata === undefined || inputdata === null) { - return - } - - const statKey = syncStats === true ? "onprem_stats" : "daily_statistics" - const dailyStats = inputdata[statKey] - if (dailyStats === undefined || dailyStats === null) { - return - } - - var appRuns = { - "key": "App Runs", - "data": [] - } - - var childorgappRuns = { - "key": "Child Org App Runs", - "data": [] - } - - var workflowRuns = { - "key": "Workflow Runs (includes subflows)", - "data": [] - } - - var subflowRuns = { - "key": "Subflow Runs", - "data": [] - } - - var appcostRuns = { - "key": "Cost of App Runs", - "data": [] - } - - for (let key in dailyStats) { - // Always skips first one as it has accumulated data in it - if (key === 0) { - continue - } - - const item = dailyStats[key] - if (item["date"] === undefined) { - console.log("No date: ", item) - continue - } - - // Check if app_executions key in item - if (item["app_executions"] !== undefined && item["app_executions"] !== null) { - appRuns["data"].push({ - key: new Date(item["date"]), - data: item["app_executions"] - }) - - // Add number - appcostRuns["data"].push({ - key: new Date(item["date"]), - data: (item["app_executions"] * invocationCost).toFixed(2) - }) - } - - if (item["child_app_executions"] !== undefined && item["child_app_executions"] !== null) { - childorgappRuns["data"].push({ - key: new Date(item["date"]), - data: item["child_app_executions"] - }) - } - - // Check if workflow_executions key in item - if (item["workflow_executions"] !== undefined && item["workflow_executions"] !== null) { - workflowRuns["data"].push({ - key: new Date(item["date"]), - data: item["workflow_executions"] - }) - } - - if (item["subflow_executions"] !== undefined && item["subflow_executions"] !== null) { - subflowRuns["data"].push({ - key: new Date(item["date"]), - data: item["subflow_executions"] - }) - } - } - - // Only add today's data if endTime is not set or if today falls within the selected date range - const today = new Date() - const shouldAddTodayData = endTime === "" || endTime === undefined || endTime === null || - (new Date(endTime) >= today.setHours(0, 0, 0, 0)) - - if (shouldAddTodayData) { - // Adds data for today - if (inputdata["daily_app_executions"] !== undefined && inputdata["daily_app_executions"] !== null) { - appRuns["data"].push({ - key: new Date(), - data: inputdata["daily_app_executions"] - }) - - appcostRuns["data"].push({ - key: new Date(), - data: (inputdata["daily_app_executions"] * invocationCost).toFixed(2) - }) - } - - if (inputdata["daily_child_app_executions"] !== undefined && inputdata["daily_app_executions"] !== null) { - childorgappRuns["data"].push({ - key: new Date(), - data: inputdata["daily_child_app_executions"] - }) - } - - if (inputdata["daily_workflow_executions"] !== undefined && inputdata["daily_workflow_executions"] !== null) { - workflowRuns["data"].push({ - key: new Date(), - data: inputdata["daily_workflow_executions"] - }) - } - - if (inputdata["daily_subflow_executions"] !== undefined && inputdata["daily_subflow_executions"] !== null) { - subflowRuns["data"].push({ - key: new Date(), - data: inputdata["daily_subflow_executions"] - }) - } - } - - // Only for parent orgs - if (childorgappRuns["data"].length > 0) { - setChildOrgsAppRuns(childorgappRuns) - } - - setSubflowRuns(subflowRuns) - setWorkflowRuns(workflowRuns) - setAppruns(appRuns) - setApprunCosts(appcostRuns) - } + console.log("sync stats: ", syncStats, statistics) const paperStyle = { textAlign: "center", @@ -708,22 +756,26 @@ const AppStats = (defaultprops) => { } */} - {syncStats === true ? null : + {/* {syncStats === true ? null : */} App runs in the selected period }> + {syncStats === true ? + + {onpremAppRuns} + : {filteredStatistics.monthly_app_executions === null || filteredStatistics.monthly_app_executions === undefined ? 0 : filteredStatistics.monthly_app_executions} - + } App Runs - } + {/* } */} {syncStats === true || currentTab === 0 ? null : { const [showSettingsMenu, setShowSettingsMenu] = useState(false); const [showCollectIngestMenu, setShowCollectIngestMenu] = useState(false); + useEffect(() => { + if (selectedCategory === "" || selectedCategory === null || selectedCategory === undefined || selectedCategory === "default") { + return + } + + if (datastoreCategories === undefined || datastoreCategories === null || datastoreCategories.length === 0) { + return + } + + if (!datastoreCategories.includes(selectedCategory)) { + setDatastoreCategories([...datastoreCategories, selectedCategory]) + } + }, [datastoreCategories, selectedCategory]) + var to_be_copied = ""; const defaultAutomation = [ { @@ -299,7 +313,16 @@ const CacheView = memo((props) => { useEffect(() => { getWorkflows() getApps() - listOrgCache(orgId, selectedCategory, 0, pageSize, page) + + var chosenCategory = selectedCategory + const urlParams = new URLSearchParams(window.location.search) + const categoryParam = urlParams.get("category") + if (categoryParam && categoryParam !== undefined && categoryParam !== "default" && categoryParam !== "") { + chosenCategory = categoryParam + setSelectedCategory(categoryParam) + } + + listOrgCache(orgId, chosenCategory, 0, pageSize, page) }, []) @@ -423,7 +446,6 @@ const CacheView = memo((props) => { setDatastoreCategories(newcategories) } - if (responseJson?.category_config !== undefined && responseJson?.category_config !== null) { if (responseJson?.category_config?.id !== undefined && responseJson?.category_config?.id !== null && responseJson?.category_config?.id !== "") { @@ -458,7 +480,12 @@ const CacheView = memo((props) => { } } } else { - toast.warn("Failed to load keys. Please try again or contact support@shuffler if this persists.") + //toast.warn("Failed to load keys. Please try again or contact support@shuffler if this persists.") + + if (category !== undefined && category !== null && category !== "" && category !== "default") { + toast.info(`No keys to load in category ${category}`) + setSelectedCategory(category) + } } }) .catch((error) => { @@ -513,8 +540,8 @@ const CacheView = memo((props) => { category: selectedCategory, } - if (dataValue?.category !== "" && dataValue?.category !== "default") { - entry.category = dataValue.category.replaceAll(" ", "_"); + if (dataValue?.category !== undefined && dataValue?.category !== "" && dataValue?.category !== "default") { + entry.category = dataValue?.category?.replaceAll(" ", "_"); } @@ -1513,7 +1540,7 @@ const CacheView = memo((props) => { name={null} /> : - + {data.value} } @@ -1712,7 +1739,7 @@ const CacheView = memo((props) => { { e.preventDefault() e.stopPropagation() @@ -1911,7 +1938,7 @@ const CacheView = memo((props) => { {selectedCategory === "protected" ?
- Protected keys are encrypted, only available to admins, and will be masked when used in workflows. This is a basic protection, and is NOT bulletproof. + Protected keys are encrypted, only available to admins, and will be masked when used in workflows. If you want unreadable secrets, use App Auth.
: null} @@ -2051,7 +2078,7 @@ const CacheView = memo((props) => {
: - + + + + + + ); +}; + +export default DashboardOnboarding; + + diff --git a/frontend/src/components/Detection.jsx b/frontend/src/components/Detection.jsx index bd75e99d..35c1a994 100644 --- a/frontend/src/components/Detection.jsx +++ b/frontend/src/components/Detection.jsx @@ -143,6 +143,7 @@ const Detection = (props) => { size="small" sx={{ mr: 2 }} value={searchQuery} + disabled onChange={(e) => setSearchQuery(e.target.value)} /> {/* + {/**/} {detectionInfo?.category === "SIGMA" || detectionInfo?.category === "SIEM" ? - + 0 ? green : red}} /> @@ -345,7 +553,7 @@ const DetectionExplorer = (props) => { - {filteredRules?.length > 0 ? + {ruleInfo?.length > 0 ? { size="small" sx={{ mr: 2 }} value={searchQuery} - onChange={(e) => setSearchQuery(e.target.value)} + onChange={(e) => { + setSearchQuery(e?.target?.value?.replaceAll(" ", "_")?.toLowerCase()) + }} /> @@ -386,7 +596,7 @@ const DetectionExplorer = (props) => { { folderDisabled={folderDisabled} isDetectionActive={isDetectionActive} + ruleDetails={rule} ruleMapping={ruleMapping} setRuleMapping={setRuleMapping} diff --git a/frontend/src/components/DetectionRuleCard.jsx b/frontend/src/components/DetectionRuleCard.jsx index 1a2e9760..a9fcb44b 100644 --- a/frontend/src/components/DetectionRuleCard.jsx +++ b/frontend/src/components/DetectionRuleCard.jsx @@ -12,16 +12,18 @@ import { FormLabel, } from "@mui/material"; -import DashboardBarchart, { LoadStats } from '../components/DashboardBarchart.jsx'; +import LineChartWrapper, { LoadStats } from "../components/LineChartWrapper.jsx"; import { Edit as EditIcon, + Refresh as RefreshIcon, } from "@mui/icons-material"; import { toast } from "react-toastify"; import ShuffleCodeEditor from "../components/ShuffleCodeEditor1.jsx"; import theme from '../theme.jsx'; +const RuleCard = (props) => { + const { ruleName, description, file_id, globalUrl, folderDisabled, isDetectionActive, availableDetection, ruleMapping, setRuleMapping, ruleDetails, key, ...otherProps } = props -const RuleCard = ({ ruleName, description, file_id, globalUrl, folderDisabled, isTenzirActive, availableDetection, ruleMapping, setRuleMapping, ...otherProps }) => { const [openCodeEditor, setOpenCodeEditor] = React.useState(false); const [fileData, setFileData] = React.useState(""); const [isEnabled, setIsEnabled] = React.useState(otherProps.is_enabled); @@ -30,35 +32,33 @@ const RuleCard = ({ ruleName, description, file_id, globalUrl, folderDisabled, i const [responseValue, setResponseValue] = React.useState("No response action") const isCloud = ["localhost:3002", "shuffler.io"].includes(window.location.host); - console.log("Rulemapping: ", ruleMapping) useEffect(() => { - - //const url = `${globalUrl}/api/v1/stats/app_executions_test2` - //const resp = LoadStats(globalUrl, ruleName) - //const resp = LoadStats(globalUrl, "app_executions_test2") - const resp = LoadStats(globalUrl, "app_executions_cloud") - resp.then((data) => { - if (data === undefined) { - setFilteredBarchart([]) - } else { - setFilteredBarchart(data) + if (key < 10) { + console.log("RuleCard Key: ", key, ruleName, file_id, otherProps) } - }) - if (ruleMapping !== undefined && ruleMapping !== null && ruleMapping.value !== undefined && ruleMapping.value !== null) { - console.log("FIX MAPPING FROM ruleMapping.value: ", ruleMapping) - } + if (ruleDetails?.title === undefined || ruleDetails?.title === null || ruleDetails?.title.length === 0) { + //toast.error("Can't load stats for this rule. Contact support@shuffler.io if this persists.") + return + } + + const resp = LoadStats(globalUrl, `detection_rule_${ruleDetails?.title.replaceAll(" ", "_").toLowerCase()}`) + resp.then((data) => { + if (data === undefined) { + setFilteredBarchart([]) + } else { + setFilteredBarchart(data) + } + }) }, []) - console.log("Response Value: ", responseValue) - const handleSwitchChange = (event) => { if (folderDisabled) { toast.warn("Enable the directory to enable individual rules"); return; } - if (!isTenzirActive) { + if (!isDetectionActive) { toast.warn("Connect to the siem first to enable/disable the rule"); return; } @@ -96,6 +96,7 @@ const RuleCard = ({ ruleName, description, file_id, globalUrl, folderDisabled, i }); }; + var parsedRulename = ruleName.charAt(0).toUpperCase() + ruleName.slice(1).replaceAll("_", " ") return ( - {ruleName.replaceAll("_", " ")} ({filteredBarchart === null || filteredBarchart.total === undefined ? 0 : filteredBarchart.total}) + + {parsedRulename} {/*({filteredBarchart === null || filteredBarchart.total === undefined ? 0 : filteredBarchart.total})*/} +
- (upload = ref)} - onChange={(event) => { - //const file = event.target.value - //const fileObject = URL.createObjectURL(actualFile) - //setFile(fileObject) - //const files = event.target.files[0] - uploadFiles(event.target.files); + + + {/* */} + (upload = ref)} + onChange={(event) => { + //const file = event.target.value + //const fileObject = URL.createObjectURL(actualFile) + //setFile(fileObject) + //const files = event.target.files[0] + uploadFiles(event.target.files); - }} - /> - + }} + /> + + + {/*
*/} + {selectedCategory === "sigma" || selectedCategory === "yara" ? + + + + + + : null} + {fileCategories !== undefined && fileCategories !== null && fileCategories.length > 1 ? ( - + + + Category + { /> + {!selectedOrganization || selectedOrganization?.creator_org === undefined || selectedOrganization?.creator_org || null || selectedOrganization?.creator_org?.length > 0 ? null : + />} Workflow Backup Repository diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index f86bc1d7..6af8691e 100755 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -437,7 +437,8 @@ const ParsedAction = (props) => { ]; const getApp = (appId, setApp) => { - fetch(globalUrl + "/api/v1/apps/" + appId + "/config?openapi=false", { + const url = `${globalUrl}/api/v1/apps/${appId}/config?openapi=false`; + fetch(url, { headers: { Accept: "application/json", }, @@ -447,7 +448,7 @@ const ParsedAction = (props) => { if (response.status === 200) { //toast("Successfully GOT app "+appId) } else { - toast("Failed getting app"); + toast.error("Failed getting app. Please try again or contact support@shuffler.io"); } return response.json(); @@ -1711,6 +1712,7 @@ const ParsedAction = (props) => { } const sortByCategoryLabel = (a, b) => { + const aHasCategoryLabel = a.category_label !== undefined && a.category_label !== null && a.category_label.length > 0 const bHasCategoryLabel = b.category_label !== undefined && b.category_label !== null && b.category_label.length > 0 @@ -1739,11 +1741,12 @@ const ParsedAction = (props) => { }) } + // Gets the most important actions first const renderedActionOptions = deduplicateByName(( - selectedApp.actions === undefined || selectedApp.actions === null ? [] : - selectedApp.actions.filter((a) => - a.category_label !== undefined && a.category_label !== null && a.category_label.length > 0).concat(sortByKey(selectedApp.actions, "label")) + selectedApp.actions === undefined || selectedApp.actions === null ? [] : + isIntegration ? selectedApp.actions : + selectedApp.actions.filter((a) => a.category_label !== undefined && a.category_label !== null && a.category_label.length > 0).concat(sortByKey(selectedApp.actions, "label")) ).sort(sortByCategoryLabel)) @@ -2981,7 +2984,6 @@ const ParsedAction = (props) => { dataLPIgnore="true" autoComplete="off" - id="checkbox-search" style={{ ...theme.palette.textFieldStyle, diff --git a/frontend/src/components/PartnerDetails.jsx b/frontend/src/components/PartnerDetails.jsx index 9ca0aeff..7b7c102c 100644 --- a/frontend/src/components/PartnerDetails.jsx +++ b/frontend/src/components/PartnerDetails.jsx @@ -223,7 +223,7 @@ const PartnerDetails = (props) => {
- Name + Company Name { />
*/}
-
+
Solutions
{ variant="text" style={{ color: theme.palette.text.primary, fontFamily: theme?.typography?.fontFamily }} > - Name + Company Name { cursor: isDisabled ? "not-allowed" : "pointer", }} fullWidth={true} - placeholder="Name" + placeholder="Company Name" type="name" id="standard-required" margin="normal" @@ -544,7 +544,8 @@ const PartnerDetails = (props) => { style={{ marginRight: "12px", color: theme.palette.text.primary, - fontFamily: theme?.typography?.fontFamily + fontFamily: theme?.typography?.fontFamily, + marginTop: 2.5, }} > Solutions @@ -895,7 +896,7 @@ const PartnerDetails = (props) => { cursor: isDisabled ? "not-allowed" : "pointer", }} fullWidth={true} - placeholder="support@shuffler.io" + placeholder="example@company.com" type="name" id="standard-required" margin="normal" diff --git a/frontend/src/components/Priorities.jsx b/frontend/src/components/Priorities.jsx index 2ff15cdf..96a80435 100644 --- a/frontend/src/components/Priorities.jsx +++ b/frontend/src/components/Priorities.jsx @@ -1362,10 +1362,10 @@ print('"' + encoded + '"')
- Notification Workflow + Error Workflow - The notification workflow triggers when an error occurs in one of your workflows. Each individual one will only start a workflow once every 2 minutes. You can point child org notifications into the parent org notification by choosing it in the list. + The error workflow triggers when an error occurs in one of your workflows. Each individual one will only start a workflow once every 2 minutes. You can point child org errors to a parent org's error workflow by choosing it in the list. {modalView} @@ -1614,12 +1614,12 @@ print('"' + encoded + '"')
} - Notifications ({ + Errors ({ notifications?.filter((notification) => showRead === true || notification.read === false).length }) - Notifications help you find potential problems with your workflows and apps.  + Error help you find potential problems with your workflows and apps.  { + const { + globalUrl, + pipelines, + workflows, + ticketWebhook, + detectionWorkflowId, + + changePipelineState, + submitPipelineWrapper, + } = props + + const [executions, setExecutions] = React.useState([]); + const [detectionTestRunning, setDetectionTestRunning] = React.useState(false); + const [detectionTestExecutionId, setDetectionTestExecutionId] = React.useState(""); + + useEffect(() => { + if (detectionWorkflowId !== "") { + handleLoadExecutions(detectionWorkflowId) + } + }, [detectionWorkflowId]) + + if (workflows === undefined || workflows === null || workflows.length === 0) { + return null + } + + const handleLoadExecutions = (workflowId, detectionTestRunning) => { + const url = `${globalUrl}/api/v2/workflows/${workflowId}/executions` + + fetch(url, { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }).then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for getting all executions"); + } + + return response.json(); + }) + .then((responseJson) => { + if (responseJson.success !== false && responseJson?.executions?.length > 0) { + if (detectionTestRunning === true) { + console.log("Checking executions in workflow: ", workflowId, responseJson.executions) + for (var executionKey in responseJson.executions) { + const curExec = responseJson.executions[executionKey] + + if (curExec.execution_id === detectionTestExecutionId) { + continue + } + + // started_at = unix timestamp + // check within the last 60 seconds + const datecomparison = (Date.now() / 1000) - 60 + if (curExec.started_at >= datecomparison) { + if (curExec?.execution_argument?.includes("rule") && curExec?.execution_argument?.includes("Test Notepad Event")) { + setDetectionTestRunning(false) + setDetectionTestExecutionId(curExec.execution_id) + } + + break; + } + } + } else { + setExecutions(responseJson.executions || []) + } + } + }) + .catch((error) => { + toast(error.toString()); + }) + } + + const runDetectionTest = () => { + setDetectionTestRunning(true) + if (ticketWebhook === "") { + setDetectionTestRunning(false) + toast.error("No ticketing webhook found. Please enable the ticketing workflow first.") + return + } + + if (detectionWorkflowId === "") { + setDetectionTestRunning(false) + toast.error("No ticketing workflow found. Please enable the ticketing workflow first.") + return + } + + if (haveDetectionPipelines() === false) { + setDetectionTestRunning(false) + toast.error("No detection pipelines found. Please deploy the Syslog (TCP) & Sigma pipelines first.") + return + } + + + // 1. Run a new pipeline which exits. + const detectionTest = `from {message: "<165>1 2025-10-06T12:34:56.789Z myhost.example.com myapp 1234 ID47 [huh eventSource=\\\"App\\\" EventID=\\\"4688\\\" NewProcessName=\\\"notepad.exe\\\" Context=\\\"Testing\\\"] This is a test log message"} | this = message.parse_syslog() | import` + + for (var pipelineKey in pipelines) { + const curPipeline = pipelines[pipelineKey] + if (curPipeline.definition === detectionTest && changePipelineState !== undefined) { + changePipelineState(curPipeline, "stop"); + } + } + + // 1. Submit it to run + // 2. Check executions if they happened recently~ + if (submitPipelineWrapper !== undefined) { + submitPipelineWrapper(detectionTest) + } + + for (var i = 0; i < 10; i++) { + setTimeout(() => { + handleLoadExecutions(detectionWorkflowId, true) + }, i * 5000) + } + + setTimeout(() => { + setDetectionTestRunning(false) + }, 60000) + } + + const haveDetectionPipelines = () => { + if (pipelines === undefined) { + toast.warn("No pipelines found. Please create the Syslog (TCP) & Sigma pipelines first.") + return false + } + + var foundCorrect = 0 + for (var pipelineKey in pipelines) { + const curPipeline = pipelines[pipelineKey] + //if (curPipeline?.definition?.includes("load_tcp") && curPipeline?.definition?.includes("import")) { + // foundCorrect += 1 + //} + + if (curPipeline?.definition?.includes("sigma") && curPipeline?.definition?.includes("export")) { + foundCorrect += 1 + } + } + + if (foundCorrect >= 1) { + return true + } + + return false + } + + return ( + + ) +} + +export default RunDetectionTest diff --git a/frontend/src/components/SchedulesTab.jsx b/frontend/src/components/SchedulesTab.jsx index cc91db50..26734560 100644 --- a/frontend/src/components/SchedulesTab.jsx +++ b/frontend/src/components/SchedulesTab.jsx @@ -7,6 +7,7 @@ import { ListItem, ListItemText, Button, + ButtonGroup, Tooltip, IconButton, Dialog, @@ -14,15 +15,21 @@ import { DialogContent, DialogActions, TextField, + Chip, + CircularProgress, } from '@mui/material'; import { - FileCopy as FileCopyIcon, - OpenInNew as OpenInNewIcon, - Padding, + FileCopy as FileCopyIcon, + OpenInNew as OpenInNewIcon, + Refresh as RefreshIcon, + Delete as DeleteIcon, + Check as CheckIcon, } from "@mui/icons-material" +import { green, yellow, red } from '../views/AngularWorkflow.jsx' import { Box, Skeleton, Typography } from '@mui/material'; import { Context } from '../context/ContextApi.jsx'; +import RunDetectionTest from '../components/RunDetectionTest.jsx'; const SchedulesTab = memo((props) => { const {globalUrl, users, } = props; @@ -30,13 +37,58 @@ const SchedulesTab = memo((props) => { const [allSchedules, setAllSchedules] = React.useState([]); const [pipelines, setPipelines] = React.useState([]); const [showLoader, setShowLoader] = React.useState(true); + const [workflows, setWorkflows] = React.useState([]); const [pipelineModalOpen, setPipelineModalOpen] = React.useState(false); - const [newPipelineValue, setNewPipelineValue] = React.useState("export | sigma /tmp/sigma_rules | to SHUFFLE_WEBHOOK"); + const [newPipelineValue, setNewPipelineValue] = React.useState(`export | sigma "/tmp/sigma_rules" | to "SHUFFLE_WEBHOOK"`); + + const [ticketWebhook, setTicketWebhook] = React.useState(""); + const [detectionWorkflowId, setDetectionWorkflowId] = React.useState(""); const { themeMode, brandColor } = useContext(Context); const theme = getTheme(themeMode, brandColor); + const handleGetWorkflows = () => { + const url = `${globalUrl}/api/v1/workflows`; + fetch(url, { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }).then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for getting all workflows"); + } + + return response.json(); + }) + .then((responseJson) => { + if (responseJson.success !== false) { + setWorkflows(responseJson || []); + + for (var i = 0; i < responseJson?.length; i++) { + if (responseJson[i].background_processing === true && responseJson[i].name.toLowerCase().includes("ingest tickets") && responseJson[i].triggers !== undefined) { + + for (var triggerkey in responseJson[i].triggers) { + if (responseJson[i].triggers[triggerkey].trigger_type === "WEBHOOK") { + setDetectionWorkflowId(responseJson[i].id) + setTicketWebhook(`${globalUrl}/api/v1/hooks/webhook_${responseJson[i].triggers[triggerkey].id}`) + setNewPipelineValue(`export | sigma /tmp/sigma_rules | to ${globalUrl}/api/v1/hooks/webhook_${responseJson[i].triggers[triggerkey].id}`) + break; + } + } + } + } + } + }) + .catch((error) => { + toast(error.toString()); + }) + } + useEffect(() => { + handleGetWorkflows() if (allSchedules.length === 0 && webHooks.length === 0 && pipelines.length === 0) { handleGetAllTriggers() } @@ -58,8 +110,11 @@ const SchedulesTab = memo((props) => { environment: pipeline.environment, }; - if (state === "start") toast("starting the pipeline"); - else toast.info("Stopping the pipeline. This may take a few minutes to propagate.") + if (state === "start") { + toast("starting the pipeline") + } else { + toast.info("Stopping a pipeline. This may take a few minutes to propagate.") + } const url = `${globalUrl}/api/v1/triggers/pipeline`; fetch(url, { @@ -144,16 +199,65 @@ const SchedulesTab = memo((props) => { }, }} > - + Run a Tenzir pipeline - Alpha feature. Deploys to the first available Orborus location. Explore Tenzir Pipelines. The example below exports everything in the Tenzir database, runs Sigma rules on it, and forwards the results to a Shuffle webhook. + Alpha feature. Deploys to the first available Orborus location. Explore Tenzir Pipelines. The example below exports everything in the Tenzir database, runs Sigma rules on it, and forwards the results to a Shuffle webhook. - -
+ +
+ + { + setNewPipelineValue(`load_tcp "0.0.0.0:1514" { read_syslog } | import`) + }} + label={"Syslog Listener (TCP)"} + variant="outlined" + color="secondary" + style={{ + marginRight: 10, + }} + /> + + { + setNewPipelineValue(`load_udp "0.0.0.0:1514", insert_newlines=true | read_syslog | import`) + }} + label={"Syslog Listener (UDP)"} + variant="outlined" + color="secondary" + style={{ + marginRight: 10, + }} + /> + + { + setNewPipelineValue(`export live=true | sigma "/tmp/sigma_rules" | to "${ticketWebhook !== "" ? ticketWebhook : "SHUFFLE_WEBHOOK"}"`) + }} + label={"Sigma Rules"} + variant="outlined" + color="secondary" + style={{ + marginRight: 10, + }} + /> + + { + setNewPipelineValue(`export live=true | to_opensearch "localhost:9200", action="create", index="shuffle_logs", user="admin", passwd="PASSWORD"`) + }} + label={"Opensearch Ingest"} + variant="outlined" + color="secondary" + style={{ + marginRight: 10, + }} + /> + { minRows={4} required fullWidth={true} - defaultValue="export | sigma /tmp/sigma_rules | to SHUFFLE_WEBHOOK" - placeholder="export | sigma /tmp/sigma_rules | to SHUFFLE_WEBHOOK" + defaultValue={`export | sigma /tmp/sigma_rules | to ${ticketWebhook !== "" ? ticketWebhook : "SHUFFLE_WEBHOOK"}`} + value={newPipelineValue} + placeholder={`export | sigma /tmp/sigma_rules | to ${ticketWebhook !== "" ? ticketWebhook : "SHUFFLE_WEBHOOK"}`} id="environment_name" margin="normal" variant="outlined" @@ -174,7 +279,7 @@ const SchedulesTab = memo((props) => { />
- + @@ -232,18 +337,18 @@ const SchedulesTab = memo((props) => { }) .then((responseJson) => { if (!responseJson.success && pipelineConfig.type !== "delete") { - toast("Failed to set pipeline: " + responseJson.reason); + toast.error("Failed to set pipeline: " + responseJson.reason); } else { if (pipelineConfig.type === "create") { - toast("Pipeline will be created: " + responseJson.reason) + toast.success("Pipeline will be created. Page will autorefresh in a bit: " + responseJson.reason) setPipelineModalOpen(false) } else if (pipelineConfig.type === "stop") { - toast("Pipeline will be stopped: " + responseJson.reason) + toast.success("Pipeline will be stopped: " + responseJson.reason) setPipelineModalOpen(false) } else { - toast("Unknown pipeline type: " + pipelineConfig.type) + toast.info("Unknown pipeline type: " + pipelineConfig.type) } } @@ -274,12 +379,7 @@ const SchedulesTab = memo((props) => { // Just use this one? - const url = - globalUrl + - "/api/v1/workflows/" + - data["workflow_id"] + - "/schedule/" + - data.id; + const url = `${globalUrl}/api/v1/workflows/${data?.workflow_id}/schedule/${data.id}`; fetch(url, { method: "DELETE", credentials: "include", @@ -414,7 +514,7 @@ const SchedulesTab = memo((props) => { //toast(error.toString()); console.log("Get schedule error: ", error.toString()); }); - }; + } const startWebHook = (trigger) => { const hookname = trigger.info.name; @@ -490,8 +590,197 @@ const SchedulesTab = memo((props) => { Triggers are Automatic Workflow starters. Status: Schedules ({allSchedules.length}), Webhooks ({webHooks.length}), Pipelines ({pipelines.length}) +
+ Pipelines + + + Controls Pipelines on your Orborus Runners, e.g. for Log Ingestion, MQ subscriptions or Sigma Detections.{" "} + + Learn more + + + +
+ + + +
+
+ + + {["Status", "Command", "Environment", "Total Runs", "Actions"].map((header, index) => ( + + ))} + + {showLoader ? ( + [...Array(6)].map((_, rowIndex) => { + return ( + + {Array(5) + .fill() + .map((_, colIndex) => { + return ( + + + + ) + })} + + ) + } + ) + + ) : ( + pipelines?.length === 0 ? ( +
+ No pipelines found. +
+ + ):( + pipelines.map((pipeline, index) => { + var bgColor = themeMode === "dark" ? "#212121" : "#FFFFFF"; + if (index % 2 === 0) { + bgColor = themeMode === "dark" ? "#1A1A1A" : "#EAEAEA"; + } + + return ( + + + + + + + + { + const copyContent = `curl -XPOST http://localhost:5160/api/v0/pipeline/delete -H "Content-Type: application/json" -d '{"id":"${pipeline.id}"}' -v` + const copyText = navigator?.clipboard?.writeText(copyContent) + if (copyText) { + toast.success("Pipeline copied to clipboard") + } else { + toast.error("Failed to copy pipeline") + } + }}> + + + + + { + changePipelineState(pipeline, "stop"); + }}> + + + + + )} + /> + + ); + }) + ) + )} +
+ +
+ +
+
+
- + Schedules @@ -903,11 +1192,9 @@ const SchedulesTab = memo((props) => { style={{ textTransform: 'none', fontSize: 16, - color:webhook.status === "running" ? '#1a1a1a' : null, - backgroundColor: webhook.status === "running" ? '#ff8544' : null, width: 150, }} - color={webhook.status === "running" ? "secondary" : "primary"} + color={"secondary"} variant={webhook.status === "running" ? "contained" : "outlined"} disabled={webhook.status === "uninitialized"} onClick={() => { @@ -929,166 +1216,7 @@ const SchedulesTab = memo((props) => { )}
-
- Pipelines - - - Controls Pipelines on your Orborus Runners, e.g. for Log Ingestion, MQ subscriptions or Sigma Detections.{" "} - - Learn more - - - -
- - - -
-
- - - {["Command", "Environment", "Total Runs", "Actions"].map((header, index) => ( - - ))} - - {showLoader ? ( - [...Array(6)].map((_, rowIndex) => { - return ( - - {Array(5) - .fill() - .map((_, colIndex) => { - return ( - - - - ) - })} - - ) - } - ) - - ): ( - pipelines?.length === 0 ? ( -
- No pipeline trigger found -
- - ):( - pipelines.map((pipeline, index) => { - var bgColor = themeMode === "dark" ? "#212121" : "#FFFFFF"; - if (index % 2 === 0) { - bgColor = themeMode === "dark" ? "#1A1A1A" : "#EAEAEA"; - } - - return ( - - - - - - - - )} - /> - - ); - }) - ) - )} -
-
+
@@ -1096,3 +1224,4 @@ const SchedulesTab = memo((props) => { }); export default SchedulesTab; + diff --git a/frontend/src/components/ShuffleCodeEditor1.jsx b/frontend/src/components/ShuffleCodeEditor1.jsx index 3424d3d2..8a26556c 100644 --- a/frontend/src/components/ShuffleCodeEditor1.jsx +++ b/frontend/src/components/ShuffleCodeEditor1.jsx @@ -147,6 +147,8 @@ const CodeEditor = (props) => { // Auto-indent JSON-like content (with safety hehe) const autoIndentContent = React.useCallback((content) => { + return content + // Safety checks :) if (!content || typeof content !== 'string' || content.trim().length === 0) { return content; @@ -173,6 +175,7 @@ const CodeEditor = (props) => { } }, []); + const [localcodedata, setlocalcodedata] = React.useState(codedata === undefined || codedata === null || codedata.length === 0 ? "" : codedata); // const {codelang, setcodelang} = props @@ -1832,6 +1835,7 @@ const CodeEditor = (props) => { display: 'flex', }} > +
{ File Editor ({localcodedata.length})
+ + + { + const indentedText = IndentJsonLikeString(localcodedata, 2) + if (indentedText !== undefined && indentedText !== null) { + setlocalcodedata(indentedText) + } else { + toast.warn("Could not indent the text. Please check the input format.", { autoClose: 5000 }) + } + }} + color="secondary" + > + + + +
:
{ width: 50, marginLeft: 100, }} - disabled={editorData === undefined || editorData.example === undefined || editorData.example === null || editorData.example.length === 0} + disabled={localcodedata === undefined || localcodedata === null || localcodedata.length === 0} onClick={() => { const indentedText = IndentJsonLikeString(localcodedata, 2) if (indentedText !== undefined && indentedText !== null) { diff --git a/frontend/src/components/WorkflowGrid.jsx b/frontend/src/components/WorkflowGrid.jsx index 218cc060..63ea3abb 100644 --- a/frontend/src/components/WorkflowGrid.jsx +++ b/frontend/src/components/WorkflowGrid.jsx @@ -20,6 +20,7 @@ import { Zoom, Chip, } from '@mui/material'; +import { useDebouncedCallback } from "../utils/useDebouncedCallback"; import WorkflowPaper from "../components/WorkflowPaper.jsx" import WorkflowPaperNew from "../components/WorkflowPaperNew.jsx" @@ -172,6 +173,7 @@ const AppGrid = props => { // value={currentRefinement} const SearchBox = ({currentRefinement, refine, isSearchStalled} ) => { var defaultSearch = "" + const [inputValue, setInputValue] = useState("") useEffect(() => { if (window !== undefined && window.location !== undefined && window.location.search !== undefined && window.location.search !== null) { const urlSearchParams = new URLSearchParams(window.location.search) @@ -185,6 +187,12 @@ const AppGrid = props => { } }, []) + useEffect(() => { + setInputValue(currentRefinement || defaultSearch || "") + }, [currentRefinement]) + + const debouncedRefine = useDebouncedCallback((value) => refine(value), 300) + if (localMessage !== inputsearch && inputsearch !== undefined && inputsearch !== null && inputsearch.length > 0) { //setLocalMessage(inputsearch) refine(inputsearch) @@ -217,12 +225,14 @@ const AppGrid = props => { autoComplete='off' type="search" color="primary" - value={currentRefinement} + value={inputValue} placeholder="Find Workflows..." id="shuffle_search_field" onChange={(event) => { removeQuery("q") - refine(event.currentTarget.value) + const value = event.currentTarget.value + setInputValue(value) + debouncedRefine(value) }} onKeyDown={(event) => { if(event.key === "Enter") { diff --git a/frontend/src/views/AgentUI.jsx b/frontend/src/views/AgentUI.jsx index fab6d012..2360a426 100644 --- a/frontend/src/views/AgentUI.jsx +++ b/frontend/src/views/AgentUI.jsx @@ -1,5 +1,6 @@ import React, { useState, useEffect, useContext, memo } from "react"; import { Context } from "../context/ContextApi.jsx"; +import AuthenticationModal from "../components/AuthenticationModal.jsx"; import { useNavigate, Link, useLocation } from "react-router-dom"; import { getTheme } from "../theme.jsx"; import { toast } from "react-toastify" @@ -21,12 +22,16 @@ import { import { CheckCircle as CheckCircleIcon, + Check as CheckIcon, HourglassDisabled as HourglassDisabledIcon, RestartAlt as RestartAltIcon, ExpandMore as ExpandMoreIcon, ExpandLess as ExpandLessIcon, Send as SendIcon, Error as ErrorIcon, + Close as CloseIcon, + OpenInNew as OpenInNewIcon, + Refresh as RefreshIcon, } from '@mui/icons-material' import { @@ -43,21 +48,26 @@ const AgentUI = (props) => { const [data, setData] = useState({}) const [openIndexes, setOpenIndexes] = useState([]) const [disableButtons, setDisableButtons] = useState(false) + const [apps, setApps] = useState([]) + const [appAuth, setAppAuth] = useState([]) - const [originalStartTime, setOriginalStartTime] = useState(0) - const [latestEndTime, setLatestEndTime] = useState(0) const [showAgentStarter, setShowAgentStarter] = useState(false) const [actionInput, setActionInput] = useState("") + const [questionAnswers, setQuestionAnswers] = useState({}) const {themeMode} = useContext(Context) const theme = getTheme(themeMode) const navigate = useNavigate(); + document.title = "Shuffle AI Agents" + const agentWrapperStyle = { width: 1000, height: 1000, margin: "auto", paddingTop: 100, + paddingBottom: 1000, + backgroundColor: theme.palette.backgroundColor, } if (data.input === undefined || data.input === null) { @@ -75,7 +85,22 @@ const AgentUI = (props) => { } if (node_id === undefined || node_id === null || node_id === "") { - return + // Look for AI agent + /* + for (var key in execution_data.results) { + const item = execution_data.results[key] + if (item?.action?.app_name !== "AI Agent") { + continue + } + + node_id = item?.action?.id + break + } + */ + + if (node_id === undefined || node_id === null || node_id === "") { + return + } } var found = false @@ -150,15 +175,22 @@ const AgentUI = (props) => { if (responseJson.success !== false) { if (responseJson.status === "EXECUTING") { // Recursively looking for updates until it's not executing anymore - setTimeout(() => { - GetExecution(execution_id, node_id, authorization) - }, 3000) + //setTimeout(() => { + // GetExecution(execution_id, node_id, authorization) + //}, 3000) } else { setDisableButtons(false) - setDisableButtons(false) } - setExecution(responseJson) + try { + if (JSON.stringify(responseJson) !== JSON.stringify(execution)) { + setExecution(responseJson) + } + } catch(e) { + console.log("Error comparing executions: ", e) + setExecution(responseJson) + } + findNodeData(responseJson, node_id) } else { setDisableButtons(false) @@ -216,12 +248,53 @@ const AgentUI = (props) => { } GetExecution(execution.execution_id, agentActionResult.action.id, execution.authorization) + setTimeout(() => { + GetExecution(execution.execution_id, agentActionResult.action.id, execution.authorization) + }, 10000) }) .catch((error) => { toast.error("Error: " + error) }) } + const getAppAuth = () => { + const url = `${globalUrl}/api/v1/apps/authentication` + fetch(url, { + method: "GET", + credentials: "include", + }) + .then((response) => { + return response.json() + }) + .then((responseJson) => { + if (responseJson.success !== false) { + setAppAuth(responseJson) + } + }) + .catch((error) => { + toast.error("Error in auth load: " + error) + }) + } + + const getApps = () => { + const url = `${globalUrl}/api/v1/apps` + fetch(url, { + method: "GET", + credentials: "include", + }) + .then((response) => { + return response.json() + }) + .then((responseJson) => { + if (responseJson.success !== false) { + setApps(responseJson) + } + }) + .catch((error) => { + toast.error("Error in app load: " + error) + }) + } + useEffect(() => { const params = new URLSearchParams(window.location.search) const executionId = params.get("execution_id") @@ -233,9 +306,15 @@ const AgentUI = (props) => { setShowAgentStarter(true) //toast.warn("No execution ID or node ID provided. Please provide execution_id and node_id in the URL.") } + + getApps() + getAppAuth() }, []) - const maxTimelineWidth = 150 + const maxTimelineWidth = 300 + + var latestEndTime = 0 + var originalStartTime = 0 const TimelineItem = (props) => { const { item, index } = props; const [hovered, setHovered] = useState(false); @@ -258,12 +337,96 @@ const AgentUI = (props) => { const categoryStyle = { - width: 20, - height: 20, + width: 25, + height: 25, marginRight: 10, + borderRadius: 5, } - const parsedCategory = item.category === "singul" ? + + const validate = validateJson(item.details) + const itemStartTime = item.start_time + var itemEndTime = item.end_time + if (item.category === "agent" && itemStartTime !== undefined && itemStartTime !== originalStartTime && (itemStartTime < originalStartTime || originalStartTime === 0)) { + console.log("Rerender 1: ", itemStartTime, originalStartTime) + originalStartTime = itemStartTime + } + + if (itemEndTime !== undefined && itemEndTime > latestEndTime) { + console.log("Rerender 2") + latestEndTime = itemEndTime + } + + if (itemEndTime === undefined || itemEndTime === null) { + // Set it to now + itemEndTime = latestEndTime + } + + if (item.category == "agent" && itemEndTime === 0) { + // Right now -> .toLocaleString() support + itemEndTime = Date.now() / 1000 + + // + + if (itemEndTime > latestEndTime) { + latestEndTime = itemEndTime + } + } + + const totalDuration = latestEndTime - originalStartTime + var currentDuration = itemStartTime - itemEndTime + var timelineMarginLeft = ((itemStartTime - originalStartTime) / totalDuration) * maxTimelineWidth + //var timelineMarginLeft = 0 + + // Calculate how long the div should be + var timelineWidth = ((itemEndTime - itemStartTime) / totalDuration) * maxTimelineWidth + + //console.log("CURRENT DURATION (1): ", currentDuration, itemStartTime, itemEndTime, originalStartTime, latestEndTime, totalDuration, timelineMarginLeft, timelineWidth) + if (totalDuration === currentDuration) { + timelineMarginLeft = 0 + timelineWidth = maxTimelineWidth + } + + // Just for simplicity's sake + if (currentDuration < -1000000 || currentDuration > 1000000) { + currentDuration = 0 + } + + if (currentDuration < 0) { + currentDuration = currentDuration * -1 + } + + const defaultTopPadding = 10 + const open = openIndexes.includes(index) + + var questions = [] + if (item?.details?.action === "finish" || item.category == "finish" || item?.details?.action == "finalise") { + item.type = "finalise" + item.category = "finalise" + item.label = item?.details?.reason || item.label + + } else if (item?.category === "ask" || item?.details?.action === "ask") { + + item.type = "question" + item.category = "ask" + item.label = item?.details?.reason || item.label + + for (var fieldKey in item?.details?.fields) { + const field = item?.details?.fields[fieldKey] + if (field?.key !== "question") { + continue + } + + questions.push({ + "question": field?.value, + "index": questions.length + 1, + }) + } + } else if (item?.details?.action === "api" && item?.details?.tool?.length > 0) { + item.label = item?.details?.reason || item.label + } + + var parsedCategory = item.category === "singul" ? @@ -273,38 +436,167 @@ const AgentUI = (props) => { :
- - const validate = validateJson(item.details) - const itemStartTime = item.start_time - var itemEndTime = item.end_time - if (itemStartTime !== undefined && itemStartTime !== originalStartTime && (itemStartTime < originalStartTime || originalStartTime === 0)) { - console.log("Rerender 1") - //setOriginalStartTime(itemStartTime) + + var showAuthentication = false + var selectedApp = {} + if (item?.details?.tool !== undefined && item?.details?.tool !== null && item?.details?.tool?.length > 0 && item?.details?.tool !== "singul" && item?.details?.tool !== item?.details?.action) { + + // Find the app and inject the image + const toolName = item.details.tool.toLowerCase().replaceAll(" ", "_").replaceAll("-", "_") + for (var appKey in apps) { + const app = apps[appKey] + + const appname = app.name.toLowerCase().replaceAll(" ", "_").replaceAll("-", "_") + if (appname !== toolName) { + continue + } + + if (app.large_image === undefined || app.large_image === null || app.large_image.length === 0) { + break + } + + selectedApp = app + + // Override the category + //item.category = app.name + //item.label = item?.details?.reason || item.label + parsedCategory = + + + + + break + } } - if (itemEndTime !== undefined && itemEndTime > latestEndTime) { - console.log("Rerender 2") - setLatestEndTime(itemEndTime) + if (!showAuthentication) { + if (item?.details?.run_details?.raw_response !== undefined && item?.details?.run_details?.raw_response !== null && item?.details?.run_details?.raw_response?.includes("app_authentication")) { + showAuthentication = true + } } - if (itemEndTime === undefined || itemEndTime === null) { - // Set it to now - itemEndTime = latestEndTime + var questionSubmitDisabled = questions.length === 0 ? true : false + for (var qKey in questions) { + const q = questions[qKey] + if (questionAnswers[q.question] === undefined || questionAnswers[q.question] === null || questionAnswers[q.question] === "") { + //console.log("EMPTY QUESTION: ", q) + questionSubmitDisabled = true + break + } else { + questionSubmitDisabled = false + } } - const totalDuration = latestEndTime - originalStartTime - const currentDuration = itemStartTime - itemEndTime - var timelineMarginLeft = ((itemStartTime - originalStartTime) / totalDuration) * maxTimelineWidth - var timelineWidth = ((itemEndTime - itemStartTime) / totalDuration) * maxTimelineWidth + const barColor = item.status === "FINISHED" ? green : + item.status === "FAILURE" || item.status == "ABORTED" ? red : + item.status === "RUNNING" || item.status === "" ? theme.palette.main : + theme.palette.surfaceColor - if (totalDuration === currentDuration) { - timelineMarginLeft = 0 - timelineWidth = maxTimelineWidth + const rerunAgentButton = + + + { + e.preventDefault() + e.stopPropagation() + + toast.info("Attempting to rerun everything.") + setDisableButtons(true) + + if (item?.details === undefined || item?.details === null || item?.details?.input === undefined || item?.details?.input === null) { + toast.error("No decision details found to rerun. Cannot proceed. Please go back to your workflow or /agents to start over.") + } else { + //console.log("DETAILS: ", item?.details) + for (var messagekey in item?.details?.input?.messages) { + const message = item?.details?.input?.messages[messagekey] + if (message.role === "user") { + setActionInput(message.content) + setDisableButtons(true) + + submitInput(message.content) + //toast.info("Rerun started. Please wait a few seconds and this page should refresh automatically.") + break + } + } + } + }} + > + + + + + + + const rerunButton = + + + { + e.preventDefault() + e.stopPropagation() + + //toast.info("Attempting to rerun this decision by itself.") + setDisableButtons(true) + RerunDecision(item.details) + }} + > + + + + + + const submitQuestions = (decisionId, questionAnswers) => { + console.log("Submitting questions: ", decisionId, questionAnswers) + if (decisionId === undefined || decisionId === null || decisionId === "") { + toast.error("No decision ID provided. Cannot submit answers.") + return + } + + if (Object.keys(questionAnswers).length === 0) { + toast.error("No answers provided. Cannot submit empty answers.") + return + } + + // Loop qu + var newArgument = {} + for (var key in questionAnswers) { + const answer = questionAnswers[key] + newArgument["question_"+(answer.index)] = answer.value + } + + const params = new URLSearchParams(window.location.search) + const executionId = params.get("execution_id") + const nodeId = params.get("node_id") + const authorization = params.get("authorization") + + const url = `${globalUrl}/api/v1/workflows/${executionId}/run?reference_execution=${executionId}&authorization=${authorization}&answer=true¬e=${encodeURIComponent(JSON.stringify(newArgument))}&agentic=true&decision_id=${decisionId}` + console.log("PARSED URL: ", url) + fetch(url, { + method: "GET", + credentials: "include", + }) + .then((response) => { + return response.json() + }) + .then((responseJson) => { + if (responseJson.success !== false) { + setTimeout(() => { + GetExecution(execution.execution_id, agentActionResult.action.id, execution.authorization) + }, 500) + + toast.success("Successfully submitted answers! The agent should continue shortly.") + } else { + toast.warn("Failed to submit answers. Please try again or contact support@shuffler.io if this persists..") + } + }) + .catch((error) => { + toast.error("Problem with submitting: " + error) + }) } - const defaultTopPadding = 10 - const open = openIndexes.includes(index) - return (
{ }} onMouseEnter={() => { if (!hovered) { - console.log("HOVER") + //console.log("HOVER") setHovered(true) } }} @@ -364,41 +656,50 @@ const AgentUI = (props) => {
{parsedCategory}
+ {/*
- {/* To ISO string from unix time */} - {new Date(item.start_time * 1000).toLocaleString()} + {item?.start_time !== undefined && item?.start_time !== null && item?.start_time !== 0 ? + new Date(item.start_time * 1000).toLocaleString() + : + null + } +
+ */}
{item.label}
- +
- {currentDuration !== 0 && !isNaN(timelineMarginLeft) && !isNaN(timelineWidth) && timelineWidth > 0 ? + {currentDuration != 0 && !isNaN(timelineMarginLeft) && !isNaN(timelineWidth) && timelineWidth > 0 ?
- : null} + minHeight: 10, + maxHeight: 10, + borderRadius: theme.palette.borderRadius, + }}> +
+ : + + + }
@@ -407,24 +708,68 @@ const AgentUI = (props) => { maxWidth: 100, display: "flex", }}> - - - { - e.preventDefault() - e.stopPropagation() + {item.category === "ask" ? + + {rerunButton} + {/* + + + { + e.preventDefault() + e.stopPropagation() - toast.info("Attempting to rerun this decision by itself.") - setDisableButtons(true) - RerunDecision(item.details) - }} - > - - + toast.info("Approving this step.") + }} + > + + + + + + + { + e.preventDefault() + e.stopPropagation() + + toast.info("Stopping on this step.") + }} + > + + + + + */} + + + + { + e.preventDefault() + e.stopPropagation() + + //http://localhost:3002/forms/aadfe022-fe93-431c-8634-de42dd7440ac?authorization=9357f6a6-7d59-44be-ad66-be27657369ac&reference_execution=0726378d-b501-470f-b850-f7fb48cd8ca4&source_node=de446bcf-ad37-4337-9f72-e069c7425fac&backend_url=https://ec4245cd2941.ngrok-free.app + const newurl = `/forms/${execution?.workflow?.id}?authorization=${execution.authorization}&reference_execution=${execution.execution_id}&source_node=${agentActionResult?.action?.id}&decision_id=${item.details.run_details.id}&backend_url=${globalUrl}` + window.open(newurl, '_blank', 'noopener,noreferrer'); + }} + > + + + + -
+ : + item.category === "agent" ? + rerunAgentButton + : + rerunButton + } {
+ + {showAuthentication && selectedApp.id !== undefined ? +
+ +
+ : null} + + {questions?.length > 0 && item?.status === "RUNNING" ? +
+ {questions.map((q, questionIndex) => { + return ( +
+ + {`${q.question}`} + + + { + console.log("Change: ", e.target.value) + try { + questionAnswers[q.question] = { + "index": questionIndex, + "value": e.target.value, + } + + setQuestionAnswers({...questionAnswers, }) + } catch (e) { + toast.warn("Something went wrong. Please contact support@shuffler.io. Details: " + e) + } + }} + + /> +
+ ) + })} + + +
+ : null} + {open ?
@@ -479,7 +887,12 @@ const AgentUI = (props) => { const TimelineRender = (props) => { const { agent_data } = props; - const actionResult = execution?.results?.length > 0 ? execution.results[0] : execution + var actionResult = execution?.results?.length > 0 ? execution.results[0] : execution + const validate = validateJson(actionResult?.result) + if (validate.valid === true) { + actionResult.result = validate.result + } + var timelineItems = [ { "label": "AI Agent 2", @@ -493,6 +906,27 @@ const AgentUI = (props) => { }, ] + // Setting up the initial item + if (agent_data?.started_at === undefined && execution?.started_at !== undefined) { + timelineItems[0].start_time = execution?.started_at + } + + if (agent_data?.completed_at === undefined && execution?.completed_at !== undefined) { + timelineItems[0].end_time = execution?.completed_at + } + + // Always prioritise the execution status first + // agent (RUNNING) = workflow (EXECUTING) + if (execution?.status !== undefined) { + timelineItems[0].status = execution?.status + } + + if (actionResult?.result?.status !== undefined && actionResult?.result?.status !== null && actionResult?.result?.status?.length > 0) { + if (timelineItems[0].status !== "FINISHED" && timelineItems[0].status !== "ABORTED" && timelineItems[0].status !== "FAILURE") { + timelineItems[0].status = actionResult?.result?.status + } + } + // Autofixer for result lol if ((agent_data?.decisions === undefined || agent_data?.decisions === null)) { const verifiedInput = validateJson(actionResult?.result) @@ -500,6 +934,7 @@ const AgentUI = (props) => { agent_data.decisions = verifiedInput.result?.decisions setAgentActionResult(actionResult) + } } @@ -516,13 +951,13 @@ const AgentUI = (props) => { } var newTimelineItem = { - "label": item.action, + "label": item?.action, "type": "decision", - "category": item.category, + "category": item?.category, - "status": item.run_details.status, - "start_time": item.run_details.started_at, - "end_time": item.run_details.completed_at, + "status": item?.run_details?.status, + "start_time": item?.run_details?.started_at, + "end_time": item?.run_details?.completed_at, } newTimelineItem.details = item @@ -577,6 +1012,14 @@ const AgentUI = (props) => { setAgentRequestLoading(true) //setShowAgentStarter(false); //GetExecution(execution?.execution_id, execution?.node_id, execution?.authorization); + // + setData({}) + setExecution(null) + setAgentRequestLoading(true) + setShowAgentStarter(true) + setActionInput(inputText) + + setAgentActionResult(null) if (inputText === undefined || inputText === null || inputText === "") { toast.error("Please provide a valid input for the AI Agent.") @@ -606,7 +1049,7 @@ const AgentUI = (props) => { }, { "name":"action", - "value":"list_tickets" + "value":"list_tickets,API" } ]} @@ -637,18 +1080,38 @@ const AgentUI = (props) => { } + const handleKeyDown = (e) => { + const isCmdEnter = e.metaKey && e.key === "Enter"; // macOS + const isCtrlEnter = e.ctrlKey && e.key === "Enter"; // Windows/Linux + if (isCmdEnter || isCtrlEnter) { + e.preventDefault() + submitInput(actionInput) + } + } + return (
+ {showAgentStarter ? - { - e.preventDefault(); - submitInput(actionInput); - }}> + { + e.preventDefault(); + submitInput(actionInput); + }} + > +
@@ -661,7 +1124,7 @@ const AgentUI = (props) => { style={{width: 450, marginRight: 20, marginTop: 30, }} multiline minRows={2} - defaultValue={execution?.execution_id || ""} + defaultValue={actionInput || ""} onChange={(e) => { setActionInput(e.target.value) }} @@ -704,6 +1167,22 @@ const AgentUI = (props) => { + + + + + + {buttonState === "timeline" ? : diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index e1adeb09..88adac1b 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -22,6 +22,7 @@ import { CodeHandler, Img, OuterLink, } from "../views/Docs.jsx"; import { InstantSearch, Configure, connectSearchBox, connectHits, Index } from 'react-instantsearch-dom'; import algoliasearch from 'algoliasearch/lite'; +import useDebouncedCallback from "../utils/useDebouncedCallback.js"; import { Zoom, Fade, @@ -275,7 +276,7 @@ export const triggers = [ { "name": "alertinfo", "example": "", - "value": "Do you want to continue the workflow? Start parameters: $exec", + "value": "## Stop or continue?\n\nDetails: $exec", }, { "name": "options", @@ -1259,6 +1260,24 @@ const AngularWorkflow = (defaultprops) => { "multiline": true, }] }, + /* + // An attempt at handling APIs directly. This ~kind of works + { + "name": "API", + "description": "Attempts to take your fields and run an API call with them, whatever they are", + "label": "Custom Action", + "example": "{\"source_data\": \"{\\\"event\\\": \\\"login\\\", \\\"user\\\": \\\"john_doe\\\", \\\"timestamp\\\": \\\"2023-10-01T12:00:00Z\\\"}\", \"standard\": \"OCSF\"}", + "parameters": [ + { + "name": "fields", + "value": "", + "description": "A JSON object with the fields to send to the API. Example: {\"url\": \"hello\", \"key2\": \"value2\"}", + "required": true, + "multiline": true, + } + ] + }, + */ { "name": "Translate standard", "description": "Translates your JSON data into a standard formats, then stores it in the Shuffle Datastore", @@ -11589,10 +11608,16 @@ const AngularWorkflow = (defaultprops) => { }; const handleDragStop = (e, app) => { + if (cy === undefined || cy == null) { + console.log("Cytoscape not initialized") + return + } + var currentnode = cy.getElementById(newNodeId); if (currentnode === undefined || currentnode === null || currentnode.length === 0) { - return; + console.log("No current node found") + return } if (parsedApp === undefined || parsedApp === null || parsedApp.data === undefined || parsedApp.data === null) { @@ -12411,11 +12436,20 @@ const AngularWorkflow = (defaultprops) => { }; const SearchBox = ({ currentRefinement, refine, isSearchStalled, }) => { + const debouncedRefine = useDebouncedCallback(refine, 500) + const lastRefinedRef = useRef(currentRefinement) + + const safeRefine = (value) => { + if (value === lastRefinedRef.current) return + lastRefinedRef.current = value + debouncedRefine(value) + } + if (document !== undefined) { const appsearchValue = document.getElementById("appsearch") if (appsearchValue !== undefined && appsearchValue !== null) { if (appsearchValue.value !== undefined && appsearchValue.value !== null && appsearchValue.value.length > 0) { - refine(appsearchValue.value) + safeRefine(appsearchValue.value) } } } @@ -12448,8 +12482,7 @@ const AngularWorkflow = (defaultprops) => { //if (event.currentTarget.value.length > 0 && !searchOpen) { // setSearchOpen(true) //} - - refine(event.currentTarget.value) + safeRefine(event.currentTarget.value) }} limit={5} /> @@ -14725,7 +14758,7 @@ const AngularWorkflow = (defaultprops) => { zIndex: 10000, }} > - Conditions can't be used for loops [ .# ]{" "} + PS: Conditions can't be used for loops [ .# ]. Use the filters list action.{" "} { const shownErrors = !isMobile && workflow.errors !== undefined && workflow.errors !== null && workflow.errors.length > 0 && showErrors && (!workflow.public || userdata.support === true) ?
{ style={{ float: "right", marginTop: 20, }} // Max 5 days in the past - disabled={userdata.region_url !== "https://shuffler.io" || executionData.started_at < (Math.floor(Date.now() / 1000) - 432000)} + disabled={executionData.started_at < (Math.floor(Date.now() / 1000) - 432000)} onClick={() => { toast("Opening logs in a new tab") setTimeout(() => { - window.open(`/api/v1/workflows/search/${executionData.execution_id}`, "_blank") + window.open(`${globalUrl}/api/v1/workflows/search/${executionData.execution_id}`, "_blank") }, 250) }} > diff --git a/frontend/src/views/ApiExplorerWrapper.jsx b/frontend/src/views/ApiExplorerWrapper.jsx index d104d43d..81d3cd39 100644 --- a/frontend/src/views/ApiExplorerWrapper.jsx +++ b/frontend/src/views/ApiExplorerWrapper.jsx @@ -1494,13 +1494,13 @@ const ApiExplorerWrapper = (props) => { />
+ style={{ + backgroundColor: theme.palette.inputColor, + padding: 15, + borderRadius: theme.palette?.borderRadius, + marginBottom: 30, + }} + > There is no Shuffle-specific documentation for this app yet outside of the general description above. Documentation is written for each api, and is a community effort. We hope to see your contribution! diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 414a7af0..26f1179b 100755 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -8,6 +8,7 @@ import { Typography, FormControlLabel, Button, + ButtonGroup, Divider, Select, MenuItem, @@ -2680,11 +2681,11 @@ const AppCreator = (defaultprops) => { setErrorCode(responseJson.reason); if (responseJson?.details !== undefined && responseJson?.details !== null) { - toast.error("Failed to build - contact support@shuffler.io: " + responseJson.details, { + toast.error("Failed to build - contact support@shuffler.io:\n\n" + responseJson.details, { autoClose: 60000 }) } else { - toast.error("Failed to build: " + responseJson.reason, { + toast.error("Failed to build: \n\n" + responseJson?.reason, { autoClose: 10000 }) } @@ -2930,7 +2931,7 @@ const AppCreator = (defaultprops) => { Query -
+ {index === extraAuth.length - 1 ? ( -
+ ); })} @@ -3431,13 +3432,13 @@ const AppCreator = (defaultprops) => { const ActionPaper = (props) => { const { data, index } = props - const [updater, setUpdater] = useState("tmp"); - const [actionsModalOpen, setActionsModalOpen] = useState(false); - const [urlPath, setUrlPath] = useState(""); - const [fileUploadEnabled, setFileUploadEnabled] = useState(false); - const [currentActionMethod, setCurrentActionMethod] = useState(actionNonBodyRequest[0]) - const [extraBodyFields, setExtraBodyFields] = useState([]); - const [urlPathQueries, setUrlPathQueries] = useState([]); + const [updater, setUpdater] = useState("tmp"); + const [actionsModalOpen, setActionsModalOpen] = useState(false); + const [urlPath, setUrlPath] = useState(""); + const [fileUploadEnabled, setFileUploadEnabled] = useState(false); + const [currentActionMethod, setCurrentActionMethod] = useState(actionNonBodyRequest[0]) + const [extraBodyFields, setExtraBodyFields] = useState([]); + const [urlPathQueries, setUrlPathQueries] = useState([]); const [currentAction, setCurrentAction] = useState({ name: "", file_field: "", @@ -3454,6 +3455,10 @@ const AppCreator = (defaultprops) => { required_bodyfields: [], }); + useEffect(() => { + console.log("Queries: ", urlPathQueries) + }, [urlPathQueries]) + const findBodyParams = (body) => { const regex = /\${(\w+)}/g; const found = body.match(regex); @@ -3462,7 +3467,7 @@ const AppCreator = (defaultprops) => { } else { setExtraBodyFields(found); } - }; + }; const UrlPathParameters = () => { const values = getCurrentPaths(urlPath); @@ -3495,28 +3500,27 @@ const AppCreator = (defaultprops) => { ) : null; }; - const HandleIndividualChip = (props) => { - const { chipData, index } = props; - const [chipRequired, setChipRequired] = useState(currentAction.required_bodyfields !== undefined ? currentAction.required_bodyfields.includes(chipData) : false); + const { chipData, index } = props; + const [chipRequired, setChipRequired] = useState(currentAction.required_bodyfields !== undefined ? currentAction.required_bodyfields.includes(chipData) : false); const parsedChip = chipData.startsWith("${") && chipData.endsWith("}") ? chipData.substring(2, chipData.length - 1) : chipData - return ( - - { + return ( + + { if (chipRequired) { currentAction["required_bodyfields"].splice(currentAction["required_bodyfields"].indexOf(chipData), 1) } else { @@ -3524,27 +3528,28 @@ const AppCreator = (defaultprops) => { } setCurrentAction(currentAction); - setChipRequired(!chipRequired); - }} - /> - - ); - }; - - const setActionField = (field, value) => { - currentAction[field] = value - setCurrentAction(currentAction) - - //setUrlPathQueries(currentAction.queries) + setChipRequired(!chipRequired); + }} + /> + + ); }; - const addPathQuery = () => { + const setActionField = (field, value) => { + currentAction[field] = value + setCurrentAction(currentAction) + + //setUrlPathQueries(currentAction.queries) + }; + + const addPathQuery = () => { urlPathQueries.push({ name: "", required: true, example: "", }); if (updater === "addupdater") { setUpdater("updater"); } else { setUpdater("addupdater"); } + setUrlPathQueries(urlPathQueries); }; @@ -3555,6 +3560,7 @@ const AppCreator = (defaultprops) => { } else { setUpdater("flipupdater"); } + setUrlPathQueries(urlPathQueries); }; @@ -3573,7 +3579,7 @@ const AppCreator = (defaultprops) => { } }; - const loopQueries = urlPathQueries.length === 0 ? null : ( + const loopQueries = urlPathQueries.length === 0 ? null : (
{ return (
-
- - Click required to flip - - } - onBlur={(e) => { - console.log("IN BLUR: ", e.target.value); - urlPathQueries[queryIndex].name = e.target.value.replaceAll("=", ""); - setUrlPathQueries(urlPathQueries); - }} - style={{flex: 3}} - InputProps={{ - style: { - color: theme.palette.text.primary, - }, - }} - /> - { - urlPathQueries[queryIndex].example = e.target.value.replaceAll( - "=", - "" - ) - - setUrlPathQueries(urlPathQueries) - }} - style={{flex: 2}} - InputProps={{ - style: { - color: theme.palette.text.primary, - }, - }} - /> -
+
+ { + urlPathQueries[queryIndex].name = e.target.value.replaceAll("=", "") + setUrlPathQueries(urlPathQueries) + }} + style={{flex: 3}} + InputProps={{ + style: { + color: theme.palette.text.primary, + }, + }} + /> + { + // E.g. for Jira -> JQL -> requires = in param + urlPathQueries[queryIndex].example = e.target.value.replaceAll("=","=") + setUrlPathQueries(urlPathQueries) + }} + style={{flex: 2}} + InputProps={{ + style: { + color: theme.palette.text.primary, + }, + }} + /> +
{ @@ -3654,7 +3651,7 @@ const AppCreator = (defaultprops) => { deletePathQuery(queryIndex); }} > - +
); @@ -4106,22 +4103,22 @@ const AppCreator = (defaultprops) => { if (request.header !== undefined && request.header !== null) { var headers = []; for (let [key, value] of Object.entries(request.header)) { - if (value === undefined) { - if (key.includes(":")) { - const keysplit = key.split(":") - key = keysplit[0].trim() - value = keysplit[1].trim() + if (value === undefined) { + if (key.includes(":")) { + const keysplit = key.split(":") + key = keysplit[0].trim() + value = keysplit[1].trim() - } else if (key.includes("=")) { - const keysplit = key.split("=") - key = keysplit[0].trim() - value = keysplit[1].trim() + } else if (key.includes("=")) { + const keysplit = key.split("=") + key = keysplit[0].trim() + value = keysplit[1].trim() - } else { - toast("Removed key: ", key) - continue - } - } + } else { + toast("Removed key: ", key) + continue + } + } if ( parameterName !== undefined && @@ -4392,9 +4389,8 @@ const AppCreator = (defaultprops) => { variant={urlPath.length > 0 ? "contained" : "outlined"} style={{ }} onClick={() => { - //console.log(urlPathQueries) - //console.log(urlPath) console.log(currentAction); + const errors = getActionErrors(); addActionToView(errors); setActionsModalOpen(false); @@ -4460,7 +4456,7 @@ const AppCreator = (defaultprops) => { return ( - {newActionModal} + {newActionModal} {error} diff --git a/frontend/src/views/AppExplorer.jsx b/frontend/src/views/AppExplorer.jsx index 1f434a83..38e7cc4f 100644 --- a/frontend/src/views/AppExplorer.jsx +++ b/frontend/src/views/AppExplorer.jsx @@ -3075,7 +3075,7 @@ const buttonBackground = "linear-gradient(to right, #f86a3e, #f34079)"; const appEnding = app?.public === true ? app?.app_version : app?.id - return `curl -L \ \\\n "${globalUrl}/api/v1/download_docker_image?image=frikky/shuffle:${app?.name.toLowerCase().replaceAll(' ', '_')}_${appEnding}" \\\n -H \"Authorization: Bearer APIKEY" \\\n -o image.zip; \\\n docker load -i image.zip` + return `curl -L \ \\\n "${globalUrl}/api/v1/download_docker_image?image=frikky/shuffle:${app?.name.toLowerCase().replaceAll(' ', '_')}_${appEnding}" \\\n -H \"Authorization: Bearer APIKEY" \\\n -o image.zip; \\\n docker load -i image.zip${!app?.public ? ` \\\n docker tag frikky/shuffle:${app?.name.toLowerCase().replaceAll(' ', '_')}_${appEnding} frikky/shuffle:${app?.name.toLowerCase().replaceAll(' ', '_')}_${app.app_version}` : ``}` } const renderedActionOptions = deduplicateByName(( @@ -3405,7 +3405,7 @@ const buttonBackground = "linear-gradient(to right, #f86a3e, #f34079)";
+
@@ -2365,6 +2417,7 @@ const Apps2 = (props) => {
+ ); }; diff --git a/frontend/src/views/DashboardViews.jsx b/frontend/src/views/DashboardViews.jsx index 0748e80a..98d03344 100644 --- a/frontend/src/views/DashboardViews.jsx +++ b/frontend/src/views/DashboardViews.jsx @@ -8,7 +8,7 @@ import { useNavigate, Link, useParams } from "react-router-dom"; import { ToastContainer, toast } from "react-toastify" import Draggable from "react-draggable"; -import DashboardBarchart, { LoadStats } from '../components/DashboardBarchart.jsx'; +import { LoadStats } from '../components/LineChartWrapper.jsx'; import { Autocomplete, @@ -828,9 +828,14 @@ const Dashboard = (props) => { }
- diff --git a/frontend/src/views/Docs.jsx b/frontend/src/views/Docs.jsx index 8b4772c6..ac1babe2 100755 --- a/frontend/src/views/Docs.jsx +++ b/frontend/src/views/Docs.jsx @@ -400,6 +400,10 @@ const Docs = (defaultprops) => { if (propkey === "app_creation") { navigate('/docs/apps#app-creation-introduction') } + + if (propkey === "api") { + navigate('/docs/API') + } } @@ -690,7 +694,8 @@ const Docs = (defaultprops) => { const Heading = (props) => { const [hover, setHover] = useState(false); - var id = props.children[0].toLowerCase().toString() + + var id = (props.children?.[0] ?? props.children ?? '').toString().toLowerCase(); if (props.level <= 3) { id = props.children[0].toLowerCase().toString().replaceAll(" ", "-"); } diff --git a/frontend/src/views/NewDashboard.jsx b/frontend/src/views/NewDashboard.jsx new file mode 100644 index 00000000..630e6dec --- /dev/null +++ b/frontend/src/views/NewDashboard.jsx @@ -0,0 +1,266 @@ +import React, { useEffect, useState, useContext, useRef, useCallback } from 'react'; +import { + Typography, + Grid, + Paper, + Box, + Stack, + Chip, + Avatar, + Divider, + Select, + MenuItem, +} from '@mui/material'; +import TrendingUpIcon from '@mui/icons-material/TrendingUp'; +import TrendingDownIcon from '@mui/icons-material/TrendingDown'; +import ErrorOutlineIcon from '@mui/icons-material/ErrorOutline'; +import TaskAltIcon from '@mui/icons-material/TaskAlt'; +import SuccessFailedRunsWidget from '../components/SuccessFailedRunsWidget.jsx'; +import RunsOverTimeWidget from '../components/RunsOverTimeWidget.jsx'; +import { Context } from '../context/ContextApi.jsx'; +import CircularProgress from '@mui/material/CircularProgress'; +import { useNavigate } from 'react-router-dom'; +import DashboardOnboarding from '../components/DashboardOnboarding.jsx'; + +const NewDashboard = (props) => { + const { globalUrl, userdata } = props; + + // const [workflows, setWorkflows] = useState([]); + const { leftSideBarOpenByClick } = useContext(Context); + const [sfwControls, setSfwControls] = useState(null); + const [loadingSfw, setLoadingSfw] = useState(true); + const [loadingRot, setLoadingRot] = useState(true); + const [loadingNoti, setLoadingNoti] = useState(true); + const [showOverlay, setShowOverlay] = useState(true); + const [totals, setTotals] = useState({ days: 30, mode: 'workflows', totalRuns: 0, successRuns: 0, failedRuns: 0, activeDays: 0, timeSavedMinutes: 0, moneySavedDollars: 0 }); + const [notifications, setNotifications] = useState([]); + const [onboardingOpen, setOnboardingOpen] = useState(() => { + try { + return localStorage.getItem("dashboard_onboarding_complete") === "true" ? false : true; + } catch { + return true; + } + }); + const [overrideDays, setOverrideDays] = useState(undefined); + const [rotMonthOverride, setRotMonthOverride] = useState(undefined); + + const navigate = useNavigate(); + const handleSfwControls = useCallback((node) => { + setSfwControls(node); + }, []); + + const formatCurrencyCompact = (value) => { + const n = Math.max(0, Number(value) || 0); + const abs = Math.abs(n); + const fmt = (x, suffix) => `${(Math.round(x * 10) / 10).toString().replace(/\.0$/, '')}${suffix}`; + if (abs >= 1e9) return `$${fmt(n / 1e9, 'B')}`; + if (abs >= 1e6) return `$${fmt(n / 1e6, 'M')}`; + if (abs >= 1e3) return `$${fmt(n / 1e3, 'k')}`; + return `$${Math.round(n).toLocaleString()}`; + }; + + const formatTimeDisplay = (mins) => { + const totalMins = Math.max(0, Math.round(mins || 0)); + if (totalMins < 60) return { display: `${totalMins}m`, title: `${totalMins} minutes` }; + const totalHours = Math.floor(totalMins / 60); + if (totalHours >= 24) { + const days = Math.floor(totalHours / 24); + return { display: `${days}d`, title: `${totalHours} hours` }; + } + return { display: `${totalHours}h`, title: `${totalHours} hours` }; + }; + + const timeFmt = formatTimeDisplay(totals.timeSavedMinutes); + const STATIC_TIME_PERCENT = '62%'; + const STATIC_MONEY_PERCENT = '46%'; + + const unreadCount = notifications.filter(n => n && n.read === false).length; + const readCount = notifications.filter(n => n && n.read === true).length; + + // Current values + // 1 Workflow run = 15 minutes + // 1 Workflow run = $25 + + const kpis = [ + { value: timeFmt.display, title: timeFmt.title, label: 'Time saved', icon: , percentage: STATIC_TIME_PERCENT, color: '#5cc879' }, + { value: formatCurrencyCompact(totals.moneySavedDollars), label: 'Money saved', icon: , percentage: STATIC_MONEY_PERCENT, color: '#5cc879' }, + { value: String(unreadCount), label: 'Total errors', icon: , percentage: "", color: '#f87171' }, + { value: String(readCount), label: 'Errors resolved', icon: , percentage: "", color: '#5cc879' }, + ]; + + const getGreeting = () => { + try { + const hour = new Date().getHours(); + if (hour < 5) return 'Good night'; + if (hour < 12) return 'Good morning'; + if (hour < 18) return 'Good afternoon'; + return 'Good evening'; + } catch { + return 'Hey'; + } + }; + + const displayName = userdata !== undefined && userdata?.username !== undefined ? userdata?.username?.split('@')[0]?.charAt(0)?.toUpperCase() + userdata?.username?.split('@')[0]?.slice(1) : 'User'; + + useEffect(() => { + let t; + const anyLoading = loadingSfw || loadingRot || loadingNoti; + if (anyLoading) { + t = setShowOverlay(true); + } else { + setShowOverlay(false); + } + return () => { if (t) clearTimeout(t); }; + }, [loadingSfw, loadingRot, loadingNoti]); + + // Auto-open onboarding when there aren't enough active days of stats + useEffect(() => { + try { + const alreadyDone = localStorage.getItem("dashboard_onboarding_complete") === "true"; + if (alreadyDone) { + setOnboardingOpen(false); + return; + } + const active = Number(totals?.activeDays || 0); + setOnboardingOpen(active < 5); + } catch { + setOnboardingOpen(true); + } + }, [totals?.activeDays]); + + // Load notifications + useEffect(() => { + const loadNotifications = async () => { + try { + const resp = await fetch(`${globalUrl}/api/v1/notifications`, { + method: 'GET', + credentials: 'include', + headers: { 'Content-Type': 'application/json' }, + }); + if (resp.status !== 200) { + setNotifications([]); + return; + } + const data = await resp.json(); + const list = Array.isArray(data?.notifications) ? data.notifications : (Array.isArray(data) ? data : []); + setNotifications(list.filter(Boolean)); + } catch (e) { + setNotifications([]); + } finally { + setLoadingNoti(false); + } + }; + + loadNotifications(); + }, [globalUrl]); + + // useEffect(() => { + // // Lightweight workflows list for selector in success/failed widget + // const loadWorkflows = async () => { + // try { + // const resp = await fetch(`${globalUrl}/api/v1/workflows`, { + // method: 'GET', + // credentials: 'include', + // headers: { 'Content-Type': 'application/json' }, + // }); + // if (resp.status !== 200) { + // return; + // } + // const data = await resp.json(); + // const list = Array.isArray(data?.workflows) ? data.workflows : (Array.isArray(data) ? data : []); + // const normalized = list.filter(Boolean).map((w, idx) => ({ id: w?.id || w?.ID || `${idx}`, name: w?.name || w?.Name || `Workflow ${idx+1}` })); + // setWorkflows(normalized); + // } catch (e) { + // // ignore + // } + // }; + + // loadWorkflows(); + // }, [globalUrl]); + + return ( +
+ setOnboardingOpen(false)} + onExplore={() => { + // Ensure overrides are set before closing modal + setOverrideDays(5); + setRotMonthOverride(new Date(new Date().getFullYear(), new Date().getMonth(), 1)); + + // Close modal immediately to trigger data fetching + setOnboardingOpen(false); + }} + headerTitle="Unlock your Dashboard" + headerSubtitle="Complete these steps to start seeing insights." + /> + {showOverlay && ( +
+
+ + Loading dashboard… +
+
+ )} + {/* Header / Greeting */} + + {`${getGreeting()}, ${displayName ?? 'User'}!`} + <> + {sfwControls} + + + + {/* KPI cards */} + + {kpis.map((kpi) => ( + + { + if (kpi.label.toLowerCase().includes('total errors')) { + // navigate to notifications page + navigate('/admin?admin_tab=notifications'); + } + }} + + > + + + {kpi.value} + {kpi.label} + + + {kpi.icon} + {kpi.percentage} + + + + + ))} + + + {/* Success/Failed widget uses its own internal sub-cards; make wrapper transparent */} + + + + + {/* Runs over time section */} + + + +
+ ); +}; + +export default NewDashboard; + + diff --git a/frontend/src/views/RunWorkflow.jsx b/frontend/src/views/RunWorkflow.jsx index 654646b7..cf8e9ad8 100644 --- a/frontend/src/views/RunWorkflow.jsx +++ b/frontend/src/views/RunWorkflow.jsx @@ -72,6 +72,7 @@ const RunWorkflow = (defaultprops) => { const [executionLoading, setExecutionLoading] = useState(false); const [executionData, setExecutionData] = React.useState({}); const [executionRunning, setExecutionRunning] = useState(false); + const [disableButtons, setDisableButtons] = useState(false); const [workflowQuestion, setWorkflowQuestion] = useState(""); const [selectedOrganization, setSelectedOrganization] = React.useState(undefined); const [apps, setApps] = React.useState([]); @@ -84,12 +85,14 @@ const RunWorkflow = (defaultprops) => { const [workflows, setWorkflows] = React.useState([]) const [boxWidth, setBoxWidth] = React.useState(500) const [inputQuestions, setInputQuestions] = React.useState([]) + const [agentic, setAgentic] = React.useState(false) const searchParams = new URLSearchParams(window.location.search) const answer = searchParams.get("answer") const execution_id = searchParams.get("reference_execution") const authorization = searchParams.get("authorization") const sourceNode = searchParams.get("source_node") + const decisionId = searchParams.get("decision_id") // ONLY for agentic workflows const backendUrl = searchParams.get("backend_url") || globalUrl useEffect(() => { @@ -162,11 +165,8 @@ const RunWorkflow = (defaultprops) => { } } - // Used to swap from login to register. True = login, false = register - // Error messages etc const [executionInfo, setExecutionInfo] = useState(""); - const handleValidateForm = (executionArgument) => { // Check if every field exists if (executionArgument === undefined || executionArgument === null) { @@ -184,9 +184,12 @@ const RunWorkflow = (defaultprops) => { } } - //console.log("EXEC: ", executionArgument) + // FIXME: Error with User Input + Required arg (?) + // Somehow validation is not happening as it should, and it just checks all + // questions if none are selected for (var key in executionArgument) { if (executionArgument[key] === undefined || executionArgument[key] === null || executionArgument[key] === "") { + console.log("Unanswered, required question: ", key) return false } } @@ -334,17 +337,18 @@ const RunWorkflow = (defaultprops) => { } const validate = validateJson(executionData.result) - return (
{workflowQuestion !== "" ? null : - +
} {workflowQuestion !== "" ? null : validate.valid === false ?
- + {validate?.result !== undefined && validate?.result !== null && validate?.result.length > 0 ? + + : null } { stop() setMessage("") - setExecutionLoading(true) setExecutionData({}) setExecutionInfo("") + setTimeout(() => { + setExecutionLoading(true) + }, 2500) + var data = { "execution_argument": executionArgument, "execution_source": "form", @@ -462,6 +469,14 @@ const RunWorkflow = (defaultprops) => { fetchBody.body = JSON.stringify(data) } + if (agentic === true) { + if (url.includes("?")) { + url += `&agentic=true&decision_id=${decisionId}` + } else { + url += `?agentic=true&decision_id=${decisionId}` + } + } + // IF there is an execution argument, we should use it fetch(url, fetchBody) .then((response) => { @@ -480,25 +495,30 @@ const RunWorkflow = (defaultprops) => { } } - if ((response.status === 401 || response.status === 403) && authorization === undefined || authorization === null || authorization.length === 0) { - toast(`This form is not available for you to run. If you this is an error, contact ${supportEmail} with a link to this form`) - } + //if ((response.status === 401 || response.status === 403) && authorization === undefined || authorization === null || authorization?.length === 0) { + // toast(`This form is not available for you to run. If you this is an error, contact ${supportEmail} with a link to this form (2)`) + //} return response.json() }) .then(responseJson => { + //if (responseJson.success === true) { + // setDisableButtons(true) + //} + setExecutionLoading(false) - if (responseJson.execution_id !== undefined && responseJson.execution_id !== null && responseJson.execution_id.length > 0) { + if (responseJson?.execution_id !== undefined && responseJson?.execution_id !== null && responseJson?.execution_id?.length > 0) { navigate(`?execution_id=${responseJson.execution_id}`) } if (responseJson.success === false) { + console.log("Failed sending execution request") - if (responseJson.reason !== undefined && responseJson.reason !== null) { + if (responseJson?.reason !== undefined && responseJson?.reason !== null) { if (responseJson?.reason?.toLowerCase().includes("already clicked")) { - setMessage("Already answered. You may close this window (2).") + setMessage("This form has been answered. You may close this window.") } else { - toast.warn(responseJson.reason) + toast.warn(responseJson?.reason) } } @@ -520,11 +540,17 @@ const RunWorkflow = (defaultprops) => { setExecutionRequest(responseJson) start() } + + // If execution_id or authorization, add them to the URL + if (responseJson?.execution_id !== undefined && responseJson?.execution_id !== null && responseJson?.execution_id?.length > 0 && responseJson?.authorization !== undefined && responseJson?.authorization !== null && responseJson?.authorization?.length > 0) { + navigate(`?execution_id=${responseJson.execution_id}&authorization=${responseJson.authorization}`) + } } }) .catch(error => { //setExecutionInfo("Error in workflow startup: " + error) - toast.warn("Error submitting form. Please try again.") + console.log("Error starting workflow: ", error) + toast.warn(`Error submitting form. Please try again: ${error}`) stop() setMessage("") @@ -597,8 +623,8 @@ const RunWorkflow = (defaultprops) => { if (realtimeMarkdown !== undefined && realtimeMarkdown !== null && realtimeMarkdown.length > 0) { const newmarkdown = realtimeMarkdown.replace(`{{ ${workflow_id} }}`, "", -1) setRealtimeMarkdown(newmarkdown) - } else if (inputWorkflow.form_control.input_markdown !== undefined && inputWorkflow.form_control.input_markdown !== null && inputWorkflow.form_control.input_markdown.length > 0) { - const newmarkdown = inputWorkflow.form_control.input_markdown.replace(`{{ ${workflow_id} }}`, "", -1) + } else if (inputWorkflow?.form_control?.input_markdown !== undefined && inputWorkflow?.form_control?.input_markdown !== null && inputWorkflow?.form_control?.input_markdown.length > 0) { + const newmarkdown = inputWorkflow?.form_control?.input_markdown.replace(`{{ ${workflow_id} }}`, "", -1) setRealtimeMarkdown(newmarkdown) } } @@ -608,10 +634,10 @@ const RunWorkflow = (defaultprops) => { console.log("Get workflow error: ", error.toString()) if (realtimeMarkdown !== undefined && realtimeMarkdown !== null && realtimeMarkdown.length > 0) { - const newmarkdown = inputWorkflow.form_control.input_markdown.replace(`{{ ${workflow_id} }}`, "", -1) + const newmarkdown = inputWorkflow?.form_control?.input_markdown.replace(`{{ ${workflow_id} }}`, "", -1) setRealtimeMarkdown(newmarkdown) - } else if (inputWorkflow.form_control.input_markdown !== undefined && inputWorkflow.form_control.input_markdown !== null && inputWorkflow.form_control.input_markdown.length > 0) { - const newmarkdown = inputWorkflow.form_control.input_markdown.replace(`{{ ${workflow_id} }}`, "", -1) + } else if (inputWorkflow?.form_control?.input_markdown !== undefined && inputWorkflow?.form_control?.input_markdown !== null && inputWorkflow?.form_control?.input_markdown.length > 0) { + const newmarkdown = inputWorkflow?.form_control?.input_markdown.replace(`{{ ${workflow_id} }}`, "", -1) setRealtimeMarkdown(newmarkdown) } }) @@ -646,6 +672,7 @@ const RunWorkflow = (defaultprops) => { trig.parameters = [] } + newexec = {} for (var paramkey in trig.parameters) { const param = trig.parameters[paramkey] if (param.name !== "input_questions") { @@ -683,6 +710,7 @@ const RunWorkflow = (defaultprops) => { } } + console.log("Setting exec arg: ", newexec) setExecutionArgument(newexec) } @@ -733,10 +761,10 @@ const RunWorkflow = (defaultprops) => { setInputQuestions(workflow.input_questions) } - if (workflow.form_control.input_markdown !== undefined && workflow.form_control.input_markdown !== null && workflow.form_control.input_markdown.length > 0) { + if (workflow?.form_control?.input_markdown !== undefined && workflow?.form_control?.input_markdown !== null && workflow?.form_control?.input_markdown.length > 0) { // Look for {{ uuid }} format, and try to run that workflow with their account // This is a hack, but a fun one. - var newmarkdown = workflow.form_control.input_markdown.replace("", "") + var newmarkdown = workflow?.form_control?.input_markdown.replace("", "") const uuidRegex = /{{\s[a-f0-9-]+\s}}/g const found = newmarkdown.match(uuidRegex) @@ -784,8 +812,8 @@ const RunWorkflow = (defaultprops) => { } } - if (workflow.status !== "WAITING") { - setMessage("Already answered. You may close this window (3).") + if (workflow.status === "EXECUTING" || workflow.status === "SUCCESS" || workflow.status === "ABORTED" || workflow.status === "STOPPED" || workflow.status === "FAILURE" || workflow.status === "FINISHED") { + setMessage("Already handled. You may close this window.") } } @@ -806,13 +834,17 @@ const RunWorkflow = (defaultprops) => { console.log("Status not 200 for workflows :O!"); } - if ((response.status === 401 || response.status === 403) && authorization === undefined || authorization === null || authorization.length === 0) { - toast(`This form is not available to you. If you think this is an error, please contact ${supportEmail} with the URL.`) - } + //if (response.status >= 400 && authorization === undefined || authorization === null || authorization.length === 0) { + // toast.warn(`This form may not be available to you. If you think this is an error, please contact ${supportEmail} with the URL.`) + //} return response.json() }) .then((responseJson) => { + if (responseJson.success === false) { + return + } + // Not sure why this is necessary. if (responseJson.isValid === undefined) { responseJson.isValid = true; @@ -1008,14 +1040,78 @@ const RunWorkflow = (defaultprops) => { return response.json(); }) .then((responseJson) => { - if (responseJson.success == false) { + if (responseJson?.success == false) { return } - if (execution_id !== undefined && execution_id !== null && authorization !== undefined && authorization !== null && execution_id.length > 0 && authorization.length > 0 && (workflow.id === undefined || workflow.id === null || workflow.id.length === 0) && responseJson.workflow !== undefined && responseJson.workflow !== null) { + if (execution_id !== undefined && execution_id !== null && authorization !== undefined && authorization !== null && execution_id.length > 0 && authorization.length > 0 && disableButtons === false && responseJson?.status !== "" && responseJson?.status !== "WAITING") { + setDisableButtons(true) + } + //if (execution_id !== undefined && execution_id !== null && authorization !== undefined && authorization !== null && execution_id.length > 0 && authorization.length > 0 && (workflow.id === undefined || workflow.id === null || workflow.id.length === 0) && responseJson.workflow !== undefined && responseJson.workflow !== null) { + if (execution_id !== undefined && execution_id !== null && authorization !== undefined && authorization !== null && execution_id.length > 0 && authorization.length > 0 && responseJson.workflow !== undefined && responseJson.workflow !== null) { setupSourcenode(responseJson.workflow, sourceNode) setWorkflow(responseJson.workflow) + + //const decisionId = searchParams.get("decision_id") // ONLY for agentic workflows + // Check for decision_id in url + if (decisionId?.length > 0 && responseJson?.workflow?.actions?.length > 0 && sourceNode?.length > 0 && responseJson?.results?.length > 0) { + console.log("Setting workflow: ", responseJson.workflow, ", EXEC RESULTS: ", responseJson.results) + + setAgentic(true) + + for (var resultkey in responseJson.results) { + const result = responseJson.results[resultkey] + if (result.action.id !== sourceNode) { + continue + } + + const validated = validateJson(result.result) + if (!validated.valid) { + console.log("Error parsing result: ", validated.error) + continue + } + + var parsedresult = validated.result + console.log("PARSED RES: ", parsedresult) + if (parsedresult?.decisions?.length > 0) { + var newexec = executionArgument + if (newexec === undefined || newexec === null || Object.keys(newexec).length === 0) { + newexec = {} + } + + for (var decisionkey in parsedresult?.decisions) { + const decision = parsedresult.decisions[decisionkey] + if (decision?.run_details?.id !== decisionId) { + continue + } + + for (var fieldkey in decision?.fields) { + const field = decision.fields[fieldkey] + if (field.key === "question" && !inputQuestions.find(q => q.name=== field.value)) { + console.log("QUESTION: ", field) + const newquestion = { + "name": field.value, + "value": field.key+"_"+fieldkey, + } + + inputQuestions.push(newquestion) + + newexec[newquestion.value] = "" + } + } + } + + setInputQuestions([...inputQuestions] ) + console.log("EXEC: ", newexec) + setExecutionArgument(newexec) + + responseJson.workflow.input_questions = inputQuestions + setWorkflow(responseJson?.workflow) + setDisableButtons(false) + } + } + } } @@ -1031,12 +1127,12 @@ const RunWorkflow = (defaultprops) => { localStorage.setItem(storageKey, JSON.stringify(value)) } - if (realtimeMarkdown !== undefined && realtimeMarkdown !== null && realtimeMarkdown.length > 0) { + if (realtimeMarkdown !== undefined && realtimeMarkdown !== null && realtimeMarkdown?.length > 0) { const newmarkdown = realtimeMarkdown.replace(`{{ ${responseJson.workflow.id} }}`, responseJson.result, -1) setRealtimeMarkdown(newmarkdown) - } else if (workflow.form_control.input_markdown !== undefined && workflow.form_control.input_markdown !== null && workflow.form_control.input_markdown.length > 0) { - const newmarkdown = workflow.form_control.input_markdown.replace(`{{ ${responseJson.workflow.id} }}`, responseJson.result, -1) + } else if (workflow?.form_control?.input_markdown !== undefined && workflow?.form_control?.input_markdown !== null && workflow?.form_control?.input_markdown.length > 0) { + const newmarkdown = workflow?.form_control?.input_markdown.replace(`{{ ${responseJson.workflow.id} }}`, responseJson.result, -1) setRealtimeMarkdown(newmarkdown) } @@ -1072,7 +1168,6 @@ const RunWorkflow = (defaultprops) => { getWorkflow(props.match.params.key, sourceNode) if (execution_id !== undefined && execution_id !== null && authorization !== undefined && authorization !== null) { - console.log("Get execution: ", execution_id) fetchUpdates(execution_id, authorization, true) } @@ -1136,13 +1231,13 @@ const RunWorkflow = (defaultprops) => { const buttonStyle = {borderRadius: 25, height: 50, fontSize: 18, backgroundImage: handleValidateForm(executionArgument) || executionLoading ? buttonBackground : "grey", color: "white"} // Check if all fields are filled in? - var disabledButtons = executionLoading || executionRunning || message.length > 0 + var disabledButtons = executionLoading || executionRunning || message.length > 0 || disableButtons if (disabledButtons === false && workflow.input_questions !== undefined && workflow.input_questions !== null && workflow.input_questions.length > 0) { // Check field values //disabledButtons = handleValidateForm(executionArgument) } - const organization = selectedOrganization !== undefined && selectedOrganization !== null ? selectedOrganization.name : "Unknown" + const organization = selectedOrganization !== undefined && selectedOrganization !== null ? selectedOrganization.name : "" const contact = selectedOrganization !== undefined && selectedOrganization !== null && selectedOrganization.org !== undefined && selectedOrganization.org !== null? selectedOrganization.org : "support@shuffler.io" //const contact = selectedOrganization !== undefined && selectedOrganization !== null && selectedOrganization.contact !== undefined && selectedOrganization.contact !== null? selectedOrganization.contact : "support@shuffler.io" @@ -1321,12 +1416,12 @@ const RunWorkflow = (defaultprops) => {
- Loading Form Details... + Loading Details...
:
- {workflowQuestion !== "" || (workflow.form_control.input_markdown !== undefined && workflow.form_control.input_markdown !== null && workflow.form_control.input_markdown.length > 0) ? + {workflowQuestion !== "" || (workflow?.form_control?.input_markdown !== undefined && workflow?.form_control?.input_markdown !== null && workflow?.form_control?.input_markdown.length > 0) ?
{ }} rehypePlugins={[rehypeRaw]} > - {workflowQuestion !== "" ? workflowQuestion : realtimeMarkdown !== undefined && realtimeMarkdown !== null && realtimeMarkdown.length > 0 ? realtimeMarkdown : workflow.form_control.input_markdown} + {workflowQuestion !== "" ? workflowQuestion : realtimeMarkdown !== undefined && realtimeMarkdown !== null && realtimeMarkdown.length > 0 ? realtimeMarkdown : workflow?.form_control?.input_markdown}
: null} {onSubmit(e)}} style={{margin: "25px 0px 15px 0px",}}> - {workflowQuestion !== "" || (workflow.form_control.input_markdown !== undefined && workflow.form_control.input_markdown !== null && workflow.form_control.input_markdown.length > 0) ? null : + {workflowQuestion !== "" || (workflow?.form_control?.input_markdown !== undefined && workflow?.form_control?.input_markdown !== null && workflow?.form_control?.input_markdown.length > 0) ? null :
{/* { {organization} - + {organization?.length > 0 && + + } {disabledButtons && message.length > 0 ? null : - + {message} } @@ -1412,6 +1509,11 @@ const RunWorkflow = (defaultprops) => { executionArgument[multiChoiceOptions[0]] = multiChoiceOptions[1] } + const parsedLabel = question?.value?.startsWith("question_") ? + "" + : + question?.value?.charAt(0)?.toUpperCase() + question?.value?.slice(1) + return (
@@ -1457,7 +1559,7 @@ const RunWorkflow = (defaultprops) => { backgroundColor: theme.palette.inputColor, marginTop: 5, }} - label={question?.value?.charAt(0)?.toUpperCase() + question?.value?.slice(1)} + label={parsedLabel} required disabled={disabledButtons} @@ -1542,7 +1644,7 @@ const RunWorkflow = (defaultprops) => { : - {disabledButtons ? "Already answered. You may close this window." : ""} + {disabledButtons ? "Question answered. You may close this window." : ""} } @@ -1565,10 +1667,13 @@ const RunWorkflow = (defaultprops) => { textTransform: "none", }} onClick={() => { - setButtonClicked("FINISHED") - setExecutionData({ - status: "FINISHED", - }) + // Timeout 2500 just in case + setTimeout(() => { + setButtonClicked("FINISHED") + setExecutionData({ + status: "FINISHED", + }) + }, 2500) onSubmit(null, execution_id, authorization, true) }}> @@ -1586,16 +1691,24 @@ const RunWorkflow = (defaultprops) => { flex: 1, textTransform: "none", }} onClick={() => { - setButtonClicked("ABORTED") - setExecutionData({ - status: "ABORTED", - }) + setTimeout(() => { + setButtonClicked("ABORTED") + setExecutionData({ + status: "ABORTED", + }) + }, 2500) onSubmit(null, execution_id, authorization, false) }}> Stop
+ + {handleValidateForm(executionArgument) === false && disabledButtons === false ? + + All required questions have not been answered yet. + + : null} :
diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 32859549..702092da 100755 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -477,7 +477,7 @@ export const HandleJsonCopy = (base, copy, base_node_name) => { //var newitem = JSON.parse(base); var newitem = validateJson(base).result - var to_be_copied = "$" + base_node_name.toLowerCase().replaceAll(" ", "_"); + var to_be_copied = "$" + base_node_name?.toLowerCase()?.replaceAll(" ", "_"); for (let copykey in copy.namespace) { if (copy.namespace[copykey].includes("Results for")) { continue; @@ -742,7 +742,7 @@ const DropzoneWrapper = memo(({ onDrop, WorkflowView }) => { const Workflows = (props) => { const { globalUrl, isLoggedIn, isLoaded, userdata, checkLogin } = props; - document.title = "Shuffle - Workflows"; + document.title = "Workflows - Shuffle"; let navigate = useNavigate(); const classes = useStyles(theme) diff --git a/frontend/src/views/Workflows2.jsx b/frontend/src/views/Workflows2.jsx index 86c28e18..02df7908 100644 --- a/frontend/src/views/Workflows2.jsx +++ b/frontend/src/views/Workflows2.jsx @@ -4,16 +4,6 @@ import { useLocation, useNavigate, Link } from "react-router-dom"; import ReactDOM from "react-dom" import { getTheme } from "../theme.jsx"; -// Material UI Icons -import Add from '@mui/icons-material/Add'; -import Search from '@mui/icons-material/Search'; -import ClearIcon from '@mui/icons-material/Clear'; -import QueryStatsIcon from '@mui/icons-material/QueryStats'; -import GridOnIcon from '@mui/icons-material/GridOn'; -import ListIcon from '@mui/icons-material/List'; -import PublishIcon from '@mui/icons-material/Publish'; -import GetAppIcon from '@mui/icons-material/GetApp'; - // Material UI & Components import { makeStyles } from "@mui/styles"; import { Navigate } from "react-router-dom"; @@ -67,6 +57,7 @@ import { // Material UI Icons import { + ContentCopy as ContentCopyIcon, Close as CloseIcon, Compare as CompareIcon, Maximize as MaximizeIcon, @@ -105,6 +96,12 @@ import { AutoAwesome as AutoAwesomeIcon, BarChart as BarChartIcon, Lock as LockIcon, + Clear as ClearIcon, + QueryStats as QueryStatsIcon, + GridOn as GridOnIcon, + List as ListIcon, + Publish as PublishIcon, + GetApp as GetAppIcon, } from "@mui/icons-material"; // Additional Components @@ -209,10 +206,10 @@ export const GetIconInfo = (action) => { key: "compare", values: ["compare", "convert", "to", "filter", "translate", "parse"], }, - { key: "assets", values: ["cmdb", "assets", "asset", "cmdb", "inventory", "host", "hosts", "device", "devices"] }, + { key: "assets", values: ["cmdb", "assets", "asset", "cmdb", "inventory", "host", "hosts", "device", "devices", "app",] }, { key: "close", values: ["close", "stop", "cancel", "block"] }, { key: "communication", values: ["communication", "comms", "email", "mail",] }, - { key: "eradication", values: ["eradication", "edr", "xdr"] }, + { key: "eradication", values: ["eradication", "edr", "xdr", "sigma", "yara",] }, { key: "iam", values: ["iam", "identity", "access", "auth", "authentication", "authorization", "oauth", "sso", "openid"] }, { key: "intel", values: ["intel", "feed", "threat intel", "threat intelligence", "ti", "t.i.", "t.i", "ti.", "rule", "technique", "tactic", "techniques", "tactics", "ioc", "indicator",] }, { key: "network", values: ["network", "net", "networking", "firewall", "proxy", "vpn", "sdwan", "sd-wan"] }, @@ -235,6 +232,7 @@ export const GetIconInfo = (action) => { values: [ "api", "password", + "passwd", "protect", ], } @@ -835,7 +833,9 @@ const Workflows2 = (props) => { setCurrTab(1); } else if (tabParam === 'all_workflows' && currTab !== 2) { setCurrTab(2); - } + } else if (tabParam === 'background_processes' && currTab !== 4) { + setCurrTab(4); + } } }, [location.search]); @@ -853,10 +853,15 @@ const Workflows2 = (props) => { 1: 'my_workflows', 2: 'all_workflows', 3: 'backup_apps', + 4: 'background_processes', }; const queryParams = new URLSearchParams(location.search); queryParams.set('tab', tabMapping[newValue]); + if (newValue === 4) { + setShowExecutionStats(true) + setView("grid") + } navigate(`${location.pathname}?${queryParams.toString()}`); }; @@ -1553,7 +1558,7 @@ const Workflows2 = (props) => { sx: { borderRadius: theme?.palette?.DialogStyle?.borderRadius, border: theme?.palette?.DialogStyle?.border, - minWidth: '440px', + minWidth: 440, fontFamily: theme?.typography?.fontFamily, backgroundColor: theme?.palette?.DialogStyle?.backgroundColor, zIndex: 1000, @@ -1566,11 +1571,11 @@ const Workflows2 = (props) => { } }} > - +
Are you sure you want to delete {selectedWorkflowId.length > 0 ? filteredWorkflows.find((w) => w.id === selectedWorkflowId)?.name : `${selectedWorkflowIndexes.length} workflow${selectedWorkflowIndexes.length === 1 ? '' : 's'}`}?
- Other workflows relying on {selectedWorkflowIndexes.length > 0 ? "them" : "it"} one will stop working + Other workflows relying on {selectedWorkflowIndexes.length > 0 ? "them" : "it"} one will stop working.
{ credentials: "include", }) .then((response) => { + setIsLoadingWorkflow(false) if (response.status !== 200) { console.log("Status not 200 for workflows :O!: ", response.status); @@ -1956,6 +1962,7 @@ const Workflows2 = (props) => { } }) .catch((error) => { + setIsLoadingWorkflow(false) toast(error.toString()); }); } @@ -2949,6 +2956,8 @@ const Workflows2 = (props) => { triggerfound = true image = wfTriggers[0].large_image + trigger.status = trigger?.status?.toLowerCase() + relevantTrigger = trigger if (trigger?.status === "running") { imageStyle.border = `3px solid ${green}` @@ -2962,6 +2971,8 @@ const Workflows2 = (props) => { triggerfound = true image = wfTriggers[1].large_image + trigger.status = trigger?.status?.toLowerCase() + relevantTrigger = trigger if (trigger?.status === "running") { imageStyle.border = `3px solid ${green}` @@ -3034,10 +3045,11 @@ const Workflows2 = (props) => { const foundTimeline = workflowTimelines.find((timeline) => timeline.id === data.id) return ( -
+
- {selectedCategory !== "" ?
{ }} /> - : null} + : null} { {currTab === 2 ? null : -
{ @@ -3183,6 +3194,7 @@ const Workflows2 = (props) => { + {appGroup.length > 0 ?
@@ -3437,7 +3449,7 @@ const Workflows2 = (props) => { {showExecutionStats === true && foundTimeline !== undefined && foundTimeline?.timeline?.length > 0 && -
+
{
-
+ {currTab === 4 ? null : +
- {currTab === 2 ? ( - - ) : ( - - // - // - // ), - onKeyDown: (e) => { - // Prevent default behavior for Enter and Backspace - if (e.key === 'Enter' || e.key === 'Backspace') { - e.preventDefault(); - e.stopPropagation(); - e.target.focus(); - } - }, - }} - clearInputOnBlur={false} - sx={{ - // Container styling - '& .MuiOutlinedInput-root': { - height: "fit-content", - borderRadius: '4px', - color: theme.palette.textFieldStyle.color, - backgroundColor: theme.palette.textFieldStyle.backgroundColor, - '& fieldset': { - borderColor: 'rgba(255, 255, 255, 0.23)', - }, - '&:hover fieldset': { - borderColor: 'rgba(255, 255, 255, 0.4)', - }, - }, + {currTab === 2 ? ( + + ) : + ( + + // + // + // ), + onKeyDown: (e) => { + // Prevent default behavior for Enter and Backspace + if (e.key === 'Enter' || e.key === 'Backspace') { + e.preventDefault(); + e.stopPropagation(); + e.target.focus(); + } + }, + }} + clearInputOnBlur={false} + sx={{ + // Container styling + '& .MuiOutlinedInput-root': { + height: "fit-content", + borderRadius: '4px', + color: theme.palette.textFieldStyle.color, + backgroundColor: theme.palette.textFieldStyle.backgroundColor, + '& fieldset': { + borderColor: 'rgba(255, 255, 255, 0.23)', + }, + '&:hover fieldset': { + borderColor: 'rgba(255, 255, 255, 0.4)', + }, + }, - // Adjust chip container to center vertically - '& .MuiInputBase-root': { - display: 'flex', - flexWrap: 'wrap', - gap: '4px', - fontSize: 18, - padding: '4px 8px', - alignItems: 'center', - height: "fit-content", // Match height - backgroundColor: theme.palette.textFieldStyle.backgroundColor, - color: theme.palette.textFieldStyle.color - }, + // Adjust chip container to center vertically + '& .MuiInputBase-root': { + display: 'flex', + flexWrap: 'wrap', + gap: '4px', + fontSize: 18, + padding: '4px 8px', + alignItems: 'center', + height: "fit-content", // Match height + backgroundColor: theme.palette.textFieldStyle.backgroundColor, + color: theme.palette.textFieldStyle.color + }, - // Rest of the styling remains the same... - }} - value={filters} - onChange={(chips) => { - setFilters(chips); - const remainingCategories = chips.map(chip => { - const match = chip.match(/\d+\.\s+(\w+)/i); - return match ? match[1] : chip; - }).filter(category => { - return usecases.some(usecase => - usecase.name.toLowerCase().includes(category.toLowerCase()) - ); - }); + // Rest of the styling remains the same... + }} + value={filters} + onChange={(chips) => { + setFilters(chips); + const remainingCategories = chips.map(chip => { + const match = chip.match(/\d+\.\s+(\w+)/i); + return match ? match[1] : chip; + }).filter(category => { + return usecases.some(usecase => + usecase.name.toLowerCase().includes(category.toLowerCase()) + ); + }); - setSelectedCategory(remainingCategories); - findWorkflow(chips); + setSelectedCategory(remainingCategories); + findWorkflow(chips); - }} - //onAdd={(chip) => { - // console.log("ADd: ", chip); - // addFilter(chip); - //}} - //onDelete={(_, index) => { - // console.log("Remove: ", index); - // removeFilter(index); - //}} - /> - )} + }} + //onAdd={(chip) => { + // console.log("ADd: ", chip); + // addFilter(chip); + //}} + //onDelete={(_, index) => { + // console.log("Remove: ", index); + // removeFilter(index); + //}} + /> + )} - { - currTab !== 2 && ( - selected.length ? selected.join(', ') : 'All Categories'} + > + + All Categories + + {usecases.map((usecase, index) => { + if (usecase?.name === "5. Verify") { + return null; + } - const percentDone = usecase.matches.length > 0 ? parseInt(usecase.matches.length / usecase.list.length * 100) : 0 - if (percentDone === 0) { - usecase = findMatches(usecase, workflows) - } + const percentDone = usecase.matches.length > 0 ? parseInt(usecase.matches.length / usecase.list.length * 100) : 0 + if (percentDone === 0) { + usecase = findMatches(usecase, workflows) + } - const category = usecase?.name.split(" ")[1] - return ( - { - if (!filters.includes(usecase?.name.toLowerCase())) { - addFilter(usecase.name) - } else { - removeFilter(filters.indexOf(usecase?.name.toLowerCase())) - } - }} - sx={{ - padding: "12px 16px", - borderBottom: index === usecases.length - 2 ? "none" : "1px solid rgba(255,255,255,0.05)", - "&:hover": { - backgroundColor: "rgba(255,255,255,0.1)" - }, - }} - > -
- -
- - {category} - - - {usecase?.matches.length}/{usecase?.list.length} - -
-
-
- ) - })} - - ) - } - { - currTab === 2 && ( - - ) - } + const category = usecase?.name.split(" ")[1] + return ( + { + if (!filters.includes(usecase?.name.toLowerCase())) { + addFilter(usecase.name) + } else { + removeFilter(filters.indexOf(usecase?.name.toLowerCase())) + } + }} + sx={{ + padding: "12px 16px", + borderBottom: index === usecases.length - 2 ? "none" : "1px solid rgba(255,255,255,0.05)", + "&:hover": { + backgroundColor: "rgba(255,255,255,0.1)" + }, + }} + > +
+ +
+ + {category} + + + {usecase?.matches.length}/{usecase?.list.length} + +
+
+
+ ) + })} + + ) + } + { + currTab === 2 && ( + + ) + } -
-
+
+
- - { + + { - const newView = !showExecutionStats - localStorage.setItem("showExecutionStats", newView) - setShowExecutionStats(!showExecutionStats) - }} - disabled={currTab === 2} - > - - - + const newView = !showExecutionStats + localStorage.setItem("showExecutionStats", newView) + setShowExecutionStats(!showExecutionStats) + }} + disabled={currTab === 2} + > + + + - - navigate("/workflows/debug")} - disabled={currTab === 2} - > - - - + + navigate("/workflows/debug")} + disabled={currTab === 2} + > + + + - - { - const newView = view === "grid" ? "list" : "grid"; - localStorage.setItem("workflowView", newView); - setView(newView); + + { + const newView = view === "grid" ? "list" : "grid"; + localStorage.setItem("workflowView", newView); + setView(newView); - if (view === "grid") { - setCurrTab(0) - } - }} - disabled={currTab === 2} - > - {view === "grid" ? - : - - } - - + if (view === "grid") { + setCurrTab(0) + } + }} + disabled={currTab === 2} + > + {view === "grid" ? + : + + } + + - - upload.click()} - disabled={currTab === 2} - > - {submitLoading ? - : - - } - - + + upload.click()} + disabled={currTab === 2} + > + {submitLoading ? + : + + } + + - (upload = ref)} - onChange={importFiles} - /> + (upload = ref)} + onChange={importFiles} + /> - - exportAllWorkflows(workflows)} - > - - - -
- -
+ + exportAllWorkflows(workflows)} + > + + + +
+ +
+
+ } + - -
{ ) : ( view === "grid" && currTab !== 2 ? ( <> -
{ + if (data.triggers.length === 0) { + return null + } + + var foundWebhook = "" + var foundtrigger = {} + for (var triggerKey in data.triggers) { + if (data.triggers[triggerKey].trigger_type === "WEBHOOK") { + foundWebhook = `${globalUrl}/api/v1/hooks/webhook_${data.triggers[triggerKey].id}` + foundtrigger = data.triggers[triggerKey] + break + } + } + + if (foundWebhook === "") { + return null + } + + var webhookName = `` + if (data?.name?.toLowerCase().includes("ingest tickets")) { + webhookName = "Send your Tickets, Alerts, Cases and Detections here. This will ingest them into Shuffle." + } + + return ( +
{ + // Find the relevant workflow paper and highlight it + const foundElement = document.getElementById(`workflowbox-${data.id}`) + if (foundElement) { + foundElement.style.border = `3px solid ${theme.palette.primary.main}` + } + }} + onMouseLeave={() => { + const foundElement = document.getElementById(`workflowbox-${data.id}`) + if (foundElement) { + foundElement.style.border = null + } + }} + > + + {webhookName} + + + + webhook + + + { + if (navigator.clipboard === undefined) { + toast("Your browser doesn't support clipboard copying, please copy manually.", { type: "error" }); + } else { + navigator.clipboard.writeText(foundWebhook); + } + }} + style={{ + color: theme.palette.textFieldStyle.color, + backgroundColor: theme.palette.platformColor, + marginRight: 10, + borderRadius: 4, + }} + id="copy_webhook_url_button" + > + + + + + ), + style: { + color: theme.palette.textFieldStyle.color, + backgroundColor: theme.palette.textFieldStyle.backgroundColor, + } + }} + /> +
+ ) + })} + +
Date: Mon, 20 Oct 2025 11:51:49 +0200 Subject: [PATCH 51/57] Go mod fixes --- backend/go-app/go.mod | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index f72866b6..b563af28 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -24,8 +24,8 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.30 - github.com/shuffle/singul v0.0.16 + github.com/shuffle/shuffle-shared v0.9.31 + github.com/shuffle/singul v0.0.17 golang.org/x/crypto v0.40.0 google.golang.org/api v0.236.0 google.golang.org/grpc v1.72.2 @@ -73,7 +73,7 @@ require ( github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/frikky/schemaless v0.0.20 // indirect + github.com/frikky/schemaless v0.0.22 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-jose/go-jose/v4 v4.0.5 // indirect From 11830fd38bcea659da84138ba8dbb82a54e78c94 Mon Sep 17 00:00:00 2001 From: Frikky Date: Mon, 20 Oct 2025 12:11:04 +0200 Subject: [PATCH 52/57] Fixed debouncecallback ref --- backend/go-app/go.sum | 2 + .../src/components/AuthenticationModal.jsx | 891 ++++++++++++++++++ frontend/src/components/DiscordChat.jsx | 2 +- frontend/src/components/LineChartWrapper.jsx | 168 +++- frontend/src/utils/useDebouncedCallback.jsx | 25 + frontend/src/views/AngularWorkflow.jsx | 2 +- 6 files changed, 1071 insertions(+), 19 deletions(-) create mode 100644 frontend/src/components/AuthenticationModal.jsx create mode 100644 frontend/src/utils/useDebouncedCallback.jsx diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index b520ef42..8d99773a 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -365,6 +365,8 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shuffle/shuffle-shared v0.9.30 h1:3CYvNyD7sTxdxoZjTVrtaDqFvSWQRKAFGaga6rPGf8A= github.com/shuffle/shuffle-shared v0.9.30/go.mod h1:PhDEizuz4SmJaSmy0+yrFWwD1mXVUsy8/knKlrqF1qw= +github.com/shuffle/shuffle-shared v0.9.31 h1:BMoDO4Sgz4+I12aHhc3cWHiCuAQ827XIxajPqCJ9cXA= +github.com/shuffle/shuffle-shared v0.9.31/go.mod h1:vfI2QDGphZGrcwuUPQ1yI/Hgc8aseFro5+2k36irfkQ= github.com/shuffle/singul v0.0.16 h1:dW+0Mln9R1aUJ0fjikpWcxbjoQWqJHxe4kSxh2tQN5E= github.com/shuffle/singul v0.0.16/go.mod h1:LYkp320A6gsoPlYbXUM+WvEPUVAuutlSsqnVKyRy4gs= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= diff --git a/frontend/src/components/AuthenticationModal.jsx b/frontend/src/components/AuthenticationModal.jsx new file mode 100644 index 00000000..62f15e7f --- /dev/null +++ b/frontend/src/components/AuthenticationModal.jsx @@ -0,0 +1,891 @@ +import React, { useState, useEffect, useContext, memo } from "react"; +import { CodeHandler, Img, OuterLink, } from '../views/Docs.jsx' +import { getTheme } from "../theme.jsx"; +import { isMobile } from "react-device-detect" +import Markdown from "react-markdown"; +import { Context } from '../context/ContextApi.jsx'; +import PaperComponent from "../components/PaperComponent.jsx"; +import { toast } from "react-toastify"; +import { v4 as uuidv4} from "uuid"; +import AuthenticationOauth2 from "../components/Oauth2Auth.jsx"; + +import { + Edit as EditIcon, + DragIndicator as DragIndicatorIcon, + Close as CloseIcon, + LockOpen as LockOpenIcon, +} from "@mui/icons-material"; + +import { + Button, + Typography, + Dialog, + DialogContent, + DialogTitle, + DialogActions, + MenuItem, + Select, + TextField, + IconButton, + Tooltip, + Divider, +} from "@mui/material"; + +const AuthenticationModal = (props) => { + const { + globalUrl, + userdata, + + selectedAppData, + getAppAuthentication, + appAuthentication, + setSelectedAction, + + selectedMeta, + setSelectedMeta, + + setAppAuthentication, + } = props; + + const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" || window.location.host === "migration.shuffler.io"; + const [selectedAuthentication, setSelectedAuthentication] = React.useState({}); + const [authenticationModalOpen, setAuthenticationModalOpen] = React.useState(false); + const [authenticationType, setAuthenticationType] = React.useState({}) + + const [appid, setAppId] = useState("") + + //const [appAuthentication, setAppAuthentication] = useState([]); + + const { themeMode, supportEmail, brandColor } = useContext(Context) + const theme = getTheme(themeMode, brandColor) + + useEffect(() => { + if (selectedAppData === undefined || selectedAppData === null || Object.getOwnPropertyNames(selectedAppData).length === 0) { + return + } + + if (selectedAppData.authentication === undefined || selectedAppData.authentication === null) { + setAuthenticationType({ + type: "", + }) + + selectedAppData.authentication = { + type: "", + required: false, + } + } else { + setAuthenticationType( + selectedAppData.authentication.type === "oauth2-app" || (selectedAppData.authentication.type === "oauth2" && selectedAppData.authentication.redirect_uri !== undefined && selectedAppData.authentication.redirect_uri !== null) ? { + type: selectedAppData.authentication.type, + redirect_uri: selectedAppData.authentication.redirect_uri, + refresh_uri: selectedAppData.authentication.refresh_uri, + token_uri: selectedAppData.authentication.token_uri, + scope: selectedAppData.authentication.scope, + client_id: selectedAppData.authentication.client_id, + client_secret: selectedAppData.authentication.client_secret, + grant_type: selectedAppData.authentication.grant_type, + } : { + type: "", + } + ) + } + }, [selectedAppData]) + + if (selectedAppData === undefined || selectedAppData === null || Object.getOwnPropertyNames(selectedAppData).length === 0) { + console.log("No app data for authentication modal"); + return null + } + + if (authenticationModalOpen === false) { + return ( + + ) + } + + function Heading(props) { + const element = React.createElement( + `h${props.level}`, + { style: { marginTop: 40 } }, + props.children + ); + return ( + + {props.level !== 1 ? ( + + ) : null} + {element} + + ); + } + + const UpdateAppAuthentication = (data) => { + if (data === undefined || data === null) { + return; + } + + const filteredData = data.filter((appAuth) => appAuth?.app?.id === appid); + if (filteredData.length === 0) { + setAppAuthentication([]); + setSelectedAuthentication({}); + } else { + setAppAuthentication(filteredData); + setSelectedAuthentication(filteredData[0]); + } + }; + + const HandleAppAuthentication = () => { + + const url = `${globalUrl}/api/v1/apps/authentication`; + + fetch(url, { + method: "GET", + headers: { + "Content-Type": "application/json", + }, + credentials: "include", + }).then((response) => { + if (response.status !== 200) { + return; + } + return response.json(); + }).then((responseJson) => { + if (responseJson.success === true) { + UpdateAppAuthentication(responseJson.data); + } else { + toast.error("Failed to get app authentication data"); + } + }).catch((error) => { + console.error("error for app is :", error); + }); + } + + const AuthenticationData = (props) => { + const selectedApp = props.app; + + const [authenticationOption, setAuthenticationOptions] = React.useState({ + app: JSON.parse(JSON.stringify(selectedApp)), + fields: {}, + label: "", + usage: [ + { + // workflow_id: workflow.id, + }, + ], + id: uuidv4(), + active: true, + }); + + if ( + selectedApp.authentication === undefined || + selectedApp.authentication.parameters === null || + selectedApp.authentication.parameters === undefined || + selectedApp.authentication.parameters.length === 0 + ) { + return ( + + + {selectedApp.name} does not require authentication + + + ); + } + + authenticationOption.app.actions = []; + + for (let paramkey in selectedApp.authentication.parameters) { + if ( + authenticationOption.fields[ + selectedApp.authentication.parameters[paramkey].name + ] === undefined + ) { + authenticationOption.fields[ + selectedApp.authentication.parameters[paramkey].name + ] = ""; + } + } + + const setNewAppAuth = (appAuthData, refresh) => { + setSelectedAuthentication(appAuthData); + var headers = { + "Content-Type": "application/json", + "Accept": "application/json", + } + + headers["Org-Id"] = userdata?.active_org?.id + + fetch(globalUrl + "/api/v1/apps/authentication", { + method: "PUT", + headers: headers, + body: JSON.stringify(appAuthData), + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for setting app auth :O!"); + + if (response.status === 400) { + toast.error("Failed setting new auth. Please try again", { + "autoClose": true, + }) + } + } + + return response.json(); + }) + .then((responseJson) => { + if (!responseJson.success) { + toast.error("Error: " + responseJson.reason, { + "autoClose": false, + }) + + } else { + HandleAppAuthentication() + setAuthenticationModalOpen(false) + getAppAuthentication() + } + }) + .catch((error) => { + console.log("New auth error: ", error.toString()); + }); + }; + + const handleSubmitCheck = () => { + if (authenticationOption.label.length === 0) { + authenticationOption.label = `Auth for ${selectedApp.name}`; + } + for (let paramkey in selectedApp.authentication.parameters) { + if ( + authenticationOption.fields[ + selectedApp.authentication.parameters[paramkey].name + ].length === 0 + ) { + if ( + selectedApp.authentication.parameters[paramkey].value !== undefined && + selectedApp.authentication.parameters[paramkey].value !== null && + selectedApp.authentication.parameters[paramkey].value.length > 0 + ) { + authenticationOption.fields[ + selectedApp.authentication.parameters[paramkey].name + ] = selectedApp.authentication.parameters[paramkey].value; + } else { + if ( + selectedApp.authentication.parameters[paramkey].schema.type === "bool" + ) { + authenticationOption.fields[ + selectedApp.authentication.parameters[paramkey].name + ] = "false"; + } else { + toast( + "Field " + + selectedApp.authentication.parameters[paramkey].name + + " can't be empty" + ); + return; + } + } + } + } + + var newAuthOption = JSON.parse(JSON.stringify(authenticationOption)); + var newFields = []; + for (let authkey in newAuthOption.fields) { + const value = newAuthOption.fields[authkey]; + newFields.push({ + "key": authkey, + "value": value, + }); + } + + newAuthOption.fields = newFields + setNewAppAuth(newAuthOption) + } + + if (authenticationOption.label === null || authenticationOption.label === undefined) { + authenticationOption.label = selectedApp.name + " authentication"; + } + + return ( +
+ +
+ Authentication for {selectedApp.name.replaceAll("_", " ", -1)} +
+
+ + + What is app authentication? + +
+ These are required fields for authenticating with {selectedApp.name} +
+ Label for you to remember + { + authenticationOption.label = event.target.value; + }} + /> + +
+ {selectedApp.authentication.parameters.map((data, index) => { + if (data.value === "" || data.value === null || data.value === undefined || data.name === "url") { + } + + + return ( +
+ + {data.name} + + {data.schema !== undefined && + data.schema !== null && + data.schema.type === "bool" ? ( + + ) : ( + { + authenticationOption.fields[data.name] = + event.target.value; + }} + /> + )} +
+ ); + })} + + + + + +
+ ); + }; + + const authenticationModal = authenticationModalOpen ? ( + {setSelectedMeta(undefined)}} + PaperProps={{ + style: { + pointerEvents: "auto", + color: theme.palette.textColor, + minWidth: 1100, + minHeight: 800, + maxHeight: 800, + padding: 15, + overflow: "hidden", + zIndex: 10012, + border: theme.palette.defaultBorder, + }, + }} + > +
+ + + + + + { + setAuthenticationModalOpen(false); + }} + > + + +
+
+ {authenticationType?.type === "oauth2" || authenticationType?.type === "oauth2-app" ? + + : + + } +
+
+ {selectedAppData?.documentation === undefined || + selectedAppData?.documentation === null || + selectedAppData?.documentation.length === 0 ? ( + +
+ + {selectedAppData?.description} + +
+ + +
+ + There is no Shuffle-specific documentation for this app yet outside of the general description above. Documentation is written for each api, and is a community effort. We hope to see your contribution! + + +
+ + + Want to help the making of, or improve this app?{" "} +
+ + Join the community on Discord! + +
+ + + Want to help change this app directly? + + {selectedAppData.reference_info === undefined || + selectedAppData.reference_info === null || + selectedAppData.reference_info.github_url === undefined || + selectedAppData.reference_info.github_url === null || + selectedAppData.reference_info.github_url.length === 0 ? ( + + + + Check it out on Github! + + + + ) : ( + + + + Check it out on Github! + + + + )} +
+ ) : ( +
+ {selectedMeta !== undefined && selectedMeta !== null && Object.getOwnPropertyNames(selectedMeta).length > 0 && selectedMeta.name !== undefined && selectedMeta.name !== null ? +
+
+ {isMobile ? null : ( + + + + + + )} + {isMobile ? null : ( +
+ )} + + {selectedMeta.read_time} minute + {selectedMeta.read_time === 1 ? "" : "s"} to read + +
+
+ {isMobile || + selectedMeta.contributors === undefined || + selectedMeta.contributors === null ? ( + "" + ) : ( +
+ {selectedMeta.contributors.slice(0, 7).map((data, index) => { + return ( + + + {data.url} + + + ); + })} +
+ )} +
+
+ : null} + + + {selectedAppData.documentation} + +
+ )} +
+
+ + ) : null; + + return authenticationModal +} + +export default AuthenticationModal; diff --git a/frontend/src/components/DiscordChat.jsx b/frontend/src/components/DiscordChat.jsx index 5d1a9d04..07394852 100644 --- a/frontend/src/components/DiscordChat.jsx +++ b/frontend/src/components/DiscordChat.jsx @@ -16,7 +16,7 @@ import { ListItemText, } from '@mui/material'; import { Search as SearchIcon } from '@mui/icons-material'; -import useDebouncedCallback from '../utils/useDebouncedCallback.js'; +import useDebouncedCallback from '../utils/useDebouncedCallback.jsx'; const searchClient = algoliasearch("JNSS5CFDZZ", "1e5f29b1550939855de5915eac3bf5f7"); diff --git a/frontend/src/components/LineChartWrapper.jsx b/frontend/src/components/LineChartWrapper.jsx index 95aa8024..307b7a62 100644 --- a/frontend/src/components/LineChartWrapper.jsx +++ b/frontend/src/components/LineChartWrapper.jsx @@ -1,6 +1,7 @@ import React, { useState, useEffect, useContext, memo, useMemo } from 'react' import {getTheme} from '../theme.jsx'; import { Context } from '../context/ContextApi.jsx'; +import { toast } from "react-toastify"; import { Typography, @@ -14,19 +15,104 @@ import { GridlineSeries, Gridline, - TooltipArea, ChartTooltip, TooltipTemplate, + TooltipArea, } from 'reaviz'; +export const LoadStats = (globalUrl, cachekey) => { + if (globalUrl === undefined) { + console.log("Error: Global URL is undefined") + return + } + + if (cachekey === undefined) { + console.log("Error: Cachekey is undefined") + return + } + + var basedata = { + "key": cachekey, + "total": 0, + "available_keys": [], + "labels": [], + "datasets": [ + { + "label": "", + "data": [], + "backgroundColor": [], + "barThickness": 15, + } + ] + } + + //const url = `${globalUrl}/api/v1/stats/app_executions_test2` + //cachekey = cachekey.replace(" ", "_", -1) + const url = `${globalUrl}/api/v1/stats/${cachekey}` + return fetch(url, { + method: "GET", + credentials: "include", + }) + .then((resp) => { + return resp.json() + }).then((respJson) => { + const selectedIndex = 0 + + //console.log("Stats response: ", respJson) + + if (respJson.success === true) { + for (let entryKey in respJson.entries) { + const entry = respJson.entries[entryKey] + basedata.labels.push(entry.date) + + basedata.datasets[0].data.push(entry.value) + basedata.datasets[0].backgroundColor.push(entry.value > 0 ? "rgba(255,255,255,0.4)" : "red") + } + + basedata.available_keys = respJson.available_keys + basedata.total = respJson.total + + return basedata + } else { + console.log("Failed to get stats") + return basedata + } + }) + .catch((err) => { + toast("Failed to get stats") + return basedata + }) +} + const LineChartWrapper = (props) => { - const {keys, inputname, height, width, border} = props + const {keys, inputname, height, width, border, color} = props const [hovered, setHovered] = useState(""); const {themeMode} = useContext(Context) const theme = getTheme(themeMode) - var inputdata = keys.data === undefined ? keys : keys.data + // Correct format: + /* keys={[ + { + key: "2025-07-23T00:00:09.409718Z", + data: 24, + }, + { + key: "2025-07-24T00:00:09.409718Z", + data: 50, + }, + { + key: "2025-07-25T00:00:09.409718Z", + data: 75, + }, + { + key: "2025-07-26T00:00:09.409718Z", + data: 42, + }, + ]} + */ + + var inputdata = keys?.data === undefined ? keys : keys.data var newname = inputname === undefined || inputname === null ? "" : inputname.trim().replaceAll("_", " ") newname = newname.charAt(0).toUpperCase() + newname.slice(1) @@ -35,7 +121,6 @@ const LineChartWrapper = (props) => { var tmpdata = inputdata?.datasets[0] if (tmpdata?.data !== undefined && tmpdata?.data !== null && tmpdata?.data.length > 0 && inputdata?.labels?.length === tmpdata?.data?.length) { - console.log("Fix it!") var newarray = [] for (var key in tmpdata.data) { var entry = { @@ -48,26 +133,25 @@ const LineChartWrapper = (props) => { inputdata = newarray } - } if (inputdata === undefined || inputdata === null) { - return ( + return null /*( Invalid linegraph data format - ) + )*/ } var defaultStyle = { color: "white", - padding: 30, + padding: "5px 5px 10px 5px", marginTop: 15, overflow: "hidden", border: "1px solid rgba(255,255,255,0.3)", borderRadius: theme.palette?.borderRadius, - backgroundColor: theme.palette.platformColor, + //backgroundColor: theme.palette.platformColor, } if (border === false) { @@ -76,11 +160,64 @@ const LineChartWrapper = (props) => { defaultStyle.backgroundColor = "transparent" } + // Check if it's a list or not + if (!Array.isArray(inputdata) || inputdata.length === 0 || (inputdata.length > 0 && (inputdata[0].key === undefined && inputdata[0].data === undefined))) { + console.log("Invalid graph data format: ", inputdata) + console.log("Expected format: [{key: 'label1', data: 10}, {key: 'label2', data: 20}]") + //inputdata = inputdata?.datasets[0]?.data + return null + } + + //console.log("FORMAT: ", inputdata) + + const tooltip = ( +
+ {data?.x ?? ''} + {data?.y ?? ''} +
+ )} + /> + } + />; + + const selectedColor = color === undefined || color === null || color === "" ? "" : color + const barseries = color === undefined || color === null || color === "" ? + + } + tooltip={tooltip} + /> + : + + } + tooltip={tooltip} + /> + return (
- - {newname} - + {newname !== "" && + + {newname} + + } { data={inputdata} series={ - - } - /> + barseries + } gridlines={ } /> diff --git a/frontend/src/utils/useDebouncedCallback.jsx b/frontend/src/utils/useDebouncedCallback.jsx new file mode 100644 index 00000000..aafa23c9 --- /dev/null +++ b/frontend/src/utils/useDebouncedCallback.jsx @@ -0,0 +1,25 @@ +import { useRef, useEffect, useCallback } from "react"; + +export const useDebouncedCallback = (callback, delay = 300) => { + const timeoutRef = useRef(null); + const savedCallback = useRef(callback); + + useEffect(() => { + savedCallback.current = callback; + }, [callback]); + + useEffect(() => () => { + if (timeoutRef.current) clearTimeout(timeoutRef.current); + }, []); + + return useCallback((...args) => { + if (timeoutRef.current) clearTimeout(timeoutRef.current); + timeoutRef.current = setTimeout(() => { + savedCallback.current(...args); + }, delay); + }, [delay]); +}; + +export default useDebouncedCallback; + + diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 88adac1b..61b3a3c3 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -22,7 +22,7 @@ import { CodeHandler, Img, OuterLink, } from "../views/Docs.jsx"; import { InstantSearch, Configure, connectSearchBox, connectHits, Index } from 'react-instantsearch-dom'; import algoliasearch from 'algoliasearch/lite'; -import useDebouncedCallback from "../utils/useDebouncedCallback.js"; +import useDebouncedCallback from "../utils/useDebouncedCallback.jsx"; import { Zoom, Fade, From 18ba7d4440bcbb6077db0ef73ae22c935a99f2d8 Mon Sep 17 00:00:00 2001 From: Frikky Date: Mon, 20 Oct 2025 12:15:32 +0200 Subject: [PATCH 53/57] New dashboard relation added --- frontend/src/App.jsx | 16 + .../src/components/RunsOverTimeWidget.jsx | 326 ++++++ .../components/SuccessFailedRunsWidget.jsx | 1000 +++++++++++++++++ 3 files changed, 1342 insertions(+) create mode 100644 frontend/src/components/RunsOverTimeWidget.jsx create mode 100644 frontend/src/components/SuccessFailedRunsWidget.jsx diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index e3c6e975..18083aea 100755 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -25,6 +25,7 @@ import AgentUI from "./views/AgentUI.jsx"; import Welcome from "./views/Welcome.jsx"; import Dashboard from "./views/Dashboard.jsx"; import DashboardView from "./views/DashboardViews.jsx"; +import NewDashboard from "./views/NewDashboard.jsx"; import AdminSetup from "./views/AdminSetup.jsx"; import Admin from "./views/Admin.jsx"; import Docs from "./views/Docs.jsx"; @@ -872,6 +873,21 @@ const App = (message, props) => { /> } /> + + + } + /> + 12k, 12,000,000 -> 12M) +function formatCompactNumber(value) { + const n = Number(value) || 0; + const abs = Math.abs(n); + if (abs >= 1e9) return `${Math.round((n / 1e9) * 10) / 10}B`; + if (abs >= 1e6) return `${Math.round((n / 1e6) * 10) / 10}M`; + if (abs >= 1e3) return `${Math.round((n / 1e3) * 10) / 10}k`; + return `${n}`; +} + +const RunsOverTimeWidget = (props) => { + const { globalUrl, onLoadingChange, monthOverride, dummyMode } = props; + const [mode, setMode] = useState('workflows'); // 'apps' | 'workflows' + const [series, setSeries] = useState([]); + const [days, setDays] = useState(365); // aggregate to last 12 months by default + const [selectedMonth, setSelectedMonth] = useState(null); // Date representing first day of target month, or null for yearly view + const [loading, setLoading] = useState(false); + + // Helper: fetch time series for a specific statistics key + const fetchSeriesForKey = async (key) => { + try { + const urlA = `${globalUrl}/api/v1/stats/${encodeURIComponent(key)}?days=${encodeURIComponent(days)}`; + const doFetch = async (u) => { + const r = await fetch(u, { method: 'GET', credentials: 'include', headers: { 'Content-Type': 'application/json' } }); + if (!r.ok) return []; + const j = await r.json(); + return Array.isArray(j?.entries) ? j.entries : []; + }; + const a = await doFetch(urlA); + if (a.length > 0) return a; + + // Optional org route fallback if present globally + const orgId = (window && window.selectedOrganization && window.selectedOrganization.id) || null; + if (orgId) { + const urlB = `${globalUrl}/api/v1/orgs/${encodeURIComponent(orgId)}/stats/${encodeURIComponent(key)}?days=${encodeURIComponent(days)}`; + const b = await doFetch(urlB); + if (b.length > 0) return b; + } + + // Final fallback: old aggregate endpoint returning daily_statistics + const fallback = await fetch(`${globalUrl}/api/v1/stats`, { method: 'GET', credentials: 'include', headers: { 'Content-Type': 'application/json' } }); + if (fallback.ok) { + const data = await fallback.json(); + const daily = Array.isArray(data?.daily_statistics) ? data.daily_statistics : []; + const valField = key; + return daily.map((d) => ({ date: d?.date, value: Number(d?.[valField] || 0) })); + } + return []; + } catch (e) { + return []; + } + }; + + // Load and transform into monthly aggregation for last 12 months + const load = async (curMode) => { + setLoading(true); + try { + // Clear current series immediately to avoid any visual overlap while switching views + setSeries([]); + if (dummyMode) { + // Bring back the older dummy series with emphasis on earlier months + const now = new Date(); + const months = []; + for (let i = 11; i >= 0; i--) { + const dt = new Date(now.getFullYear(), now.getMonth() - i, 1); + months.push(new Date(dt.getFullYear(), dt.getMonth(), 1)); + } + const base = [20, 18, 22, 24, 23, 21, 15, 12, 9, 15, 24, 20]; + const dummy = months.map((m, idx) => ({ key: m, id: `${m.getFullYear()}-${m.getMonth()}`, data: base[idx] })); + setSeries(dummy); + return; + } + const key = curMode === 'apps' ? 'app_executions' : 'workflow_executions'; + const entries = await fetchSeriesForKey(key); + // Normalize variants: {Date, Value} or {date, value} + const normalized = (entries || []).map((d) => ({ + date: d?.Date ? new Date(d.Date) : (d?.date ? new Date(d.date) : new Date()), + value: Number(d?.Value ?? d?.value ?? 0), + })); + + // If a month is selected, show DAILY bars for that month + if (selectedMonth instanceof Date) { + const year = selectedMonth.getFullYear(); + const month = selectedMonth.getMonth(); + + // Build all days for selected month + const firstDay = new Date(year, month, 1); + const nextMonthFirst = new Date(year, month + 1, 1); + const numDays = Math.round((nextMonthFirst - firstDay) / (1000 * 60 * 60 * 24)); + + // Sum values per day (normalize time to midnight) + const byDayKey = (d) => `${d.getFullYear()}-${d.getMonth()}-${d.getDate()}`; + const sumPerDay = new Map(); + normalized.forEach((p) => { + if (!p?.date || Number.isNaN(p.value)) return; + const d = p.date; + if (d.getFullYear() !== year || d.getMonth() !== month) return; + const dayKey = byDayKey(new Date(d.getFullYear(), d.getMonth(), d.getDate())); + sumPerDay.set(dayKey, (sumPerDay.get(dayKey) || 0) + p.value); + }); + + const dailySeries = Array.from({ length: numDays }, (_, i) => { + const d = new Date(year, month, i + 1); + const k = byDayKey(d); + const v = sumPerDay.get(k) || 0; + return { key: d, id: k, data: v }; + }); + + // Ensure consistent ordering + setSeries(dailySeries.sort((a, b) => a.key - b.key)); + return; + } + + // Otherwise, show MONTHLY aggregation for last 12 months including current month + const now = new Date(); + const months = []; + for (let i = 11; i >= 0; i--) { + const dt = new Date(now.getFullYear(), now.getMonth() - i, 1); + months.push({ y: dt.getFullYear(), m: dt.getMonth(), key: new Date(dt.getFullYear(), dt.getMonth(), 1) }); + } + + const byMonthKey = (d) => `${d.getFullYear()}-${d.getMonth()}`; + const sumPerMonth = new Map(); + normalized.forEach((p) => { + if (!p?.date || Number.isNaN(p.value)) return; + const k = byMonthKey(new Date(p.date.getFullYear(), p.date.getMonth(), 1)); + sumPerMonth.set(k, (sumPerMonth.get(k) || 0) + p.value); + }); + + const monthlySeries = months.map((mm) => { + const k = `${mm.y}-${mm.m}`; + const v = sumPerMonth.get(k) || 0; + return { key: mm.key, id: k, data: v }; + }); + + setSeries(monthlySeries); + } finally { + setLoading(false); + } + }; + + // Apply month override (e.g. onboarding Explore Now) - consolidated with main load effect + useEffect(() => { + if (monthOverride instanceof Date) { + // Clear series immediately to prevent visual overlap + setSeries([]); + setSelectedMonth(new Date(monthOverride.getFullYear(), monthOverride.getMonth(), 1)); + setDays(370); + } + }, [monthOverride]); + + useEffect(() => { + load(mode); + }, [mode, globalUrl, days, selectedMonth, dummyMode]); + + // Notify parent on loading changes + useEffect(() => { + if (typeof onLoadingChange === 'function') { + onLoadingChange(loading); + } + }, [loading, onLoadingChange]); + + const barData = useMemo(() => ( + (Array.isArray(series) ? series : []).map((d) => { + const dt = new Date(d.key); + const label = selectedMonth instanceof Date + ? String(dt.getDate()) // day of month for daily view + : dt.toLocaleString('default', { month: 'short' }); + return { key: label, data: Number(d?.data || 0) }; + }) + ), [series, mode, selectedMonth]); + + // Build month dropdown options for last 12 months + const monthOptions = useMemo(() => { + const now = new Date(); + const opts = []; + for (let i = 0; i < 12; i++) { + const dt = new Date(now.getFullYear(), now.getMonth() - i, 1); + opts.push(dt); + } + return opts; + }, []); + + const tooltip = ( +
+ {data?.x ?? ''} + {data?.y ?? ''} +
+ )} + /> + } + />; + + return ( +
+
+ Runs over time ({mode === "workflows" ? "Workflows" : "Apps"}) + + v && setMode(v)} + sx={{ + height: 37, + backgroundColor: 'rgba(255,255,255,0.06)', + border: '1px solid rgba(255,255,255,0.22)', + borderRadius: '30px', + padding: '2px', + "& .MuiToggleButton-root": { + border: "none", + borderRadius: "30px", + color: "#fff", + padding: "6px 16px", + textTransform: "none", + fontSize: "14px", + "&.Mui-selected": { + backgroundColor: "#fff", + color: "#222", + fontWeight: "600", + "&:hover": { + backgroundColor: "#fff", + }, + }, + "&:hover": { + backgroundColor: "rgba(255, 255, 255, 0.2)", + }, + }, + }} + > + + + Workflows + + + + + Apps + + + + + View Month + + + +
+ +
+
+ } />} + gridlines={} />} + yAxis={ + formatCompactNumber(d)} /> + } + /> + } + /> + } + animated={false} + /> +
+
+
+ ); +}; + +export default RunsOverTimeWidget; + + diff --git a/frontend/src/components/SuccessFailedRunsWidget.jsx b/frontend/src/components/SuccessFailedRunsWidget.jsx new file mode 100644 index 00000000..e9a3bd09 --- /dev/null +++ b/frontend/src/components/SuccessFailedRunsWidget.jsx @@ -0,0 +1,1000 @@ +import React, { useEffect, useMemo, useState } from "react"; +import { + Typography, + FormControl, + InputLabel, + Select, + MenuItem, + ToggleButton, + ToggleButtonGroup, + Box, +} from "@mui/material"; +import { + AreaChart, + AreaSeries, + Area, + GridlineSeries, + Gridline, + ChartTooltip, + TooltipArea, + LinearXAxis, + LinearXAxisTickSeries, + LinearXAxisTickLabel, + LinearYAxis, + LinearYAxisTickSeries, + LinearYAxisTickLabel, +} from "reaviz"; +import theme from "../theme"; + +// KPI configuration constants +const RUN_MINUTES_SAVED_PER_WORKFLOW = 15; // minutes saved per workflow run +const RUN_DOLLARS_SAVED_PER_WORKFLOW = 25; // dollars saved per workflow run + +// Filters for status selection +const statusOptions = [ + { key: "ALL", label: "All" }, + { key: "FINISHED", label: "Success" }, + { key: "FAILED", label: "Failed" }, +]; + +// Response Object (Coming from backend) +// { +// "key": "app_executions", +// "value": 70, +// "date": "2025-10-15T19:36:03.928872+05:30" +// } + +// Date formatting helpers +// This is used to format the date in the format of YYYY-MM-DD +function formatDay(dateInput) { + try { + return new Date(dateInput).toISOString().slice(0, 10); + } catch { + return String(dateInput); + } +} + +// This is used to format the date in the format of YYYY-MM +function formatMonth(dateInput) { + const dt = new Date(dateInput); + const month = String(dt.getMonth() + 1).padStart(2, "0"); + return `${dt.getFullYear()}-${month}`; +} + +// Aggregate values by day or by month +// This is used for area chart toggle button (Daily / Monthly) +function bucketSeries(items, resolution) { + const map = new Map(); + for (const item of items) { + const key = + resolution === "monthly" ? formatMonth(item.key) : formatDay(item.key); + const value = Number(item.data || 0); + map.set(key, (map.get(key) || 0) + value); + } + return map; +} + +// Normalize API entries to a consistent structure +// for e.g, {date: "2025-10-15T19:36:03.928872+05:30", value: 70} +// will be normalized to {key: "2025-10-15", id: "2025-10-15", data: 70} +function normalizeEntries(arr) { + return (arr || []).map((d) => ({ + key: d?.date ? new Date(d.date) : new Date(), + id: d?.date || Math.random().toString(36).slice(2), + data: Number(d?.value ?? 0), + })); +} + +// Build continuous key sequence from start to end, aligned by resolution (Daily / Monthly) +function buildBackfilledKeys(allKeys, resolution) { + if (allKeys.length === 0) return allKeys; + + const start = new Date(allKeys[0]); + let end = new Date(allKeys[allKeys.length - 1]); + const today = new Date(); + + if (resolution === "monthly") { + const monthToday = new Date(today.getFullYear(), today.getMonth(), 1); + if (monthToday > end) end = monthToday; + } else { + // To ensure that the last day is included in the series + const dayToday = new Date( + today.getFullYear(), + today.getMonth(), + today.getDate() + ); + if (dayToday > end) end = dayToday; + } + + const addKey = (dt) => + resolution === "monthly" ? formatMonth(dt) : formatDay(dt); + const step = (dt) => { + if (resolution === "monthly") { + dt.setMonth(dt.getMonth() + 1); + dt.setDate(1); + } else { + dt.setDate(dt.getDate() + 1); + } + }; + + const sequence = []; + const cursor = new Date(start); + if (resolution === "monthly") cursor.setDate(1); + while (cursor <= end) { + sequence.push(addKey(cursor)); + step(cursor); + } + return sequence; +} + +// Ensure area series has at least two points +function ensureMinTwoPoints(arr) { + if (arr.length === 1) { + return [ + { key: 0, data: arr[0].data }, + { key: 1, data: arr[0].data }, + ]; + } + return arr; +} + +// Compact number formatter for axis ticks (e.g. 12,000 -> 12k, 12000000 -> 12M, 12000000000 -> 12B) +function formatCompactNumber(value) { + const n = Number(value) || 0; + const abs = Math.abs(n); + if (abs >= 1e9) return `${Math.round((n / 1e9) * 10) / 10}B`; + if (abs >= 1e6) return `${Math.round((n / 1e6) * 10) / 10}M`; + if (abs >= 1e3) return `${Math.round((n / 1e3) * 10) / 10}k`; + return `${n}`; +} + +// Compute X axis ticks and label formatter +function computeTicks(allKeys, days, resolution) { + const maxTicks = 12; + const xInterval = Math.max(1, Math.floor(allKeys.length / maxTicks)); + let tickValues = Array.from( + { length: Math.ceil(allKeys.length / xInterval) }, + (_, i) => i * xInterval + ); + const lastIdx = allKeys.length - 1; + if (lastIdx >= 0 && tickValues[tickValues.length - 1] !== lastIdx) { + tickValues = [...tickValues, lastIdx]; + } + + // Format the label for the x axis + // if resolution is monthly, it will return the date in the format of YYYY-MM + // if resolution is daily, it will return the date in the format of YYYY-MM-DD + // if days is greater than 90, it will return the date in the format of YYYY-MM-DD + // else it will return the date in the format of MM-DD + const formatLabel = (idx) => { + const key = allKeys[idx]; + if (!key) return ""; + if (resolution === "monthly") return key; + if (days > 90) return key; + const parts = key.split("-"); + if (parts.length >= 3) return `${parts[1]}-${parts[2]}`; + return key; + }; + + return { tickValues, formatLabel }; +} + +// Compute upper bound for Y axis with padding +// Just to ensure that the area chart is not touching the top of the chart +function computePaddedMax(okArr, failArr) { + const rawMaxOk = okArr.reduce((m, p) => Math.max(m, Number(p?.data || 0)), 0); + const rawMaxFail = failArr.reduce( + (m, p) => Math.max(m, Number(p?.data || 0)), + 0 + ); + return Math.max(1, Math.ceil(Math.max(rawMaxOk, rawMaxFail) * 1.1 + 1)); +} + +// Build grouped series and matching color scheme based on filter selection +// This is used to build the grouped series and matching color scheme based on filter selection (Success / Failed / All) +function buildGroupedSeries(selectedStatus, okArr, failArr) { + let grouped = []; + let scheme = []; + + if (selectedStatus === "ALL") { + if (failArr.length > okArr.length) { + grouped = [ + { key: "Successful Runs", data: okArr }, + { key: "Failed Runs", data: failArr }, + ]; + scheme = ["#ef4444", "#22c55e"]; + } else { + grouped = [ + { key: "Failed Runs", data: failArr }, + { key: "Successful Runs", data: okArr }, + ]; + scheme = ["#22c55e", "#ef4444"]; + } + } else if (selectedStatus === "FAILED") { + grouped = [{ key: "Failed Runs", data: failArr }]; + scheme = ["#ef4444"]; + } else { + grouped = [{ key: "Successful Runs", data: okArr }]; + scheme = ["#22c55e"]; + } + + return { grouped, scheme }; +} + +const SuccessFailedRunsWidget = (props) => { + const { globalUrl, workflows, onControlsChange, onLoadingChange, onTotalsChange, overrideDays, dummyMode } = props; + + const [mode, setMode] = useState("workflows"); // 'workflows' | 'apps' + const [days, setDays] = useState(30); + const daysOptions = [5, 10, 15, 30, 60, 90, 180, 230, 365]; + + const [resolution, setResolution] = useState("daily"); // 'daily' | 'monthly' + const [selectedWorkflow, setSelectedWorkflow] = useState("ALL"); + const [selectedStatus, setSelectedStatus] = useState("ALL"); + const [seriesOk, setSeriesOk] = useState([]); + const [seriesFail, setSeriesFail] = useState([]); + const [loading, setLoading] = useState(false); + const [wfTotals, setWfTotals] = useState({ ok: 0, fail: 0, activeDays: 0 }); + + useEffect(() => { + try { + if (typeof onTotalsChange !== "function") return; + const totalOk = Math.max(0, Number(wfTotals.ok) || 0); + const totalFail = Math.max(0, Number(wfTotals.fail) || 0); + const totalRuns = totalOk + totalFail; + const activeDays = Math.max(0, Number(wfTotals.activeDays) || 0); + const timeSavedMinutes = totalRuns * RUN_MINUTES_SAVED_PER_WORKFLOW; + const moneySavedDollars = totalRuns * RUN_DOLLARS_SAVED_PER_WORKFLOW; + // Do not trigger parent updates when switching mode to avoid page blink + onTotalsChange({ days, totalRuns, successRuns: totalOk, failedRuns: totalFail, activeDays, timeSavedMinutes, moneySavedDollars }); + } catch { + onTotalsChange({ days, totalRuns: 0, successRuns: 0, failedRuns: 0, activeDays: 0, timeSavedMinutes: 0, moneySavedDollars: 0 }); + } + }, [wfTotals, days, onTotalsChange]); + + const workflowItems = useMemo(() => { + const base = [{ id: "ALL", name: "All Workflows" }]; + if (!Array.isArray(workflows)) return base; + return base.concat( + workflows + .filter((w) => w?.id && w?.name) + .map((w) => ({ id: w.id, name: w.name })) + ); + }, [workflows]); + + const fetchSeriesForKey = async (key) => { + try { + + const urlA = `${globalUrl}/api/v1/stats/${encodeURIComponent(key)}?days=${encodeURIComponent(days)}`; + const doFetch = async (u) => { + const r = await fetch(u, { method: "GET", credentials: "include" }); + if (!r.ok) return []; + const j = await r.json(); + return Array.isArray(j?.entries) ? j.entries : []; + }; + const a = await doFetch(urlA); + if (a.length > 0) return a; + + // Optional orgId fallback if exposed globally in app + // const orgId = (window && window.selectedOrganization && window.selectedOrganization.id) || null; + // if (orgId) { + // const urlB = `${globalUrl}/api/v1/orgs/${encodeURIComponent(orgId)}/stats/${encodeURIComponent(key)}?days=${encodeURIComponent(days)}`; + // const b = await doFetch(urlB); + // if (b.length > 0) return b; + // } + // return []; + } catch (e) { + return []; + } + }; + + const fetchSeries = async () => { + setLoading(true); + try { + if (dummyMode) { + // 10-day wave with a couple of bumps for a more dynamic preview + const today = new Date(); + const mk = (n, v) => ({ key: new Date(today.getFullYear(), today.getMonth(), today.getDate() - n), id: `${n}`, data: v }); + + // Success shows two bumps (days -8..-6 and -2..0) + const okVals = [7, 4, 9, 4, 6, 9, 7, 5, 8, 6]; // oldest -> newest + const failVals = [1, 0, 1, 2, 1, 1, 0, 1, 2, 1]; // small, non-zero noise + + const okSeries = okVals.map((v, idx) => mk(okVals.length - 1 - idx, v)); + const failSeries = failVals.map((v, idx) => mk(failVals.length - 1 - idx, v)); + + setSeriesOk(okSeries); + setSeriesFail(failSeries); + return; + } + const successKey = + mode === "workflows" + ? "workflow_executions_finished" + : "app_executions"; + const failedKey = + mode === "workflows" + ? "workflow_executions_failed" + : "app_executions_failed"; + const [succ, fail] = await Promise.all([ + fetchSeriesForKey(successKey), + fetchSeriesForKey(failedKey), + ]); + + let okSeries = normalizeEntries(succ); + let failSeries = normalizeEntries(fail); + + // Fallback: if empty, derive from /api/v1/stats daily_statistics + if (okSeries.length === 0 && failSeries.length === 0) { + const resp = await fetch(`${globalUrl}/api/v1/stats`, { + method: "GET", + credentials: "include", + headers: { "Content-Type": "application/json" }, + }); + if (resp.ok) { + const data = await resp.json(); + const fieldOk = + mode === "workflows" + ? "workflow_executions_finished" + : "app_executions"; + const fieldFail = + mode === "workflows" + ? "workflow_executions_failed" + : "app_executions_failed"; + const list = Array.isArray(data?.daily_statistics) + ? data.daily_statistics + : []; + const cutoff = new Date(); + cutoff.setDate(cutoff.getDate() - days); + okSeries = list + .filter(Boolean) + .map((d) => ({ + key: new Date(d?.date || Date.now()), + id: d?.date || Math.random().toString(36).slice(2), + data: Number(d?.[fieldOk] || 0), + })) + .filter((p) => p.key >= cutoff); + failSeries = list + .filter(Boolean) + .map((d) => ({ + key: new Date(d?.date || Date.now()), + id: `f-${d?.date || Math.random().toString(36).slice(2)}`, + data: Number(d?.[fieldFail] || 0), + })) + .filter((p) => p.key >= cutoff); + } + } + + setSeriesOk(okSeries); + setSeriesFail(failSeries); + } catch (e) { + setSeriesOk([]); + setSeriesFail([]); + } finally { + setLoading(false); + } + }; + + // Only fetch on first render and when days window or mode changes + const firstLoadRef = React.useRef(true); + useEffect(() => { + if (firstLoadRef.current) { + firstLoadRef.current = false; + fetchSeries(); + return; + } + fetchSeries(); + }, [days, globalUrl, mode]); + + // Apply external days override (e.g. after onboarding completes) + useEffect(() => { + if (typeof overrideDays === 'number' && overrideDays > 0 && overrideDays !== days) { + setDays(overrideDays); + } + }, [overrideDays]); + +// For the KPIs : Time saved and Money saved + useEffect(() => { + let aborted = false; + const run = async () => { + // Skip fetching/storing real stats while onboarding preview is shown + // if (dummyMode) { + // if (!aborted) setWfTotals({ ok: 0, fail: 0, activeDays: 0 }); + // return; + // } + try { + const totalEntries = await fetchSeriesForKey("workflow_executions"); + const series = normalizeEntries(totalEntries); + const dayKey = (d) => { + try { return new Date(d?.date || d?.key).toISOString().slice(0,10); } catch { return null; } + }; + const dayTotals = new Map(); + for (const it of series) { + const k = dayKey(it); if (!k) continue; dayTotals.set(k, (dayTotals.get(k) || 0) + (Number(it?.data)||0)); + } + const activeDays = Array.from(dayTotals.values()).filter(v => v > 0).length; + const ok = series.reduce((s, p) => s + (Number(p?.data)||0), 0); + if (!aborted) setWfTotals({ ok, fail: 0, activeDays }); + } catch { + if (!aborted) setWfTotals({ ok: 0, fail: 0, activeDays: 0 }); + } + }; + run(); + return () => { aborted = true; }; + }, [globalUrl, days, dummyMode, overrideDays]); + + // Notify parent about loading state changes + useEffect(() => { + if (typeof onLoadingChange === "function") { + onLoadingChange(loading); + } + }, [loading, onLoadingChange]); + + // Build filters UI once here; optionally render externally via onControlsChange + const controlsNode = React.useMemo(() => ( +
+ {/* + Workflow + + */} + + Filter + + + + Last + + + v && setMode(v)} + sx={{ + height: 37, + backgroundColor: 'rgba(255,255,255,0.06)', + border: '1px solid rgba(255,255,255,0.22)', + borderRadius: '30px', + padding: '2px', + "& .MuiToggleButton-root": { + border: "none", + borderRadius: "30px", + color: "#fff", + padding: "6px 16px", + textTransform: "none", + fontSize: "14px", + "&.Mui-selected": { + backgroundColor: "#fff", + color: "#222", + fontWeight: "600", + "&:hover": { + backgroundColor: "#fff", + }, + }, + "&:hover": { + backgroundColor: "rgba(255, 255, 255, 0.2)", + }, + }, + }} + > + + + Workflows + + + + + + Apps + + + + { + if (!v) return; + setResolution(v); + if (v === 'monthly' && days !== 180) { + setDays(180); + } else if (v === 'daily' && days !== 30) { + setDays(30); + } + }} + sx={{ + height: 37, + backgroundColor: 'rgba(255,255,255,0.06)', + border: '1px solid rgba(255,255,255,0.22)', + borderRadius: '30px', + padding: '2px', + "& .MuiToggleButton-root": { + border: "none", + borderRadius: "30px", + color: "#fff", + padding: "6px 16px", + textTransform: "none", + fontSize: "14px", + "&.Mui-selected": { + backgroundColor: "#fff", + color: "#222", + fontWeight: "600", + "&:hover": { + backgroundColor: "#fff", + }, + }, + "&:hover": { + backgroundColor: "rgba(255, 255, 255, 0.2)", + }, + }, + }} + > + + + Daily + + + + + + Monthly + + + +
+ ), + [selectedStatus, mode, days, resolution, workflowItems] + ); + + useEffect(() => { + if (typeof onControlsChange === "function") { + onControlsChange(controlsNode); + return () => { + onControlsChange(null); + }; + } + }, [onControlsChange, controlsNode]); + + return ( +
+ {/* Top controls row: title left, all filters on the right */} +
+ {/* Title moved inside the area chart card */} + + {!onControlsChange && ( + <> + {controlsNode} + + )} +
+ + {/* Content row: area chart (left) + ring gauges (right) in separate sub-cards */} +
+
+ Successful vs Failed Runs ({mode === "workflows" ? "Workflows" : "Apps"}) + {(() => { + // Build unified timeline by day or by month + const useOk = Array.isArray(seriesOk) ? seriesOk : []; + const useFail = Array.isArray(seriesFail) ? seriesFail : []; + + const okMap = bucketSeries(useOk, resolution); + const failMap = bucketSeries(useFail, resolution); + let allKeys = Array.from( + new Set([...okMap.keys(), ...failMap.keys()]) + ); + allKeys.sort((a, b) => new Date(a) - new Date(b)); + allKeys = buildBackfilledKeys(allKeys, resolution); + + if (allKeys.length === 0) { + return ( +
+ + No data available + +
+ ); + } + + let okArr = allKeys.map((k, i) => ({ + key: i, + data: okMap.get(k) || 0, + })); + let failArr = allKeys.map((k, i) => ({ + key: i, + data: failMap.get(k) || 0, + })); + okArr = ensureMinTwoPoints(okArr); + failArr = ensureMinTwoPoints(failArr); + + const { grouped, scheme } = buildGroupedSeries( + selectedStatus, + okArr, + failArr + ); + const { tickValues, formatLabel } = computeTicks( + allKeys, + days, + resolution + ); + const paddedMax = computePaddedMax(okArr, failArr); + + return ( + formatCompactNumber(d)} />} + /> + } + /> + } + xAxis={ + formatLabel(Number(d))} + /> + } + tickValues={tickValues} + /> + } + /> + } + gridlines={} />} + series={ + + } + colorScheme={scheme} + tooltip={ + { + const idx = Math.max(0, Number(d?.x ?? 0)); + const rows = (grouped || []).map( + (seriesItem, i) => { + const point = Array.isArray(seriesItem?.data) + ? seriesItem.data[ + Math.min( + idx, + seriesItem.data.length - 1 + ) + ] + : null; + const value = Number(point?.data || 0); + return { + label: seriesItem?.key, + value, + color: scheme[scheme.length - 1 - i], + }; + } + ); + return ( +
+
+ {formatLabel(idx)} +
+
+ {rows.reverse().map((r) => ( +
+ + + {r.label} + + + {r.value} + +
+ ))} +
+
+ ); + }} + /> + } + /> + } + /> + } + /> + ); + })()} +
+
+
+ Successful Runs +
+
+ Failed Runs +
+
+
+ + X: {resolution === "monthly" ? "Date (month)" : "Date (MM-DD)"} + + |Y: Runs +
+
+
+ + {/* Ring gauges */} +
+
+ + {mode === "workflows" ? "Workflows" : "Apps"} Success Rates + +
+ {(() => { + const totalOk = (seriesOk || []).reduce( + (s, p) => s + (p?.data || 0), + 0 + ); + const totalFail = (seriesFail || []).reduce( + (s, p) => s + (p?.data || 0), + 0 + ); + const total = totalOk + totalFail; + const okPct = total > 0 ? Math.round((totalOk / total) * 100) : 0; + const failPct = + total > 0 ? Math.round((totalFail / total) * 100) : 0; + return ( + <> + + + + ); + })()} +
+
+
+
+
+ ); +}; + +export default SuccessFailedRunsWidget; + +// Lightweight SVG ring to avoid RadialGauge runtime issues +function Ring({ title, color, bg, percent }) { + const stroke = 9; + const r = 60; + const c = 2 * Math.PI * r; + const filled = (Math.max(0, Math.min(100, Number(percent) || 0)) / 100) * c; + + return ( +
+
+ + + + + {Math.round(Math.max(0, Math.min(100, Number(percent) || 0)))}% + + + {title} +
+
+ ); +} + +// (Old) custom area/line removed in favor of Reaviz AreaChart grouped + +function LegendDot({ color }) { + return ( + + ); +} From 23a6b4eff397903f0f4d26bb95e9a2498d3d4428 Mon Sep 17 00:00:00 2001 From: Frikky Date: Mon, 20 Oct 2025 12:42:10 +0200 Subject: [PATCH 54/57] Singul build finally worked --- backend/go-app/go.sum | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 8d99773a..2f614c0b 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -152,8 +152,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0= github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4= -github.com/frikky/schemaless v0.0.20 h1:S/A2pQcRN9qa2RnufvxwCeM06trjG0JLTF3urt1tFQI= -github.com/frikky/schemaless v0.0.20/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= +github.com/frikky/schemaless v0.0.22 h1:aMc7cc/lr1zpogjGWbY0j6J2f6QqyfPbbW6Y9JgTAqE= +github.com/frikky/schemaless v0.0.22/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY= github.com/fsouza/go-dockerclient v1.12.1 h1:FMoLq+Zhv9Oz/rFmu6JWkImfr6CBgZOPcL+bHW4gS0o= github.com/fsouza/go-dockerclient v1.12.1/go.mod h1:OqsgJJcpCwqyM3JED7TdfM9QVWS5O7jSYwXxYKmOooY= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= @@ -363,12 +363,10 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.9.30 h1:3CYvNyD7sTxdxoZjTVrtaDqFvSWQRKAFGaga6rPGf8A= -github.com/shuffle/shuffle-shared v0.9.30/go.mod h1:PhDEizuz4SmJaSmy0+yrFWwD1mXVUsy8/knKlrqF1qw= github.com/shuffle/shuffle-shared v0.9.31 h1:BMoDO4Sgz4+I12aHhc3cWHiCuAQ827XIxajPqCJ9cXA= github.com/shuffle/shuffle-shared v0.9.31/go.mod h1:vfI2QDGphZGrcwuUPQ1yI/Hgc8aseFro5+2k36irfkQ= -github.com/shuffle/singul v0.0.16 h1:dW+0Mln9R1aUJ0fjikpWcxbjoQWqJHxe4kSxh2tQN5E= -github.com/shuffle/singul v0.0.16/go.mod h1:LYkp320A6gsoPlYbXUM+WvEPUVAuutlSsqnVKyRy4gs= +github.com/shuffle/singul v0.0.17 h1:mxaPtj6z85Nf6tl7L2gwDliTfEZtRQqApuu9iKcP75o= +github.com/shuffle/singul v0.0.17/go.mod h1:8c42n1NahhCIPxzLxwp9eYbWkvY4+ct0jfbhkRsRRsY= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= From d26b7779e1ab7af2cc253c3d525447d706054251 Mon Sep 17 00:00:00 2001 From: Frikky Date: Tue, 21 Oct 2025 00:34:57 +0200 Subject: [PATCH 55/57] Tons of minor fixes --- backend/go-app/go.mod | 2 +- backend/go-app/go.sum | 4 +- backend/go-app/main.go | 49 ++++++- frontend/src/components/AdminNavBar.jsx | 2 + frontend/src/components/Billing.jsx | 85 +++++++++-- frontend/src/components/CacheView.jsx | 10 +- .../src/components/DashboardOnboarding.jsx | 85 +++++++++-- frontend/src/components/LeftSideBar.jsx | 40 +++--- frontend/src/components/LicencePopup.jsx | 14 +- frontend/src/components/OrganizationTab.jsx | 64 ++------- frontend/src/components/Priorities.jsx | 1 + .../components/SuccessFailedRunsWidget.jsx | 5 +- frontend/src/views/LoginPage.jsx | 4 +- frontend/src/views/NewDashboard.jsx | 133 +++++++++++++----- frontend/src/views/Workflows2.jsx | 106 +++++++++++++- 15 files changed, 463 insertions(+), 141 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index b563af28..6a7775c9 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -24,7 +24,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.31 + github.com/shuffle/shuffle-shared v0.9.32 github.com/shuffle/singul v0.0.17 golang.org/x/crypto v0.40.0 google.golang.org/api v0.236.0 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 2f614c0b..1f1f22c7 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -363,8 +363,8 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.9.31 h1:BMoDO4Sgz4+I12aHhc3cWHiCuAQ827XIxajPqCJ9cXA= -github.com/shuffle/shuffle-shared v0.9.31/go.mod h1:vfI2QDGphZGrcwuUPQ1yI/Hgc8aseFro5+2k36irfkQ= +github.com/shuffle/shuffle-shared v0.9.32 h1:hsF2YkKHgaNpqhh2oZs31BgPSjN+YjGNnd9WmD0qh3w= +github.com/shuffle/shuffle-shared v0.9.32/go.mod h1:vfI2QDGphZGrcwuUPQ1yI/Hgc8aseFro5+2k36irfkQ= github.com/shuffle/singul v0.0.17 h1:mxaPtj6z85Nf6tl7L2gwDliTfEZtRQqApuu9iKcP75o= github.com/shuffle/singul v0.0.17/go.mod h1:8c42n1NahhCIPxzLxwp9eYbWkvY4+ct0jfbhkRsRRsY= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index d24a7bbe..633cd6f4 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -1282,7 +1282,6 @@ func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { } count := len(users) - if count == 0 { log.Printf("[WARNING] No users - redirecting for management user") resp.WriteHeader(200) @@ -4662,6 +4661,45 @@ func runInitEs(ctx context.Context) { } } + // Self-cleaning + go func() { + cursor := "" + cnt := 0 + newCtx := context.Background() + for _, org := range activeOrgs { + if len(org.Id) == 0 { + log.Printf("[DEBUG] No ID found for org with name '%s'. Why was it made?", org.Name) + continue + } + + log.Printf("[INFO] Starting self-cleanup of cache keys for org %s", org.Id) + + for { + keys, newCursor, err := shuffle.GetAllCacheKeys(newCtx, org.Id, "", 1000, cursor) + if err != nil { + //log.Printf("[ERROR] Failed getting all cache keys for cleanup: %s", err) + break + } + + if newCursor == cursor || len(newCursor) == 0 { + break + } + + if len(keys) == 0 { + break + } + + cursor = newCursor + cnt += 1 + if cnt > 10 { + break + } + } + + log.Printf("[INFO] Finished self-cleanup of cache keys for org %s", org.Id) + } + }() + log.Printf("[INFO] Finished INIT (ES)") } @@ -5481,6 +5519,11 @@ func initHandlers() { r.HandleFunc("/api/v2/workflows/{key}/executions", shuffle.GetWorkflowExecutionsV2).Methods("GET", "OPTIONS") r.HandleFunc("/api/v2/workflows/generate/llm", shuffle.HandleWorkflowGenerationResponse).Methods("POST", "OPTIONS") r.HandleFunc("/api/v2/workflows/edit/llm", shuffle.HandleEditWorkflowWithLLM).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v2/workflows/generate", shuffle.GenerateSingulWorkflows).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v2/datastore", shuffle.HandleListCacheKeys).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v2/datastore", shuffle.HandleSetDatastoreKey).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v2/datastore/category/{category_key}", shuffle.HandleListCacheKeys).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v2/datastore/automate", shuffle.HandleDatastoreCategoryConfig).Methods("POST", "OPTIONS") // New for recommendations in Shuffle r.HandleFunc("/api/v1/recommendations/get_actions", shuffle.HandleActionRecommendation).Methods("POST", "OPTIONS") @@ -5575,6 +5618,10 @@ func initHandlers() { r.HandleFunc("/api/v1/orgs/{orgId}/stats/{key}", shuffle.GetSpecificStats).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/statistics", shuffle.HandleGetStatistics).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/stats", shuffle.HandleGetStatistics).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/stats", shuffle.HandleAppendStatistics).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/stats/{key}", shuffle.GetSpecificStats).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/orgs/{orgId}/cache", shuffle.HandleListCacheKeys).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/cache", shuffle.HandleSetCacheKey).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/cache/{cache_key}", shuffle.HandleDeleteCacheKey).Methods("DELETE", "OPTIONS") diff --git a/frontend/src/components/AdminNavBar.jsx b/frontend/src/components/AdminNavBar.jsx index fbbc391d..9ebea75b 100644 --- a/frontend/src/components/AdminNavBar.jsx +++ b/frontend/src/components/AdminNavBar.jsx @@ -1,5 +1,6 @@ import React, { useState, useEffect, useContext, memo } from 'react'; import { useNavigate, useLocation } from 'react-router-dom'; + import OrganizationTab from '../components/OrganizationTab.jsx'; import PartnerTab from '../components/PartnerTab.jsx'; import UserManagmentTab from '../components/UserManagmentTab.jsx'; @@ -20,6 +21,7 @@ import { FmdGoodOutlined as FmdGoodOutlinedIcon, GroupOutlined as GroupOutlinedIcon } from '@mui/icons-material'; + import theme, { getTheme } from '../theme.jsx'; import { Button, Skeleton, Tooltip } from '@mui/material'; import { Index } from 'react-instantsearch-dom'; diff --git a/frontend/src/components/Billing.jsx b/frontend/src/components/Billing.jsx index b93848b1..2bc8290b 100644 --- a/frontend/src/components/Billing.jsx +++ b/frontend/src/components/Billing.jsx @@ -47,7 +47,9 @@ import { CheckCircle, Padding, Edit, - Search as SearchIcon + Search as SearchIcon, + CheckCircle as CheckCircleIcon, + Cancel as CancelIcon, } from "@mui/icons-material"; //import { useAlert @@ -61,6 +63,62 @@ import { Context } from "../context/ContextApi.jsx"; import DeleteIcon from '@mui/icons-material/Delete'; import { DataGrid } from "@mui/x-data-grid"; +const ProductionStatus = ({ selectedOrganization, userdata, isCloud, theme }) => { + var isProdStatusOn; + if (selectedOrganization !== undefined && selectedOrganization?.subscriptions !== undefined && selectedOrganization?.subscriptions[0] !== undefined) { + isProdStatusOn = selectedOrganization?.subscriptions[0]?.name?.toLowerCase()?.includes("enterprise") && selectedOrganization?.subscriptions[0]?.active; + } else { + isProdStatusOn = false; + } + const rows = [ + { label: 'Licensed', ok: isProdStatusOn }, + { label: 'Multi-Tenant', ok: isProdStatusOn }, + { label: 'High Availability', ok: isProdStatusOn }, + { label: 'Robust Infrastructure', ok: isProdStatusOn }, + ]; + + return ( +
+
+ Production Status +
+ + {isProdStatusOn ? "ON" : "OFF"} +
+
+ + Monitor your production status to stay informed about available features. + +
+ {rows.map((row) => ( +
+ {row.ok ? ( + + ) : ( + + )} + {row.label} +
+ ))} +
+ + + + Shuffle Enterprise is designed for organizations that require scalability, high availability, dedicated support and more to run mission-critical workflows in production environments. + + + More about upgrading below. If you want to know more, please contact support@shuffler.io directly. + +
+ ); +}; + const Billing = memo((props) => { const { globalUrl, userdata, serverside, billingInfo, stripeKey,isLoaded, selectedOrganization, handleGetOrg, clickedFromOrgTab, removeCookie} = props; //const alert = useAlert(); @@ -2085,27 +2143,32 @@ const Billing = memo((props) => { return ( -
-
+
+
+ + {isCloud ? null : } + {addDealModal} - {clickedFromOrgTab ? - Billing & Licensing : - - Billing & Licensing - } + {clickedFromOrgTab ? + Billing & Licensing + : + + Billing & Licensing + + } {userdata?.org_status?.includes("integration_partner") && userdata?.org_status?.includes("sub_org") ? null : <> {clickedFromOrgTab ? {isCloud ? "Get more out of Shuffle by adding your credit card, such as no App Run limitations, and priority support from our team. We use Stripe to manage subscriptions and do not store any of your billing information. You can manage your subscription and billing information below." : - !(selectedOrganization?.subscriptions !== undefined && selectedOrganization?.subscriptions.length > 0 && selectedOrganization?.subscriptions[0]?.name?.toLowerCase().includes("enterprise") && selectedOrganization?.subscriptions[0]?.active) ? "Shuffle is an Enterprise automation platform, and a license is required. We do however offer a Scale license with HA guarantees, along with support hours. By buying a license on https://shuffler.io, you can get access to the license immediately, and if Cloud Syncronisation is enabled, the UI in your local instance will also update." : "Here you can check your license and billing information." + !(selectedOrganization?.subscriptions !== undefined && selectedOrganization?.subscriptions.length > 0 && selectedOrganization?.subscriptions[0]?.name?.toLowerCase().includes("enterprise") && selectedOrganization?.subscriptions[0]?.active) ? "Shuffle is an Enterprise automation platform, and a license is required at scale. We offer a license with HA guarantees, higher limits, along along with support hours. By buying a license on https://shuffler.io, you can get access to the license immediately, and if Cloud Syncronisation is enabled, the UI in your local instance will also update." : "Here you can check your license and billing information." } : {isCloud ? "Get more out of Shuffle by adding your credit card, such as no App Run limitations, and priority support from our team. We use Stripe to manage subscriptions and do not store any of your billing information. You can manage your subscription and billing information below." : - !(selectedOrganization?.subscriptions !== undefined && selectedOrganization?.subscriptions.length > 0 && selectedOrganization?.subscriptions[0]?.name?.toLowerCase().includes("enterprise") && selectedOrganization?.subscriptions[0]?.active) ? "Shuffle is an Enterprise automation platform, and a license is required. We do however offer a Scale license with HA guarantees, along with support hours. By buying a license on https://shuffler.io, you can get access to the license immediately, and if Cloud Syncronisation is enabled, the UI in your local instance will also update." : "Here you can check your license and billing information." + !(selectedOrganization?.subscriptions !== undefined && selectedOrganization?.subscriptions.length > 0 && selectedOrganization?.subscriptions[0]?.name?.toLowerCase().includes("enterprise") && selectedOrganization?.subscriptions[0]?.active) ? "Shuffle is an Enterprise automation platform, and a license is required at scale. We offer a Scale license with HA guarantees, along with support hours. By buying a license on https://shuffler.io, you can get access to the license immediately, and if Cloud Syncronisation is enabled, the UI in your local instance will also update." : "Here you can check your license and billing information." } } } @@ -3454,7 +3517,7 @@ const PaddingWrapper = memo(({ clickedFromOrgTab, children }) => { height: '100%', boxSizing: 'border-box', overflow: 'hidden', - maxHeight: "1700px", + maxHeight: 3000, overflowY: "auto", scrollbarColor: theme.palette.scrollbarColorTransparent, scrollbarWidth: 'thin' diff --git a/frontend/src/components/CacheView.jsx b/frontend/src/components/CacheView.jsx index bbc93d11..3c19da36 100644 --- a/frontend/src/components/CacheView.jsx +++ b/frontend/src/components/CacheView.jsx @@ -575,7 +575,10 @@ const CacheView = memo((props) => { .then((responseJson) => { setAddCache(responseJson); toast.success("Edit saved"); - listOrgCache(orgId, selectedCategory, 0, pageSize, page); + setTimeout(() => { + listOrgCache(orgId, selectedCategory, 0, pageSize, page); + }, 7500); + setModalOpen(false); }) .catch((error) => { @@ -613,7 +616,10 @@ const CacheView = memo((props) => { .then((responseJson) => { setAddCache(responseJson); toast.success("New key added!"); - listOrgCache(orgId, selectedCategory, 0, pageSize, page); + + setTimeout(() => { + listOrgCache(orgId, selectedCategory, 0, pageSize, page); + }, 5000); setModalOpen(false); }) .catch((error) => { diff --git a/frontend/src/components/DashboardOnboarding.jsx b/frontend/src/components/DashboardOnboarding.jsx index 3ad7a728..54cebcc2 100644 --- a/frontend/src/components/DashboardOnboarding.jsx +++ b/frontend/src/components/DashboardOnboarding.jsx @@ -6,7 +6,10 @@ import { Stack, styled, } from "@mui/material"; + import theme from "../theme.jsx"; +import { toast } from "react-toastify"; +import { useNavigate } from 'react-router-dom'; // Simple icon placeholders; replace with proper assets if desired const StepIcon = styled("div")(({ completed }) => ({ @@ -127,6 +130,9 @@ const DashboardOnboarding = ({ footer, globalUrl, onExplore, + setOnboardingOpen, + isProdStatusOn, + isCloud, }) => { // Internal completion state only; handlers are defined separately const [completed, setCompleted] = React.useState({ @@ -140,6 +146,7 @@ const DashboardOnboarding = ({ const [checkingWait, setCheckingWait] = React.useState(false); const [flashKeys, setFlashKeys] = React.useState([]); const [waitProgress, setWaitProgress] = React.useState(0); + const navigate = useNavigate(); // Load persisted completion state React.useEffect(() => { @@ -294,7 +301,7 @@ const DashboardOnboarding = ({ { index: 5, key: 'invite', - title: 'Invite more team members (optional)', + title: 'Invite your team members', description: 'Add teammates to collaborate in your org.', primaryCta: { label: 'Open users page', onClick: handleOpenUsers }, completed: completed.invite, @@ -320,7 +327,7 @@ const DashboardOnboarding = ({ if (!open) return null; return ( - + {/* Blur overlay with visible background */} {/* Header */} @@ -382,8 +391,55 @@ const DashboardOnboarding = ({ + {!isCloud ? ( +
{ + navigate("/admin?admin_tab=billingstats") + }} + > + + + {isProdStatusOn ? "Production" : "NOT Production"} + +
+ ) : null} +
+ {/* Steps list with a single continuous rail */} {/* Base grey rail */} @@ -427,12 +483,25 @@ const DashboardOnboarding = ({ {footer} + + diff --git a/frontend/src/components/LeftSideBar.jsx b/frontend/src/components/LeftSideBar.jsx index 0ca71daf..c25c6fc2 100644 --- a/frontend/src/components/LeftSideBar.jsx +++ b/frontend/src/components/LeftSideBar.jsx @@ -1226,7 +1226,7 @@ const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, }) => { +
+ } + {!isCloud ? (
{ cursor: "pointer", }} onClick={() => { - navigate("/admin?admin_tab=prodstatus") + navigate("/admin?admin_tab=billingstats") }} > { color: isProdStatusOn ? "#2BC07E" : "#FD4C62", }} > - {expandLeftNav ? isProdStatusOn ? "Prod. Status ON" : "Prod. Status OFF" : isProdStatusOn ? "ON" : "OFF"} + {expandLeftNav ? isProdStatusOn ? "Production" : "NOT production" : isProdStatusOn ? "ON" : "OFF"}
) : null} - - {userdata?.licensed !== true && !userdata?.org_status?.includes("integration_partner") && expandLeftNav && !isProdStatusOn && -
- -
- } + { {!isPaidPlan && (