From fd2996f101d19ee7fe4d26913c3e7301cb5c1863 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Tue, 18 Feb 2025 13:24:10 +0100 Subject: [PATCH 01/33] allow to change service accounts for worker and apps Signed-off-by: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> --- functions/onprem/orborus/orborus.go | 39 +++++++++++++---------------- functions/onprem/worker/worker.go | 13 ++++++---- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 093faf79..2ee3c441 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -75,9 +75,13 @@ var workerVersion = os.Getenv("SHUFFLE_WORKER_VERSION") var newWorkerImage = os.Getenv("SHUFFLE_WORKER_IMAGE") var dockerSwarmBridgeMTU = os.Getenv("SHUFFLE_SWARM_BRIDGE_DEFAULT_MTU") var dockerSwarmBridgeInterface = os.Getenv("SHUFFLE_SWARM_BRIDGE_DEFAULT_INTERFACE") +var maxCPUPercent = 90 + +// Kubernetes settings var isKubernetes = os.Getenv("IS_KUBERNETES") var kubernetesNamespace = os.Getenv("KUBERNETES_NAMESPACE") -var maxCPUPercent = 90 +var workerServiceAccountName = os.Getenv("SHUFFLE_WORKER_SERVICE_ACCOUNT_NAME") +var appServiceAccountName = os.Getenv("SHUFFLE_APP_SERVICE_ACCOUNT_NAME") // var baseimagename = "docker.pkg.github.com/shuffle/shuffle" // var baseimagename = "ghcr.io/frikky" @@ -1000,6 +1004,10 @@ func deployK8sWorker(image string, identifier string, env []string) error { env = append(env, fmt.Sprintf("SHUFFLE_USE_GHCR_OVERRIDE_FOR_AUTODEPLOY=%s", os.Getenv("SHUFFLE_USE_GHCR_OVERRIDE_FOR_AUTODEPLOY"))) } + if len(appServiceAccountName) > 0 { + env = append(env, fmt.Sprintf("SHUFFLE_APP_SERVICE_ACCOUNT_NAME=%s", appServiceAccountName)) + } + clientset, _, err := shuffle.GetKubernetesClient() if err != nil { log.Printf("[ERROR] Error getting kubernetes client:", err) @@ -1008,18 +1016,6 @@ func deployK8sWorker(image string, identifier string, env []string) error { //env = append(env, fmt.Sprintf("KUBERNETES_CONFIG=%s", config.String())) - // FIXME: When a service account is used, the account is also mounted in the pod - // The volume mount location is: - // /var/run/secrets/kubernetes.io/serviceaccount - - // Look for if there is a default service account in use - if len(os.Getenv("KUBERNETES_SERVICE_ACCOUNT")) > 0 { - log.Printf("[DEBUG] Using Kubernetes service account %s", os.Getenv("KUBERNETES_SERVICE_ACCOUNT")) - env = append(env, fmt.Sprintf("KUBERNETES_SERVICE_ACCOUNT=%s", os.Getenv("KUBERNETES_SERVICE_ACCOUNT"))) - - // use k8s downward API to find it if we are in a pod - } - // Check if namespace exist as variable. If so, make it if len(os.Getenv("KUBERNETES_NAMESPACE")) > 0 && !namespacemade { kubernetesNamespace = os.Getenv("KUBERNETES_NAMESPACE") @@ -1199,7 +1195,8 @@ func deployK8sWorker(image string, identifier string, env []string) error { Containers: []corev1.Container{ containerAttachment, }, - DNSPolicy: corev1.DNSClusterFirst, + DNSPolicy: corev1.DNSClusterFirst, + ServiceAccountName: workerServiceAccountName, }, }, }, @@ -2296,7 +2293,7 @@ func main() { // Manual command = overrides to allow starting of Tenzir from the frontend anyway. os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") - tenzirDisabled = false + tenzirDisabled = false // Removed either way toBeRemoved.Data = append(toBeRemoved.Data, incRequest) @@ -2306,7 +2303,7 @@ func main() { if strings.Contains(fmt.Sprintf("%s", err), "node available") { // Disabling until UI is updated os.Setenv("SHUFFLE_SKIP_PIPELINES", "true") - tenzirDisabled = true + tenzirDisabled = true log.Printf("[ERROR] Failed to start tenzir, reason: %s", err) err = shuffle.CreateOrgNotification( @@ -2625,7 +2622,7 @@ func deployTenzirNode() error { // return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES") //log.Printf("[INFO] Pipelines are enabled by user") } else { - return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES") + return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES") } if isKubernetes == "true" { @@ -2825,12 +2822,12 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri }, } - // FIXME: Is this necessary? Seems to screw up networking: + // FIXME: Is this necessary? Seems to screw up networking: // conflicting options: hostname and the network mode /* - if isKubernetes != "true" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" { - hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) - } + if isKubernetes != "true" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" { + hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) + } */ resp, err := dockercli.ContainerCreate(ctx, config, hostConfig, networkingConfig, nil, containerName) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 996764fd..6e48043a 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -56,6 +56,7 @@ var logsDisabled = os.Getenv("SHUFFLE_LOGS_DISABLED") var cleanupEnv = strings.ToLower(os.Getenv("CLEANUP")) var swarmNetworkName = os.Getenv("SHUFFLE_SWARM_NETWORK_NAME") var dockerApiVersion = strings.ToLower(os.Getenv("DOCKER_API_VERSION")) +var appServiceAccountName = os.Getenv("SHUFFLE_APP_SERVICE_ACCOUNT_NAME") var baseimagename = "frikky/shuffle" var kubernetesNamespace = os.Getenv("KUBERNETES_NAMESPACE") @@ -105,11 +106,11 @@ var window = shuffle.NewTimeWindow(10 * time.Second) // Images to be autodeployed in the latest version of Shuffle. var autoDeploy = map[string]string{ - "http:1.4.0": "frikky/shuffle:http_1.4.0", - "http:1.3.0": "frikky/shuffle:http_1.3.0", - "shuffle-tools:1.2.0": "frikky/shuffle:shuffle-tools_1.2.0", - "shuffle-subflow:1.0.0": "frikky/shuffle:shuffle-subflow_1.0.0", - "shuffle-subflow:1.1.0": "frikky/shuffle:shuffle-subflow_1.1.0", + "http:1.4.0": "frikky/shuffle:http_1.4.0", + "http:1.3.0": "frikky/shuffle:http_1.3.0", + "shuffle-tools:1.2.0": "frikky/shuffle:shuffle-tools_1.2.0", + "shuffle-subflow:1.0.0": "frikky/shuffle:shuffle-subflow_1.0.0", + "shuffle-subflow:1.1.0": "frikky/shuffle:shuffle-subflow_1.1.0", // "shuffle-tools-fork:1.0.0": "frikky/shuffle:shuffle-tools-fork_1.0.0", } @@ -592,6 +593,8 @@ func deployk8sApp(image string, identifier string, env []string) error { Env: buildEnvVars(envMap), }, }, + DNSPolicy: corev1.DNSClusterFirst, + ServiceAccountName: appServiceAccountName}, }, }, }, From e554943c8d518b87391ac4308629de5247ac4fe5 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Tue, 18 Feb 2025 16:22:48 +0100 Subject: [PATCH 02/33] add common kubernetes labels Signed-off-by: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> --- functions/onprem/orborus/orborus.go | 25 +++++++++++++++------ functions/onprem/worker/worker.go | 35 ++++++++++++++++++----------- 2 files changed, 40 insertions(+), 20 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 093faf79..894f6594 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -1078,8 +1078,17 @@ func deployK8sWorker(image string, identifier string, env []string) error { } } - containerLabels := map[string]string{ - "container": "shuffle-worker", + labels := map[string]string{ + "app.kubernetes.io/name": "shuffle-worker", + "app.kubernetes.io/instance": identifier, + // "app.kubernetes.io/version": "", + "app.kuvernetes.io/part-of": "shuffle", + "app.kubernetes.io/managed-by": "shuffle-orborus", + } + + matchLabels := map[string]string{ + "app.kubernetes.io/name": "shuffle-worker", + "app.kubernetes.io/instance": identifier, } containerAttachment := corev1.Container{ @@ -1184,16 +1193,17 @@ func deployK8sWorker(image string, identifier string, env []string) error { deployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: identifier, + Name: identifier, + Labels: labels, }, Spec: appsv1.DeploymentSpec{ Replicas: int32Ptr(replicaNumberInt32), Selector: &metav1.LabelSelector{ - MatchLabels: containerLabels, + MatchLabels: matchLabels, }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: containerLabels, + Labels: labels, }, Spec: corev1.PodSpec{ Containers: []corev1.Container{ @@ -1214,10 +1224,11 @@ func deployK8sWorker(image string, identifier string, env []string) error { // kubectl expose deployment shuffle-workers --type=NodePort --port=33333 --target-port=33333 service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: identifier, + Name: identifier, + Labels: labels, }, Spec: corev1.ServiceSpec{ - Selector: containerLabels, + Selector: matchLabels, Ports: []corev1.ServicePort{ { Protocol: "TCP", diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 996764fd..064301b9 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -487,9 +487,22 @@ func deployk8sApp(image string, identifier string, env []string) error { //fix naming convention // podUuid := uuid.NewV4().String() - // podName := fmt.Sprintf("%s-%s", value, podUuid) + // name := fmt.Sprintf("%s-%s", value, podUuid) // replace identifier "_" with "-" - podName := strings.ReplaceAll(identifier, "_", "-") + name := strings.ReplaceAll(identifier, "_", "-") + + labels := map[string]string{ + "app.kubernetes.io/name": "shuffle-app", + "app.kubernetes.io/instance": name, + // "app.kubernetes.io/version": "", + "app.kuvernetes.io/part-of": "shuffle", + "app.kubernetes.io/managed-by": "shuffle-worker", + } + + matchLabels := map[string]string{ + "app.kubernetes.io/name": "shuffle-app", + "app.kubernetes.io/instance": name, + } // pod := &corev1.Pod{ // ObjectMeta: metav1.ObjectMeta{ @@ -569,20 +582,17 @@ func deployk8sApp(image string, identifier string, env []string) error { deployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: podName, + Name: name, + Labels: labels, }, Spec: appsv1.DeploymentSpec{ Replicas: int32Ptr(replicaNumberInt32), Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": podName, - }, + MatchLabels: matchLabels, }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": podName, - }, + Labels: labels, }, Spec: corev1.PodSpec{ Containers: []corev1.Container{ @@ -606,12 +616,11 @@ func deployk8sApp(image string, identifier string, env []string) error { // kubectl expose deployment {podName} --type=NodePort --port=80 --target-port=80 service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: podName, + Name: name, + Labels: labels, }, Spec: corev1.ServiceSpec{ - Selector: map[string]string{ - "app": podName, - }, + Selector: matchLabels, Ports: []corev1.ServicePort{ { Protocol: "TCP", From ead70439be8464356e78cff8c8da1cf2513e59b1 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Tue, 18 Feb 2025 16:32:36 +0100 Subject: [PATCH 03/33] fix typo, format file Signed-off-by: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> --- functions/onprem/orborus/orborus.go | 30 ++++++++++++++--------------- functions/onprem/worker/worker.go | 14 +++++++------- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 894f6594..bd551a99 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -1082,7 +1082,7 @@ func deployK8sWorker(image string, identifier string, env []string) error { "app.kubernetes.io/name": "shuffle-worker", "app.kubernetes.io/instance": identifier, // "app.kubernetes.io/version": "", - "app.kuvernetes.io/part-of": "shuffle", + "app.kubernetes.io/part-of": "shuffle", "app.kubernetes.io/managed-by": "shuffle-orborus", } @@ -2237,7 +2237,7 @@ func main() { if incRequest.Type == "PIPELINE_CREATE" || incRequest.Type == "PIPELINE_START" || incRequest.Type == "PIPELINE_STOP" || incRequest.Type == "PIPELINE_DELETE" { os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") - tenzirDisabled = false + tenzirDisabled = false // Running NEW or editing pipelines err := handlePipeline(incRequest) @@ -2262,7 +2262,7 @@ func main() { } else if incRequest.Type == "CATEGORY_UPDATE" { os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") - tenzirDisabled = false + tenzirDisabled = false err = handleFileCategoryChange() if err != nil { @@ -2307,7 +2307,7 @@ func main() { // Manual command = overrides to allow starting of Tenzir from the frontend anyway. os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") - tenzirDisabled = false + tenzirDisabled = false // Removed either way toBeRemoved.Data = append(toBeRemoved.Data, incRequest) @@ -2317,7 +2317,7 @@ func main() { if strings.Contains(fmt.Sprintf("%s", err), "node available") { // Disabling until UI is updated os.Setenv("SHUFFLE_SKIP_PIPELINES", "true") - tenzirDisabled = true + tenzirDisabled = true log.Printf("[ERROR] Failed to start tenzir, reason: %s", err) err = shuffle.CreateOrgNotification( @@ -2636,7 +2636,7 @@ func deployTenzirNode() error { // return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES") //log.Printf("[INFO] Pipelines are enabled by user") } else { - return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES") + return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES") } if isKubernetes == "true" { @@ -2836,12 +2836,12 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri }, } - // FIXME: Is this necessary? Seems to screw up networking: + // FIXME: Is this necessary? Seems to screw up networking: // conflicting options: hostname and the network mode /* - if isKubernetes != "true" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" { - hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) - } + if isKubernetes != "true" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" { + hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) + } */ resp, err := dockercli.ContainerCreate(ctx, config, hostConfig, networkingConfig, nil, containerName) @@ -3049,7 +3049,6 @@ func createPipeline(command, identifier string) (string, error) { return "", err } - body, err := ioutil.ReadAll(resp.Body) if err != nil { log.Printf("[ERROR] Failed reading response body: %s", err) @@ -3067,8 +3066,8 @@ func createPipeline(command, identifier string) (string, error) { } type PipelineResponse struct { - ID string `json:"id"` - Message string `json:"message"` + ID string `json:"id"` + Message string `json:"message"` Severity string `json:"severity"` } @@ -3291,7 +3290,7 @@ func handleFileCategoryChange() error { tenzirStorageFolder = "/tmp/" } - tenzirStorageFolder = strings.TrimRight(tenzirStorageFolder, "/") + tenzirStorageFolder = strings.TrimRight(tenzirStorageFolder, "/") sigmaPath := fmt.Sprintf("%s/sigma_rules", tenzirStorageFolder) err = extractZIP("files.zip", sigmaPath) if err != nil { @@ -3299,7 +3298,6 @@ func handleFileCategoryChange() error { return err } - log.Printf("[DEBUG] Detection files copied to '%s' successfully.", sigmaPath) return nil @@ -3383,7 +3381,7 @@ func removeFileCategory() error { tenzirStorageFolder = "/tmp/" } - tenzirStorageFolder = strings.TrimRight(tenzirStorageFolder, "/") + tenzirStorageFolder = strings.TrimRight(tenzirStorageFolder, "/") //sigmaPath := "/var/lib/tenzir/sigma_rules/*" sigmaPath := fmt.Sprintf("%s/sigma_rules", tenzirStorageFolder) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 064301b9..6a580229 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -23,8 +23,8 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/filters" - "github.com/docker/docker/api/types/mount" "github.com/docker/docker/api/types/image" + "github.com/docker/docker/api/types/mount" dockerclient "github.com/docker/docker/client" // This is for automatic removal of certain code :) @@ -105,11 +105,11 @@ var window = shuffle.NewTimeWindow(10 * time.Second) // Images to be autodeployed in the latest version of Shuffle. var autoDeploy = map[string]string{ - "http:1.4.0": "frikky/shuffle:http_1.4.0", - "http:1.3.0": "frikky/shuffle:http_1.3.0", - "shuffle-tools:1.2.0": "frikky/shuffle:shuffle-tools_1.2.0", - "shuffle-subflow:1.0.0": "frikky/shuffle:shuffle-subflow_1.0.0", - "shuffle-subflow:1.1.0": "frikky/shuffle:shuffle-subflow_1.1.0", + "http:1.4.0": "frikky/shuffle:http_1.4.0", + "http:1.3.0": "frikky/shuffle:http_1.3.0", + "shuffle-tools:1.2.0": "frikky/shuffle:shuffle-tools_1.2.0", + "shuffle-subflow:1.0.0": "frikky/shuffle:shuffle-subflow_1.0.0", + "shuffle-subflow:1.1.0": "frikky/shuffle:shuffle-subflow_1.1.0", // "shuffle-tools-fork:1.0.0": "frikky/shuffle:shuffle-tools-fork_1.0.0", } @@ -495,7 +495,7 @@ func deployk8sApp(image string, identifier string, env []string) error { "app.kubernetes.io/name": "shuffle-app", "app.kubernetes.io/instance": name, // "app.kubernetes.io/version": "", - "app.kuvernetes.io/part-of": "shuffle", + "app.kubernetes.io/part-of": "shuffle", "app.kubernetes.io/managed-by": "shuffle-worker", } From 065573811eb7ab80f174118ce08e305df4240c51 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Wed, 19 Feb 2025 10:30:14 +0100 Subject: [PATCH 04/33] fix typo Signed-off-by: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> --- functions/onprem/worker/worker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 6e48043a..d212055f 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -594,7 +594,7 @@ func deployk8sApp(image string, identifier string, env []string) error { }, }, DNSPolicy: corev1.DNSClusterFirst, - ServiceAccountName: appServiceAccountName}, + ServiceAccountName: appServiceAccountName, }, }, }, From 80aa1a3fd2a1b30b60a62fab9b80088736be2d91 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Wed, 19 Feb 2025 11:30:40 +0100 Subject: [PATCH 05/33] update cleanup and label selectors Signed-off-by: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> --- functions/onprem/orborus/orborus.go | 44 ++++++----------------------- functions/onprem/worker/worker.go | 3 +- 2 files changed, 11 insertions(+), 36 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index bd551a99..15baf03c 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -216,8 +216,7 @@ func skipCheckInCleanup(name string) bool { func cleanupExistingNodes(ctx context.Context) error { if isKubernetes == "true" { - // of course, this doesn't clean up "nodes" but - // rather pods, services, roles etc. + // Cleanup all workers created by orborus and all apps created by workers. if kubernetesNamespace == "" { kubernetesNamespace = "default" @@ -229,62 +228,38 @@ func cleanupExistingNodes(ctx context.Context) error { return err } - // Delete all pods - pods, err := clientset.CoreV1().Pods(kubernetesNamespace).List(context.Background(), metav1.ListOptions{}) - if err != nil { - log.Printf("[ERROR] Failed listing pods: %s", err) - return err - } - - for _, pod := range pods.Items { - // check if pod.Name starts with: - // "backend-", "frontend-", "orborus-", "opensearch-" or "memcached-" - if skipCheckInCleanup(pod.Name) { - continue - } - - err := clientset.CoreV1().Pods(kubernetesNamespace).Delete(context.Background(), pod.Name, metav1.DeleteOptions{}) - if err != nil { - log.Printf("[ERROR] Failed deleting pod %s: %s", pod.Name, err) - } - } - // Delete all services - services, err := clientset.CoreV1().Services(kubernetesNamespace).List(context.Background(), metav1.ListOptions{}) + services, err := clientset.CoreV1().Services(kubernetesNamespace).List(context.Background(), metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name in (shuffle-worker, shuffle-app),app.kubernetes.io/managed-by in (shuffle-orborus, shuffle-worker)", + }) if err != nil { log.Printf("[ERROR] Failed listing services: %s", err) return err } for _, service := range services.Items { - if skipCheckInCleanup(service.Name) { - continue - } - err := clientset.CoreV1().Services(kubernetesNamespace).Delete(context.Background(), service.Name, metav1.DeleteOptions{}) if err != nil { log.Printf("[ERROR] Failed deleting service %s: %s", service.Name, err) } } - deployments, err := clientset.AppsV1().Deployments(kubernetesNamespace).List(context.Background(), metav1.ListOptions{}) + deployments, err := clientset.AppsV1().Deployments(kubernetesNamespace).List(context.Background(), metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name in (shuffle-worker, shuffle-app),app.kubernetes.io/managed-by in (shuffle-orborus, shuffle-worker)", + }) if err != nil { log.Printf("[ERROR] Failed listing deployments: %s", err) return err } for _, deployment := range deployments.Items { - if skipCheckInCleanup(deployment.Name) { - continue - } - err := clientset.AppsV1().Deployments(kubernetesNamespace).Delete(context.Background(), deployment.Name, metav1.DeleteOptions{}) if err != nil { log.Printf("[ERROR] Failed deleting deployment %s: %s", deployment.Name, err) } } - log.Printf("[INFO] Cleaned up all pods and services in namespace %s. Waiting 10 seconds for cleanup to reflect", kubernetesNamespace) + log.Printf("[INFO] Cleaned up all services and deployments in namespace %s. Waiting 10 seconds for cleanup to reflect", kubernetesNamespace) time.Sleep(10 * time.Second) @@ -3546,9 +3521,8 @@ func getRunningWorkers(ctx context.Context, workerTimeout int) int { return 0 } - labelSelector := "app=shuffle-worker" pods, podErr := clientset.CoreV1().Pods(kubernetesNamespace).List(ctx, metav1.ListOptions{ - LabelSelector: labelSelector, + LabelSelector: "app.kubernetes.io/name=shuffle-worker", }) if podErr != nil { log.Printf("[ERROR] Failed getting running workers: %s", podErr) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 6a580229..9e319796 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -966,7 +966,8 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] func cleanupKubernetesExecution(clientset *kubernetes.Clientset, workflowExecution shuffle.WorkflowExecution, namespace string) error { // workerName := fmt.Sprintf("worker-%s", workflowExecution.ExecutionId) - labelSelector := fmt.Sprintf("app=shuffle-app,executionId=%s", workflowExecution.ExecutionId) + // FIXME: The executionId label is currently not set + labelSelector := fmt.Sprintf("app.kubernetes.io/name=shuffle-app,executionId=%s", workflowExecution.ExecutionId) podList, err := clientset.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{ LabelSelector: labelSelector, From e769418ecf73d77681cfb1a548e03158c9d4da51 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Wed, 26 Feb 2025 09:04:53 +0100 Subject: [PATCH 06/33] add legacy labels Signed-off-by: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> --- functions/onprem/orborus/orborus.go | 2 ++ functions/onprem/worker/worker.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 15baf03c..3f07f60e 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -1059,6 +1059,8 @@ func deployK8sWorker(image string, identifier string, env []string) error { // "app.kubernetes.io/version": "", "app.kubernetes.io/part-of": "shuffle", "app.kubernetes.io/managed-by": "shuffle-orborus", + // Keep legacy labels for backward compatibility + "container": "shuffle-worker", } matchLabels := map[string]string{ diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 9e319796..ed376797 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -497,6 +497,8 @@ func deployk8sApp(image string, identifier string, env []string) error { // "app.kubernetes.io/version": "", "app.kubernetes.io/part-of": "shuffle", "app.kubernetes.io/managed-by": "shuffle-worker", + // Keep legacy labels for backward compatibility + "app": name, } matchLabels := map[string]string{ From 2681eb923e805be61fcacc23decf7029dc132142 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Wed, 26 Feb 2025 17:18:30 +0100 Subject: [PATCH 07/33] fix helm release workflow Signed-off-by: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> --- .github/workflows/helm-release.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/helm-release.yml b/.github/workflows/helm-release.yml index 67af8ea7..1204a634 100644 --- a/.github/workflows/helm-release.yml +++ b/.github/workflows/helm-release.yml @@ -10,7 +10,7 @@ on: branches: - nightly paths: - - "charts/**" + - "functions/kubernetes/charts/**" permissions: contents: read @@ -32,16 +32,16 @@ jobs: sudo apt-get install helm -y --no-install-recommends - name: Set versions - id: set_versions run: | if [[ ${{ github.event_name }} == 'release' ]]; then - CHART_VERSION="${{ github.event.release.tag_name }}" APP_VERSION="${{ github.event.release.tag_name }}" + CHART_VERSION="${{ github.event.release.tag_name }}" else - CHART_VERSION="0.0.0-nightly-untagged-latest" APP_VERSION="nightly" + CHART_VERSION="0.0.0-nightly-untagged-latest" fi + echo "APP_VERSION set to ${APP_VERSION}" echo "CHART_VERSION set to ${CHART_VERSION}. Validating..." # https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string @@ -54,17 +54,17 @@ jobs: exit 1; fi - echo "CHART_VERSION=${CHART_VERSION}" >> $GITHUB_OUTPUT - echo "APP_VERSION=${APP_VERSION}" >> $GITHUB_OUTPUT + echo "CHART_VERSION=${CHART_VERSION}" >> "$GITHUB_ENV" + echo "APP_VERSION=${APP_VERSION}" >> "$GITHUB_ENV" - name: Update helm dependencies - run: helm dependency update ./charts/shuffle + run: helm dependency update ./functions/kubernetes/charts/shuffle - name: Package Helm chart - run: helm package ./charts/shuffle --version "${CHART_VERSION}" --app-version="${APP_VERSION}" --destination ./charts + run: helm package ./functions/kubernetes/charts/shuffle --version "${CHART_VERSION}" --app-version="${APP_VERSION}" --destination ./functions/kubernetes/charts - name: Login to OCI registry (ghcr.io) run: helm registry login ghcr.io --username ${{ github.actor }} --password ${{ secrets.GITHUB_TOKEN }} - name: Push helm chart - run: helm push ./charts/shuffle-*.tgz oci://ghcr.io/shuffle/shuffle/charts + run: helm push ./functions/kubernetes/charts/shuffle-*.tgz oci://ghcr.io/shuffle/charts From 3d02907947cb1e9bfa507a5c3ed95fb81e02e441 Mon Sep 17 00:00:00 2001 From: Piyush Kumar Pradhan Date: Mon, 3 Mar 2025 16:59:11 +0530 Subject: [PATCH 08/33] removed the linting --- frontend/src/defaultCytoscapeStyle.jsx | 31 ++- frontend/src/views/AngularWorkflow.jsx | 305 +++++++++++++++++++++++-- 2 files changed, 317 insertions(+), 19 deletions(-) diff --git a/frontend/src/defaultCytoscapeStyle.jsx b/frontend/src/defaultCytoscapeStyle.jsx index d7a6b68e..723ad938 100644 --- a/frontend/src/defaultCytoscapeStyle.jsx +++ b/frontend/src/defaultCytoscapeStyle.jsx @@ -95,23 +95,42 @@ const data = [ { selector: `node[type="COMMENT"]`, css: { - label: function(element) { - return element.data("label") - }, + label: function (element) { + return element.data("label") + }, shape: "roundrectangle", color: "data(color)", width: "data(width)", height: "data(height)", - padding: "0px", + padding: "5px", margin: "0px", "background-color": "data(backgroundcolor)", "background-image": "data(backgroundimage)", "border-color": "#ffffff", - "text-margin-x": "0px", + "text-margin-x": "data(textMarginX)", + "text-margin-y": "data(textMarginY)", "z-index": 4999, "border-radius": "5px", "background-opacity": "0.5", - "text-wrap": "wrap", + "text-wrap": "wrap", + "text-max-width": "data(width)", + "text-halign": "data(textHalign)", + "text-valign": "data(textValign)" + }, + }, + { + selector: `node[type="RESIZE-HANDLE"]`, + css: { + shape: "ellipse", + width: "8px", + height: "8px", + "border-width": 1, + "border-color": "white", + "z-index": 5002, + "overlay-opacity": 0, + "cursor": "nwse-resize", + "opacity": 0, + "pointer-events": "auto", }, }, { diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 16d5e9d0..37e58463 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1074,6 +1074,7 @@ const AngularWorkflow = (defaultprops) => { // Special multi-workflow edgecase handler for events if (distributedFromParent === "" && suborgWorkflows === []) { } else { + console.log("here"); if (cy !== undefined && cy !== null) { cy.removeListener("select"); cy.removeListener("unselect"); @@ -1118,7 +1119,8 @@ const AngularWorkflow = (defaultprops) => { !el.data("isButton") && !el.data("isDescriptor") && !el.data("isSuggestion") && - el.data("type") !== "COMMENT") { + el.data("type") !== "COMMENT" && + el.data("type") !== "RESIZE-HANDLE") { return true } @@ -2325,6 +2327,7 @@ const AngularWorkflow = (defaultprops) => { var newBranches = []; var newVBranches = []; var newComments = []; + var newResizes = []; for (let cyelementsKey in cyelements) { if (cyelements[cyelementsKey].data === undefined) { continue; @@ -2509,7 +2512,40 @@ const AngularWorkflow = (defaultprops) => { //console.log(curworkflowComment) newComments.push(curworkflowComment); - } else { + } else if (type === "RESIZE-HANDLE") { + if (useworkflow.resizes === undefined || useworkflow.resizes === null) { + useworkflow.resizes = []; + } + + var curworkflowResize = useworkflow.resizes.find( + (a) => a.id === cyelements[cyelementsKey].data()["id"] + ); + + if (curworkflowResize === undefined) { + curworkflowResize = cyelements[cyelementsKey].data(); + } + + // Ensure width and height are properly parsed + const parsedHeight = parseInt(curworkflowResize["height"]); + if (!isNaN(parsedHeight)) { + curworkflowResize.height = parsedHeight; + } else { + curworkflowResize.height = 150; // Default value if parsing fails + } + + const parsedWidth = parseInt(curworkflowResize["width"]); + if (!isNaN(parsedWidth)) { + curworkflowResize.width = parsedWidth; + } else { + curworkflowResize.width = 200; // Default value if parsing fails + } + + // Update position from Cytoscape + curworkflowResize.position = cyelements[cyelementsKey].position(); + + newResizes.push(curworkflowResize); + } + else { toast("No handler for type: " + type); } } @@ -6522,6 +6558,21 @@ const AngularWorkflow = (defaultprops) => { } setSelectedComment(data); + } else if (data.type === "RESIZE-HANDLE") { + + const parentNode = cy.getElementById(data.attachedTo); + if (parentNode) { + console.log("Resizing parent node:", parentNode); + + // Get the parent node's data + const parentData = parentNode.data(); + if (parentData?.type === "COMMENT") { + + // Set the parent node's data as the selected comment + setSelectedComment(parentData); + } + } + return; // Exit after handling the resize handle } else { toast("Can't handle node type " + data.type); return; @@ -9650,7 +9701,8 @@ const AngularWorkflow = (defaultprops) => { !el.data("isButton") && !el.data("isDescriptor") && !el.data("isSuggestion") && - el.data("type") !== "COMMENT") { + el.data("type") !== "COMMENT" && + el.data("type") !== "RESIZE-HANDLE") { return true } @@ -9732,6 +9784,27 @@ const AngularWorkflow = (defaultprops) => { cy.on("mouseover", "node", (e) => onNodeHover(e)); cy.on("mouseout", "node", (e) => onNodeHoverOut(e)); + cy.on("mouseover", "node[type='RESIZE-HANDLE']", (e) => { + const nodeId = e.target.id(); + + // Check the node ID to determine the cursor style based on position + if (nodeId.includes("bottom-right")) { + cy.container().style.cursor = "nwse-resize"; // Bottom-right resize cursor + } else if (nodeId.includes("top-right")) { + cy.container().style.cursor = "nesw-resize"; // Top-right resize cursor + } else if (nodeId.includes("top-left")) { + cy.container().style.cursor = "nwse-resize"; // Top-left resize cursor + } else if (nodeId.includes("bottom-left")) { + cy.container().style.cursor = "nesw-resize"; // Bottom-left resize cursor + } else { + cy.container().style.cursor = "default"; // Default cursor for other cases + } + }); + + cy.on("mouseout", "node[type='RESIZE-HANDLE']", (e) => { + cy.container().style.cursor = ""; + }); + // Handles dragging cy.on("drag", "node", (e) => onNodeDrag(e, selectedAction)); cy.on("free", "node", (e) => onNodeDragStop(e, selectedAction)); @@ -15699,6 +15772,79 @@ const AngularWorkflow = (defaultprops) => { setSelectedComment(selectedComment); }} /> +
+
+
Justify
+ +
+
+
Align
+ +
+
Height
@@ -15803,7 +15949,6 @@ const AngularWorkflow = (defaultprops) => { defaultValue={selectedComment["backgroundimage"]} onChange={(event) => { selectedComment.backgroundimage = event.target.value; - console.log("Comment: ", selectedComment) setSelectedComment(selectedComment); }} /> @@ -19696,12 +19841,115 @@ const AngularWorkflow = (defaultprops) => { ); }; + const setupResizeHandlers = (cy, nodeId) => { + let height; + let width; + let resizeTimeout; + + cy.on("drag", ".resize-handle", (event) => { + if (resizeTimeout) return; + + resizeTimeout = setTimeout(() => { + resizeTimeout = null; + const handle = event.target; + const parent = cy.$(`#${nodeId}`); // Fetch the main node directly + + // Check if the parent node exists + if (!parent || parent.empty()) { + console.warn(`Parent node (${nodeId}) not found.`); + return; + } + + if (!handle?.position()) { + console.warn(`Handle position is undefined for ${handle.id()}`); + return; + } + + const handlePos = handle.position(); + const parentPos = parent.position(); + + if (!parentPos) { + console.warn(`Parent position is undefined for ${nodeId}`); + return; + } + + // Calculate new width & height based on handle movement + const newWidth = Math.abs(handlePos.x - parentPos.x) * 2; + const newHeight = Math.abs(handlePos.y - parentPos.y) * 2; + + // Apply min/max constraints + const constrainedWidth = Math.max(100, Math.min(newWidth, 500)); + const constrainedHeight = Math.max(50, Math.min(newHeight, 300)); + + // Update node size + parent.style({ + width: constrainedWidth, + height: constrainedHeight, + }); + + // Store dimensions for state update on drag end + height = Math.floor(constrainedHeight); + width = Math.floor(constrainedWidth); + + // Update handle positions + cy.$(".resize-handle").forEach((corner) => { + if (!corner?.id() || !corner.position()) return; + + const { x, y } = parent.position(); + const offsetX = corner.id().includes("left") ? -constrainedWidth / 2 : constrainedWidth / 2; + const offsetY = corner.id().includes("top") ? -constrainedHeight / 2 : constrainedHeight / 2; + + corner.position({ x: x + offsetX, y: y + offsetY }); + }); + }, 16); // Throttle to ~60 FPS + }); + + // Update state when resizing ends + cy.on("free", ".resize-handle", (event) => { + const data = event.target.data(); + const parentNode = cy.getElementById(data.attachedTo); + + if (parentNode) { + + parentNode.data({ + ...parentNode.data(), + width: Math.floor(width), + height: Math.floor(height), + }); + setSelectedComment((prev) => ({ + ...prev, + width: Math.floor(width), + height: Math.floor(height), + })); + } + }); + + }; + + + const setupNodeDragHandler = (cy, nodeId) => { + cy.on("drag", `#${nodeId}`, (event) => { + const node = event.target; + const { x, y } = node.position(); + const width = parseFloat(node.style("width")); + const height = parseFloat(node.style("height")); + + // Move resize handles with the node + cy.$(".resize-handle").forEach((corner) => { + const offsetX = corner.id().includes("left") ? -width / 2 : width / 2; + const offsetY = corner.id().includes("top") ? -height / 2 : height / 2; + + corner.position({ x: x + offsetX, y: y + offsetY }); + }); + }); + }; + const addCommentNode = () => { const newId = uuidv4(); - const position = { - x: 300, - y: 300, - }; + const position = { x: 300, y: 300 }; + const width = 250; + const height = 150; + const handleOffset = 10; // Move handles outside the node cy.add({ group: "nodes", @@ -19711,20 +19959,51 @@ const AngularWorkflow = (defaultprops) => { type: "COMMENT", is_valid: true, decorator: true, - width: 250, - height: 150, - position: position, + width, + height, + position, backgroundcolor: "#1f2023", color: "#ffffff", + textHalign: "center", + textValign: "center", + textMarginX: "0px", + textMarginY: "0px", }, - position: position, + position, }); + + // Define corner positions relative to the main node + const corners = [ + { id: `${newId}-top-left`, dx: -width / 2 - handleOffset, dy: -height / 2 - handleOffset }, + { id: `${newId}-top-right`, dx: width / 2 + handleOffset, dy: -height / 2 - handleOffset }, + { id: `${newId}-bottom-left`, dx: -width / 2 - handleOffset, dy: height / 2 + handleOffset }, + { id: `${newId}-bottom-right`, dx: width / 2 + handleOffset, dy: height / 2 + handleOffset }, + ]; + + // Add resize handles **without** the parent property + corners.forEach((corner) => { + cy.add({ + group: "nodes", + data: { + id: corner.id, + type: "RESIZE-HANDLE", + is_valid: true, + attachedTo: newId, + decorator: true, + }, // No parent to avoid edges + position: { x: position.x + corner.dx, y: position.y + corner.dy }, + classes: "resize-handle", + }); + }); + + setupResizeHandlers(cy, newId); + setupNodeDragHandler(cy, newId); }; const RightSideBar = (props) => { var defaultReturn = null - if (Object.getOwnPropertyNames(selectedComment).length > 0) { + if (Object.getOwnPropertyNames(selectedComment).length > 0 && selectedComment.hasOwnProperty('id')) { defaultReturn = } else if (Object.getOwnPropertyNames(selectedTrigger).length > 0) { if (selectedTrigger.trigger_type === undefined) { From 1e1cfec75eaad6011a3211587fc4bfbc83efdedd Mon Sep 17 00:00:00 2001 From: Piyush Kumar Pradhan Date: Mon, 3 Mar 2025 17:08:35 +0530 Subject: [PATCH 09/33] emove debug console logs. --- frontend/src/views/AngularWorkflow.jsx | 1 - 1 file changed, 1 deletion(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 37e58463..fbcf5552 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1074,7 +1074,6 @@ const AngularWorkflow = (defaultprops) => { // Special multi-workflow edgecase handler for events if (distributedFromParent === "" && suborgWorkflows === []) { } else { - console.log("here"); if (cy !== undefined && cy !== null) { cy.removeListener("select"); cy.removeListener("unselect"); From bf160421466db7cb685dc65988bc8b9b6b27e8f1 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Thu, 6 Mar 2025 09:47:05 +0100 Subject: [PATCH 10/33] use tag 2.0.0 with a default helm installation Signed-off-by: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> --- functions/kubernetes/charts/shuffle/values.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/values.yaml b/functions/kubernetes/charts/shuffle/values.yaml index 35492f94..fb580513 100644 --- a/functions/kubernetes/charts/shuffle/values.yaml +++ b/functions/kubernetes/charts/shuffle/values.yaml @@ -104,7 +104,7 @@ backend: image: registry: ghcr.io repository: shuffle/shuffle-backend - tag: nightly + tag: 2.0.0 digest: "" ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' @@ -573,7 +573,7 @@ frontend: image: registry: ghcr.io repository: shuffle/shuffle-frontend - tag: nightly + tag: 2.0.0 digest: "" ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' @@ -978,7 +978,7 @@ orborus: image: registry: ghcr.io repository: shuffle/shuffle-orborus - tag: nightly + tag: 2.0.0 digest: "" ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' @@ -1383,7 +1383,7 @@ worker: image: registry: ghcr.io repository: shuffle/shuffle-worker - tag: nightly + tag: 2.0.0 digest: "" ## ServiceAccount configuration From 0a370d60ceaaa2ae7d075a2135fe50c1dd5fc740 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Thu, 6 Mar 2025 12:57:49 +0100 Subject: [PATCH 11/33] raise chart and appVersion Signed-off-by: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> --- functions/extensions/k8s/shuffle/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/functions/extensions/k8s/shuffle/Chart.yaml b/functions/extensions/k8s/shuffle/Chart.yaml index bd42981f..eb53a0b5 100755 --- a/functions/extensions/k8s/shuffle/Chart.yaml +++ b/functions/extensions/k8s/shuffle/Chart.yaml @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.2.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "1.16.0" +appVersion: "2.0.0" From 41f623dd396e8de1fe8c3f439da1b005bd02bc51 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Tue, 18 Mar 2025 10:14:33 +0100 Subject: [PATCH 12/33] run as user 1001 by default Signed-off-by: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> --- functions/kubernetes/charts/shuffle/README.md | 13 +++++++------ .../charts/shuffle/values.schema.json | 12 ++++++------ functions/kubernetes/charts/shuffle/values.yaml | 17 +++++++++-------- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/README.md b/functions/kubernetes/charts/shuffle/README.md index 495d2690..2ae2926f 100644 --- a/functions/kubernetes/charts/shuffle/README.md +++ b/functions/kubernetes/charts/shuffle/README.md @@ -184,8 +184,8 @@ SHUFFLE_ENCRYPTION_MODIFIER: "MyShuffleEncryptionModifier" | `backend.podSecurityContext.fsGroup` | Set fsGroup in backend pods' Security Context | `1001` | | `backend.containerSecurityContext.enabled` | Enabled backend container' Security Context | `true` | | `backend.containerSecurityContext.seLinuxOptions` | Set SELinux options in backend container | `{}` | -| `backend.containerSecurityContext.runAsUser` | Set runAsUser in backend container' Security Context | `1000` | -| `backend.containerSecurityContext.runAsGroup` | Set runAsGroup in backend container' Security Context | `1000` | +| `backend.containerSecurityContext.runAsUser` | Set runAsUser in backend container' Security Context | `1001` | +| `backend.containerSecurityContext.runAsGroup` | Set runAsGroup in backend container' Security Context | `1001` | | `backend.containerSecurityContext.runAsNonRoot` | Set runAsNonRoot in backend container' Security Context | `true` | | `backend.containerSecurityContext.readOnlyRootFilesystem` | Set readOnlyRootFilesystem in backend container' Security Context | `true` | | `backend.containerSecurityContext.privileged` | Set privileged in backend container' Security Context | `false` | @@ -302,8 +302,8 @@ SHUFFLE_ENCRYPTION_MODIFIER: "MyShuffleEncryptionModifier" | `frontend.podSecurityContext.fsGroup` | Set fsGroup in frontend pods' Security Context | `1001` | | `frontend.containerSecurityContext.enabled` | Enabled frontend container' Security Context | `false` | | `frontend.containerSecurityContext.seLinuxOptions` | Set SELinux options in frontend container | `{}` | -| `frontend.containerSecurityContext.runAsUser` | Set runAsUser in frontend container' Security Context | `101` | -| `frontend.containerSecurityContext.runAsGroup` | Set runAsGroup in frontend container' Security Context | `101` | +| `frontend.containerSecurityContext.runAsUser` | Set runAsUser in frontend container' Security Context | `1001` | +| `frontend.containerSecurityContext.runAsGroup` | Set runAsGroup in frontend container' Security Context | `1001` | | `frontend.containerSecurityContext.runAsNonRoot` | Set runAsNonRoot in frontend container' Security Context | `true` | | `frontend.containerSecurityContext.readOnlyRootFilesystem` | Set readOnlyRootFilesystem in frontend container' Security Context | `true` | | `frontend.containerSecurityContext.privileged` | Set privileged in frontend container' Security Context | `false` | @@ -409,8 +409,8 @@ SHUFFLE_ENCRYPTION_MODIFIER: "MyShuffleEncryptionModifier" | `orborus.podSecurityContext.fsGroup` | Set fsGroup in orborus pods' Security Context | `1001` | | `orborus.containerSecurityContext.enabled` | Enabled orborus container' Security Context | `true` | | `orborus.containerSecurityContext.seLinuxOptions` | Set SELinux options in orborus container | `{}` | -| `orborus.containerSecurityContext.runAsUser` | Set runAsUser in orborus container' Security Context | `101` | -| `orborus.containerSecurityContext.runAsGroup` | Set runAsGroup in orborus container' Security Context | `101` | +| `orborus.containerSecurityContext.runAsUser` | Set runAsUser in orborus container' Security Context | `1001` | +| `orborus.containerSecurityContext.runAsGroup` | Set runAsGroup in orborus container' Security Context | `1001` | | `orborus.containerSecurityContext.runAsNonRoot` | Set runAsNonRoot in orborus container' Security Context | `true` | | `orborus.containerSecurityContext.readOnlyRootFilesystem` | Set readOnlyRootFilesystem in orborus container' Security Context | `true` | | `orborus.containerSecurityContext.privileged` | Set privileged in orborus container' Security Context | `false` | @@ -605,3 +605,4 @@ SHUFFLE_ENCRYPTION_MODIFIER: "MyShuffleEncryptionModifier" ### Other Parameters + diff --git a/functions/kubernetes/charts/shuffle/values.schema.json b/functions/kubernetes/charts/shuffle/values.schema.json index 7a85b9fa..609e37df 100644 --- a/functions/kubernetes/charts/shuffle/values.schema.json +++ b/functions/kubernetes/charts/shuffle/values.schema.json @@ -367,12 +367,12 @@ "runAsUser": { "type": "number", "description": "Set runAsUser in backend container' Security Context", - "default": 1000 + "default": 1001 }, "runAsGroup": { "type": "number", "description": "Set runAsGroup in backend container' Security Context", - "default": 1000 + "default": 1001 }, "runAsNonRoot": { "type": "boolean", @@ -1056,12 +1056,12 @@ "runAsUser": { "type": "number", "description": "Set runAsUser in frontend container' Security Context", - "default": 101 + "default": 1001 }, "runAsGroup": { "type": "number", "description": "Set runAsGroup in frontend container' Security Context", - "default": 101 + "default": 1001 }, "runAsNonRoot": { "type": "boolean", @@ -1675,12 +1675,12 @@ "runAsUser": { "type": "number", "description": "Set runAsUser in orborus container' Security Context", - "default": 101 + "default": 1001 }, "runAsGroup": { "type": "number", "description": "Set runAsGroup in orborus container' Security Context", - "default": 101 + "default": 1001 }, "runAsNonRoot": { "type": "boolean", diff --git a/functions/kubernetes/charts/shuffle/values.yaml b/functions/kubernetes/charts/shuffle/values.yaml index 35492f94..221b0bb5 100644 --- a/functions/kubernetes/charts/shuffle/values.yaml +++ b/functions/kubernetes/charts/shuffle/values.yaml @@ -235,8 +235,8 @@ backend: containerSecurityContext: enabled: true seLinuxOptions: {} - runAsUser: 1000 - runAsGroup: 1000 + runAsUser: 1001 + runAsGroup: 1001 runAsNonRoot: true readOnlyRootFilesystem: true privileged: false @@ -683,11 +683,12 @@ frontend: ## @param frontend.podSecurityContext.fsGroup Set fsGroup in frontend pods' Security Context ## podSecurityContext: - enabled: false + enabled: false # The default shuffle frontend image does not support running as non-root, because /etc/nginx/nginx.conf is written on startup fsGroupChangePolicy: Always sysctls: [] supplementalGroups: [] fsGroup: 1001 + ## Configure Container Security Context ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container ## @param frontend.containerSecurityContext.enabled Enabled frontend container' Security Context @@ -702,10 +703,10 @@ frontend: ## @param frontend.containerSecurityContext.seccompProfile.type Set seccomp profile in frontend container ## containerSecurityContext: - enabled: false + enabled: false # The default shuffle frontend image does not support running as non-root, because /etc/nginx/nginx.conf is written on startup seLinuxOptions: {} - runAsUser: 101 - runAsGroup: 101 + runAsUser: 1001 + runAsGroup: 1001 runAsNonRoot: true readOnlyRootFilesystem: true privileged: false @@ -1107,8 +1108,8 @@ orborus: containerSecurityContext: enabled: true seLinuxOptions: {} - runAsUser: 101 - runAsGroup: 101 + runAsUser: 1001 + runAsGroup: 1001 runAsNonRoot: true readOnlyRootFilesystem: true privileged: false From 9b9e2e1048d2a9fd2f649a6831d48bd64e70baad Mon Sep 17 00:00:00 2001 From: Frikky Date: Mon, 24 Mar 2025 10:01:10 +0100 Subject: [PATCH 13/33] Sync up with speedups done for workflow runs on cloud and removed old unused code --- backend/go-app/go.mod | 2 +- backend/go-app/go.sum | 4 +- backend/go-app/walkoff.go | 506 +------------------------------- functions/onprem/orborus/go.mod | 2 +- functions/onprem/orborus/go.sum | 4 +- functions/onprem/worker/go.mod | 2 +- functions/onprem/worker/go.sum | 4 +- 7 files changed, 11 insertions(+), 513 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index a8e0b107..1a07e775 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -20,7 +20,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.18 + github.com/shuffle/shuffle-shared v0.8.19 golang.org/x/crypto v0.36.0 google.golang.org/api v0.176.1 google.golang.org/grpc v1.68.1 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index d4b660fc..1214c85c 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -333,8 +333,8 @@ github.com/sendgrid/sendgrid-go v3.14.0+incompatible h1:KDSasSTktAqMJCYClHVE94Fc github.com/sendgrid/sendgrid-go v3.14.0+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.18 h1:7f7cV+P2pr/g44i+AI8P0UheEe8oG8O2V8loBfe9YSw= -github.com/shuffle/shuffle-shared v0.8.18/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= +github.com/shuffle/shuffle-shared v0.8.19 h1:HXqU62sPhVzv9MeJnA5ZpPYwwbitVz1EtMMABbY3t74= +github.com/shuffle/shuffle-shared v0.8.19/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 1878d1b0..434b5a42 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -1056,7 +1056,7 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request ctx := context.Background() if workflow.ID == "" || workflow.ID != id { - tmpworkflow, err := shuffle.GetWorkflow(ctx, id) + tmpworkflow, err := shuffle.GetWorkflow(ctx, id, true) if err != nil { //log.Printf("[WARNING] Failed getting the workflow locally (execution setup): %s", err) return shuffle.WorkflowExecution{}, "Failed getting workflow", err @@ -1141,507 +1141,6 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request return shuffle.WorkflowExecution{}, "Failed unmarshal during execution", err } - /* - makeNew := true - start, startok := request.URL.Query()["start"] - if request.Method == "POST" { - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Printf("[ERROR] Failed request POST read: %s", err) - return shuffle.WorkflowExecution{}, "Failed getting body", err - } - - // This one doesn't really matter. - log.Printf("[INFO] Running POST execution with body of length %d for workflow %s", len(string(body)), workflowExecution.Workflow.ID) - - if len(body) >= 4 { - if body[0] == 34 && body[len(body)-1] == 34 { - body = body[1 : len(body)-1] - } - if body[0] == 34 && body[len(body)-1] == 34 { - body = body[1 : len(body)-1] - } - } - - sourceAuth, sourceAuthOk := request.URL.Query()["source_auth"] - if sourceAuthOk { - //log.Printf("\n\n\nSETTING SOURCE WORKFLOW AUTH TO %s!!!\n\n\n", sourceAuth[0]) - workflowExecution.ExecutionSourceAuth = sourceAuth[0] - } else { - //log.Printf("Did NOT get source workflow") - } - - sourceNode, sourceNodeOk := request.URL.Query()["source_node"] - if sourceNodeOk { - //log.Printf("\n\n\nSETTING SOURCE WORKFLOW NODE TO %s!!!\n\n\n", sourceNode[0]) - workflowExecution.ExecutionSourceNode = sourceNode[0] - } else { - //log.Printf("Did NOT get source workflow") - } - - //workflowExecution.ExecutionSource = "default" - sourceWorkflow, sourceWorkflowOk := request.URL.Query()["source_workflow"] - if sourceWorkflowOk { - //log.Printf("Got source workflow %s", sourceWorkflow) - workflowExecution.ExecutionSource = sourceWorkflow[0] - } else { - //log.Printf("Did NOT get source workflow") - } - - sourceExecution, sourceExecutionOk := request.URL.Query()["source_execution"] - if sourceExecutionOk { - //log.Printf("[INFO] Got source execution%s", sourceExecution) - workflowExecution.ExecutionParent = sourceExecution[0] - } else { - //log.Printf("Did NOT get source execution") - } - - if len(string(body)) < 50 { - //log.Println(body) - // String in string - //log.Println(body) - - //if string(body)[0] == "\"" && string(body)[string(body) - log.Printf("[DEBUG] Body: %s", string(body)) - } - - var execution shuffle.ExecutionRequest - err = json.Unmarshal(body, &execution) - if err != nil { - log.Printf("[WARNING] Failed execution POST unmarshalling for execution %s - continuing anyway: %s", execution.ExecutionId, err) - //return shuffle.WorkflowExecution{}, "", err - } - - if execution.Start == "" && len(body) > 0 { - execution.ExecutionArgument = string(body) - } - - // FIXME - this should have "execution_argument" from executeWorkflow frontend - //log.Printf("EXEC: %#v", execution) - if len(execution.ExecutionArgument) > 0 { - workflowExecution.ExecutionArgument = execution.ExecutionArgument - } - - if len(execution.ExecutionSource) > 0 { - workflowExecution.ExecutionSource = execution.ExecutionSource - } - - //log.Printf("Execution data: %#v", execution) - if len(execution.Start) == 36 && len(workflow.Actions) > 0 { - log.Printf("[INFO] Should start execution on node %s", execution.Start) - workflowExecution.Start = execution.Start - - found := false - for _, action := range workflow.Actions { - if action.ID == execution.Start { - found = true - break - } - } - - if !found { - log.Printf("[ERROR] Action %s was NOT found! Exiting execution.", execution.Start) - return shuffle.WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", workflow.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", workflow.Start)) - } - } else if len(execution.Start) > 0 { - //log.Printf("[INFO] !") - //log.Printf("[ERROR] START ACTION %s IS WRONG ID LENGTH %d!", execution.Start, len(execution.Start)) - //return shuffle.WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", execution.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", execution.Start)) - } - - if len(execution.ExecutionId) == 36 { - workflowExecution.ExecutionId = execution.ExecutionId - } else { - sessionToken := uuid.NewV4() - workflowExecution.ExecutionId = sessionToken.String() - } - } else { - // Check for parameters of start and ExecutionId - // This is mostly used for user input trigger - - answer, answerok := request.URL.Query()["answer"] - referenceId, referenceok := request.URL.Query()["reference_execution"] - if answerok && referenceok && len(answer) > 0 && len(referenceId) > 0 { - // If answer is false, reference execution with result - log.Printf("[INFO] Answer is OK AND reference is OK!") - if answer[0] == "false" { - log.Printf("Should update reference and return, no need for further execution!") - - // Get the reference execution - oldExecution, err := shuffle.GetWorkflowExecution(ctx, referenceId[0]) - if err != nil { - log.Printf("Failed getting execution (execution) %s: %s", referenceId[0], err) - return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed getting execution ID %s because it doesn't exist.", referenceId[0]), err - } - - if oldExecution.Workflow.ID != id { - log.Println("Wrong workflowid!") - return shuffle.WorkflowExecution{}, fmt.Sprintf("Bad ID %s", referenceId), errors.New("Bad ID") - } - - newResults := []shuffle.ActionResult{} - //log.Printf("%#v", oldExecution.Results) - for _, result := range oldExecution.Results { - log.Printf("%s - %s", result.Action.ID, start[0]) - if result.Action.ID == start[0] { - note, noteok := request.URL.Query()["note"] - if noteok && len(note) > 0 { - result.Result = fmt.Sprintf("User note: %s", note[0]) - } else { - result.Result = fmt.Sprintf("User clicked %s", answer[0]) - } - - // Stopping the whole thing - result.CompletedAt = int64(time.Now().Unix()) - result.Status = "ABORTED" - oldExecution.Status = result.Status - oldExecution.Result = result.Result - oldExecution.LastNode = result.Action.ID - } - - newResults = append(newResults, result) - } - - oldExecution.Results = newResults - err = shuffle.SetWorkflowExecution(ctx, *oldExecution, true) - if err != nil { - log.Printf("Error saving workflow execution actionresult setting: %s", err) - return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed setting workflowexecution actionresult in execution: %s", err), err - } - - return shuffle.WorkflowExecution{}, "", nil - } - } - - if referenceok { - log.Printf("Handling an old execution continuation!") - // Will use the old name, but still continue with NEW ID - oldExecution, err := shuffle.GetWorkflowExecution(ctx, referenceId[0]) - if err != nil { - log.Printf("Failed getting execution (execution) %s: %s", referenceId[0], err) - return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed getting execution ID %s because it doesn't exist.", referenceId[0]), err - } - - workflowExecution = *oldExecution - } - - if len(workflowExecution.ExecutionId) == 0 { - sessionToken := uuid.NewV4() - workflowExecution.ExecutionId = sessionToken.String() - } else { - log.Printf("Using the same executionId as before: %s", workflowExecution.ExecutionId) - makeNew = false - } - - // Don't override workflow defaults - } - - if startok { - //log.Printf("\n\n[INFO] Setting start to %s based on query!\n\n", start[0]) - //workflowExecution.Workflow.Start = start[0] - workflowExecution.Start = start[0] - } - - // FIXME - regex uuid, and check if already exists? - if len(workflowExecution.ExecutionId) != 36 { - log.Printf("Invalid uuid: %s", workflowExecution.ExecutionId) - return shuffle.WorkflowExecution{}, "Invalid uuid", err - } - - // FIXME - find owner of workflow - // FIXME - get the actual workflow itself and build the request - // MAYBE: Don't send the workflow within the pubsub, as this requires more data to be sent - // Check if a worker already exists for company, else run one with: - // locations, project IDs and subscription names - - // When app is executed: - // Should update with status execution (somewhere), which will trigger the next node - // IF action.type == internal, we need the internal watcher to be running and executing - // This essentially means the WORKER has to be the responsible party for new actions in the INTERNAL landscape - // Results are ALWAYS posted back to cloud@execution_id? - if makeNew { - workflowExecution.Type = "workflow" - //workflowExecution.Stream = "tmp" - //workflowExecution.WorkflowQueue = "tmp" - //workflowExecution.SubscriptionNameNodestream = "testcompany-nodestream" - //workflowExecution.Locations = []string{"europe-west2"} - workflowExecution.ProjectId = gceProject - workflowExecution.WorkflowId = workflow.ID - workflowExecution.StartedAt = int64(time.Now().Unix()) - workflowExecution.CompletedAt = 0 - workflowExecution.Authorization = uuid.NewV4().String() - - // Status for the entire workflow. - workflowExecution.Status = "EXECUTING" - } - - if len(workflowExecution.ExecutionSource) == 0 { - log.Printf("[INFO] No execution source (trigger) specified. Setting to default") - workflowExecution.ExecutionSource = "default" - } else { - log.Printf("[INFO] Execution source is %s for execution ID %s in workflow %s", workflowExecution.ExecutionSource, workflowExecution.ExecutionId, workflowExecution.Workflow.ID) - } - - workflowExecution.ExecutionVariables = workflow.ExecutionVariables - if len(workflowExecution.Start) == 0 && len(workflowExecution.Workflow.Start) > 0 { - workflowExecution.Start = workflowExecution.Workflow.Start - } - - startnodeFound := false - newStartnode := "" - for _, item := range workflowExecution.Workflow.Actions { - if item.ID == workflowExecution.Start { - startnodeFound = true - } - - if item.IsStartNode { - newStartnode = item.ID - } - } - - if !startnodeFound { - log.Printf("[INFO] Couldn't find startnode %s. Remapping to %#v", workflowExecution.Start, newStartnode) - - if len(newStartnode) > 0 { - workflowExecution.Start = newStartnode - } else { - return shuffle.WorkflowExecution{}, fmt.Sprintf("Startnode couldn't be found"), errors.New("Startnode isn't defined in this workflow..") - } - } - - childNodes := shuffle.FindChildNodes(workflowExecution.Workflow, workflowExecution.Start, []string{}, []string{}) - - startFound := false - newActions := []shuffle.Action{} - defaultResults := []shuffle.ActionResult{} - - for _, action := range workflowExecution.Workflow.Actions { - //action.LargeImage = "" - if action.ID == workflowExecution.Start { - startFound = true - } - //log.Println(action.Environment) - - if action.Environment == "" { - return shuffle.WorkflowExecution{}, fmt.Sprintf("Environment is not defined for %s", action.Name), errors.New("Environment not defined!") - } - - action.LargeImage = "" - if len(action.Label) == 0 { - action.Label = action.ID - } - //log.Printf("LABEL: %s", action.Label) - newActions = append(newActions, action) - - // If the node is NOT found, it's supposed to be set to SKIPPED, - // as it's not a childnode of the startnode - // This is a configuration item for the workflow itself. - if len(workflowExecution.Results) > 0 { - defaultResults = []shuffle.ActionResult{} - for _, result := range workflowExecution.Results { - if result.Status == "WAITING" { - result.Status = "FINISHED" - result.Result = "Continuing" - } - - defaultResults = append(defaultResults, result) - } - } else if len(workflowExecution.Results) == 0 && !workflowExecution.Workflow.Configuration.StartFromTop { - found := false - for _, nodeId := range childNodes { - if nodeId == action.ID { - //log.Printf("Found %s", action.ID) - found = true - } - } - - if !found { - if action.ID == workflowExecution.Start { - continue - } - - //log.Printf("[WARNING] Set %s to SKIPPED as it's NOT a childnode of the startnode.", action.ID) - curaction := shuffle.Action{ - AppName: action.AppName, - AppVersion: action.AppVersion, - Label: action.Label, - Name: action.Name, - ID: action.ID, - } - //action - //curaction.Parameters = [] - defaultResults = append(defaultResults, shuffle.ActionResult{ - Action: curaction, - ExecutionId: workflowExecution.ExecutionId, - Authorization: workflowExecution.Authorization, - Result: "Skipped because it's not under the startnode", - StartedAt: 0, - CompletedAt: 0, - Status: "SKIPPED", - }) - } - } - } - - removeTriggers := []string{} - for triggerIndex, trigger := range workflowExecution.Workflow.Triggers { - //log.Printf("[INFO] ID: %s vs %s", trigger.ID, workflowExecution.Start) - if trigger.ID == workflowExecution.Start { - if trigger.AppName == "User Input" { - startFound = true - break - } - } - - if trigger.AppName == "User Input" || trigger.AppName == "Shuffle Workflow" { - found := false - for _, node := range childNodes { - if node == trigger.ID { - found = true - break - } - } - - if !found { - //log.Printf("SHOULD SET TRIGGER %s TO BE SKIPPED", trigger.ID) - - curaction := shuffle.Action{ - AppName: "shuffle-subflow", - AppVersion: trigger.AppVersion, - Label: trigger.Label, - Name: trigger.Name, - ID: trigger.ID, - } - - defaultResults = append(defaultResults, shuffle.ActionResult{ - Action: curaction, - ExecutionId: workflowExecution.ExecutionId, - Authorization: workflowExecution.Authorization, - Result: "Skipped because it's not under the startnode", - StartedAt: 0, - CompletedAt: 0, - Status: "SKIPPED", - }) - } else { - // Replaces trigger with the subflow - //if trigger.AppName == "Shuffle Workflow" { - // replaceActions := false - // workflowAction := "" - // for _, param := range trigger.Parameters { - // if param.Name == "argument" && !strings.Contains(param.Value, ".#") { - // replaceActions = true - // } - - // if param.Name == "startnode" { - // workflowAction = param.Value - // } - // } - - // if replaceActions { - // replacementNodes, newBranches, lastnode := shuffle.GetReplacementNodes(ctx, workflowExecution, trigger, trigger.Label) - // log.Printf("REPLACEMENTS: %d, %d", len(replacementNodes), len(newBranches)) - // if len(replacementNodes) > 0 { - // for _, action := range replacementNodes { - // found := false - - // for subActionIndex, subaction := range newActions { - // if subaction.ID == action.ID { - // found = true - // //newActions[subActionIndex].Name = action.Name - // newActions[subActionIndex].Label = action.Label - // break - // } - // } - - // if !found { - // action.SubAction = true - // newActions = append(newActions, action) - // } - - // // Check if it's already set to have a value - // for resultIndex, result := range defaultResults { - // if result.Action.ID == action.ID { - // defaultResults = append(defaultResults[:resultIndex], defaultResults[resultIndex+1:]...) - // break - // } - // } - // } - - // for _, branch := range newBranches { - // workflowExecution.Workflow.Branches = append(workflowExecution.Workflow.Branches, branch) - // } - - // // Append branches: - // // parent -> new inner node (FIRST one) - // for branchIndex, branch := range workflowExecution.Workflow.Branches { - // if branch.DestinationID == trigger.ID { - // log.Printf("REPLACE DESTINATION WITH %s!!", workflowAction) - // workflowExecution.Workflow.Branches[branchIndex].DestinationID = workflowAction - // } - - // if branch.SourceID == trigger.ID { - // log.Printf("REPLACE SOURCE WITH LASTNODE %s!!", lastnode) - // workflowExecution.Workflow.Branches[branchIndex].SourceID = lastnode - // } - // } - - // // Remove the trigger - // removeTriggers = append(removeTriggers, workflowExecution.Workflow.Triggers[triggerIndex].ID) - // } - - // log.Printf("NEW ACTION LENGTH %d, RESULT: %d, Triggers: %d, BRANCHES: %d", len(newActions), len(defaultResults), len(workflowExecution.Workflow.Triggers), len(workflowExecution.Workflow.Branches)) - // } - //} - _ = triggerIndex - } - } - } - - //newTriggers := []shuffle.Trigger{} - //for _, trigger := range workflowExecution.Workflow.Triggers { - // found := false - // for _, triggerId := range removeTriggers { - // if trigger.ID == triggerId { - // found = true - // break - // } - // } - - // if found { - // log.Printf("[WARNING] Removed trigger %s during execution", trigger.ID) - // continue - // } - - // newTriggers = append(newTriggers, trigger) - //} - //workflowExecution.Workflow.Triggers = newTriggers - - if !startFound { - if len(workflowExecution.Start) == 0 && len(workflowExecution.Workflow.Start) > 0 { - workflowExecution.Start = workflow.Start - } else if len(workflowExecution.Workflow.Actions) > 0 { - workflowExecution.Start = workflowExecution.Workflow.Actions[0].ID - } else { - log.Printf("[ERROR] Startnode %s doesn't exist!!", workflowExecution.Start) - return shuffle.WorkflowExecution{}, fmt.Sprintf("Workflow action %s doesn't exist in workflow", workflowExecution.Start), errors.New(fmt.Sprintf(`Workflow start node "%s" doesn't exist. Exiting!`, workflowExecution.Start)) - } - } - - //log.Printf("EXECUTION START: %s", workflowExecution.Start) - - // Verification for execution environments - workflowExecution.Results = defaultResults - workflowExecution.Workflow.Actions = newActions - onpremExecution := true - _ = onpremExecution - environments := []string{} - - if len(workflowExecution.ExecutionOrg) == 0 && len(workflow.ExecutingOrg.Id) > 0 { - workflowExecution.ExecutionOrg = workflow.ExecutingOrg.Id - } - */ - - //workflowExecution, execInfo, _, workflowExecErr := shuffle.PrepareWorkflowExecution(ctx, workflow, request, int64(maxExecutionDepth)) err = shuffle.SetWorkflowExecution(ctx, workflowExecution, true) if err != nil { log.Printf("[ERROR] Failed setting workflow execution during init (2): %s", err) @@ -1652,7 +1151,6 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request environments := execInfo.Environments var allEnvs []shuffle.Environment if len(workflowExecution.ExecutionOrg) > 0 { - //log.Printf("[INFO] Executing ORG: %s", workflowExecution.ExecutionOrg) allEnvironments, err := shuffle.GetEnvironments(ctx, workflowExecution.ExecutionOrg) if err != nil { @@ -1905,7 +1403,7 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { log.Printf("[INFO] Inside execute workflow for ID %s", fileId) ctx := context.Background() - workflow, err := shuffle.GetWorkflow(ctx, fileId) + workflow, err := shuffle.GetWorkflow(ctx, fileId, true) if err != nil && workflow.ID == "" { log.Printf("[WARNING] Failed getting the workflow locally (execute workflow): %s", err) resp.WriteHeader(401) diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 343efc26..0b35fe90 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -10,7 +10,7 @@ require ( github.com/docker/docker v27.5.0+incompatible github.com/docker/go-connections v0.5.0 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.18 + github.com/shuffle/shuffle-shared v0.8.19 k8s.io/api v0.30.2 k8s.io/apimachinery v0.30.2 ) diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index 8a1a0443..39b13f42 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -301,8 +301,8 @@ github.com/sendgrid/sendgrid-go v3.14.0+incompatible h1:KDSasSTktAqMJCYClHVE94Fc github.com/sendgrid/sendgrid-go v3.14.0+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.18 h1:7f7cV+P2pr/g44i+AI8P0UheEe8oG8O2V8loBfe9YSw= -github.com/shuffle/shuffle-shared v0.8.18/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= +github.com/shuffle/shuffle-shared v0.8.19 h1:HXqU62sPhVzv9MeJnA5ZpPYwwbitVz1EtMMABbY3t74= +github.com/shuffle/shuffle-shared v0.8.19/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 797a3de9..035ac090 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -8,7 +8,7 @@ require ( github.com/docker/docker v27.5.0+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.18 + github.com/shuffle/shuffle-shared v0.8.19 k8s.io/api v0.30.2 k8s.io/apimachinery v0.30.2 k8s.io/client-go v0.30.2 diff --git a/functions/onprem/worker/go.sum b/functions/onprem/worker/go.sum index 9abeb169..028e4dd8 100644 --- a/functions/onprem/worker/go.sum +++ b/functions/onprem/worker/go.sum @@ -294,8 +294,8 @@ github.com/sendgrid/sendgrid-go v3.14.0+incompatible h1:KDSasSTktAqMJCYClHVE94Fc github.com/sendgrid/sendgrid-go v3.14.0+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.18 h1:7f7cV+P2pr/g44i+AI8P0UheEe8oG8O2V8loBfe9YSw= -github.com/shuffle/shuffle-shared v0.8.18/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= +github.com/shuffle/shuffle-shared v0.8.19 h1:HXqU62sPhVzv9MeJnA5ZpPYwwbitVz1EtMMABbY3t74= +github.com/shuffle/shuffle-shared v0.8.19/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= From 17f779ffe275a1d00acf1c19563ac2040b7279c1 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Mon, 24 Mar 2025 14:31:46 +0100 Subject: [PATCH 14/33] remove the v prefix from chart version Signed-off-by: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> --- .github/workflows/helm-release.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/helm-release.yml b/.github/workflows/helm-release.yml index 9a70f838..b3851987 100644 --- a/.github/workflows/helm-release.yml +++ b/.github/workflows/helm-release.yml @@ -35,8 +35,9 @@ jobs: - name: Set versions run: | if [[ ${{ github.event_name }} == 'release' ]]; then - APP_VERSION="${{ github.event.release.tag_name }}" - CHART_VERSION="${{ github.event.release.tag_name }}" + VERSION="${{ github.event.release.tag_name }}" + APP_VERSION="${VERSION}" + CHART_VERSION="${VERSION#v}" # Remove the v prefix else APP_VERSION="nightly" CHART_VERSION="0.0.0-nightly-untagged-latest" From d152faec9a356321234cf562d967fcc1d82b9c37 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Mon, 24 Mar 2025 18:06:03 +0100 Subject: [PATCH 15/33] remove v prefix from app version too Signed-off-by: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> --- .github/workflows/helm-release.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/helm-release.yml b/.github/workflows/helm-release.yml index b3851987..4c3c6b6f 100644 --- a/.github/workflows/helm-release.yml +++ b/.github/workflows/helm-release.yml @@ -35,9 +35,13 @@ jobs: - name: Set versions run: | if [[ ${{ github.event_name }} == 'release' ]]; then - VERSION="${{ github.event.release.tag_name }}" + TAG_NAME="${{ github.event.release.tag_name }}" + + # Remove the v prefix + VERSION=${TAG_NAME#v} + APP_VERSION="${VERSION}" - CHART_VERSION="${VERSION#v}" # Remove the v prefix + CHART_VERSION="${VERSION}" else APP_VERSION="nightly" CHART_VERSION="0.0.0-nightly-untagged-latest" From e9b10a73cfef71dd7f2180637e49b363517b55a8 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Mon, 24 Mar 2025 18:16:30 +0100 Subject: [PATCH 16/33] use app version image tag by default Signed-off-by: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> --- .../kubernetes/charts/shuffle/Chart.yaml | 4 +-- functions/kubernetes/charts/shuffle/README.md | 6 +++++ .../charts/shuffle/templates/_helpers.tpl | 10 ++++---- .../charts/shuffle/values.schema.json | 25 +++++++++++++++++++ .../kubernetes/charts/shuffle/values.yaml | 18 ++++++------- 5 files changed, 47 insertions(+), 16 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/Chart.yaml b/functions/kubernetes/charts/shuffle/Chart.yaml index f11f1726..0a4e4fa3 100644 --- a/functions/kubernetes/charts/shuffle/Chart.yaml +++ b/functions/kubernetes/charts/shuffle/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: shuffle description: A Helm chart for deploying Shuffle on Kubernetes type: application -version: 0.0.0 -appVersion: 0.0.0 +version: 0.0.0 # Set during publishing in GitHub actions +appVersion: nightly # Overwritten during publishing in GitHub actions dependencies: - name: common version: ^2.23.0 diff --git a/functions/kubernetes/charts/shuffle/README.md b/functions/kubernetes/charts/shuffle/README.md index 495d2690..95329d8e 100644 --- a/functions/kubernetes/charts/shuffle/README.md +++ b/functions/kubernetes/charts/shuffle/README.md @@ -148,6 +148,7 @@ SHUFFLE_ENCRYPTION_MODIFIER: "MyShuffleEncryptionModifier" | ----------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------- | | `backend.image.registry` | backend image registry | `ghcr.io` | | `backend.image.repository` | backend image repository | `shuffle/shuffle-backend` | +| `backend.image.tag` | backend image tag (immutable tags are recommended, defaults to appVersion) | `""` | | `backend.image.digest` | backend image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended) | `""` | | `backend.image.pullPolicy` | backend image pull policy | `IfNotPresent` | | `backend.image.pullSecrets` | backend image pull secrets | `[]` | @@ -265,6 +266,7 @@ SHUFFLE_ENCRYPTION_MODIFIER: "MyShuffleEncryptionModifier" | ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------- | | `frontend.image.registry` | frontend image registry | `ghcr.io` | | `frontend.image.repository` | frontend image repository | `shuffle/shuffle-frontend` | +| `frontend.image.tag` | frontend image tag (immutable tags are recommended, defaults to appVersion) | `""` | | `frontend.image.digest` | frontend image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended) | `""` | | `frontend.image.pullPolicy` | frontend image pull policy | `IfNotPresent` | | `frontend.image.pullSecrets` | frontend image pull secrets | `[]` | @@ -373,6 +375,7 @@ SHUFFLE_ENCRYPTION_MODIFIER: "MyShuffleEncryptionModifier" | ----------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------- | | `orborus.image.registry` | orborus image registry | `ghcr.io` | | `orborus.image.repository` | orborus image repository | `shuffle/shuffle-orborus` | +| `orborus.image.tag` | orborus image tag (immutable tags are recommended, defaults to appVersion) | `""` | | `orborus.image.digest` | orborus image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended) | `""` | | `orborus.image.pullPolicy` | orborus image pull policy | `IfNotPresent` | | `orborus.image.pullSecrets` | orborus image pull secrets | `[]` | @@ -481,6 +484,7 @@ SHUFFLE_ENCRYPTION_MODIFIER: "MyShuffleEncryptionModifier" | ---------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------ | | `worker.image.registry` | worker image registry | `ghcr.io` | | `worker.image.repository` | worker image repository | `shuffle/shuffle-worker` | +| `worker.image.tag` | worker image tag (immutable tags are recommended, defaults to appVersion) | `""` | | `worker.image.digest` | worker image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended) | `""` | | `worker.serviceAccount.create` | Specifies whether a ServiceAccount should be created | `true` | | `worker.serviceAccount.name` | The name of the ServiceAccount to use. | `""` | @@ -582,6 +586,7 @@ SHUFFLE_ENCRYPTION_MODIFIER: "MyShuffleEncryptionModifier" | `volumePermissions.enabled` | Enable init container that changes the owner/group of the PV mount point to `runAsUser:fsGroup` | `false` | | `volumePermissions.image.registry` | OS Shell + Utility image registry | `docker.io` | | `volumePermissions.image.repository` | OS Shell + Utility image repository | `bitnami/os-shell` | +| `volumePermissions.image.tag` | OS Shell + Utility image tag (immutable tags are recommended) | `12-debian-12-r30` | | `volumePermissions.image.pullPolicy` | OS Shell + Utility image pull policy | `IfNotPresent` | | `volumePermissions.image.pullSecrets` | OS Shell + Utility image pull secrets | `[]` | | `volumePermissions.resourcesPreset` | Set init container resources according to one common preset (allowed values: none, nano, small, medium, large, xlarge, 2xlarge). This is ignored if volumePermissions.resources is set (volumePermissions.resources is recommended for production). | `nano` | @@ -605,3 +610,4 @@ SHUFFLE_ENCRYPTION_MODIFIER: "MyShuffleEncryptionModifier" ### Other Parameters + diff --git a/functions/kubernetes/charts/shuffle/templates/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/_helpers.tpl index 6bd29238..dc93dfc2 100644 --- a/functions/kubernetes/charts/shuffle/templates/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/_helpers.tpl @@ -119,14 +119,14 @@ app.kubernetes.io/name: shuffle-app Return the proper image name (for the init container volume-permissions image) */}} {{- define "shuffle.volumePermissions.image" -}} -{{- include "common.images.image" ( dict "imageRoot" .Values.volumePermissions.image "global" .Values.global ) -}} +{{- include "common.images.image" ( dict "imageRoot" .Values.volumePermissions.image "global" .Values.global "chart" .Chart ) -}} {{- end -}} {{/* Return the proper Shuffle backend image name */}} {{- define "shuffle.backend.image" -}} -{{- include "common.images.image" ( dict "imageRoot" .Values.backend.image "global" .Values.global ) -}} +{{- include "common.images.image" ( dict "imageRoot" .Values.backend.image "global" .Values.global "chart" .Chart ) -}} {{- end -}} {{/* @@ -140,7 +140,7 @@ Return the proper Docker Image Registry Secret Names for the backend pod Return the proper Shuffle frontend image name */}} {{- define "shuffle.frontend.image" -}} -{{- include "common.images.image" ( dict "imageRoot" .Values.frontend.image "global" .Values.global ) -}} +{{- include "common.images.image" ( dict "imageRoot" .Values.frontend.image "global" .Values.global "chart" .Chart ) -}} {{- end -}} {{/* @@ -154,7 +154,7 @@ Return the proper Docker Image Registry Secret Names for the frontend pod Return the proper Shuffle orborus image name */}} {{- define "shuffle.orborus.image" -}} -{{- include "common.images.image" ( dict "imageRoot" .Values.orborus.image "global" .Values.global ) -}} +{{- include "common.images.image" ( dict "imageRoot" .Values.orborus.image "global" .Values.global "chart" .Chart ) -}} {{- end -}} {{/* @@ -168,7 +168,7 @@ Return the proper Docker Image Registry Secret Names for the orborus pod Return the proper Shuffle worker image name */}} {{- define "shuffle.worker.image" -}} -{{- include "common.images.image" ( dict "imageRoot" .Values.worker.image "global" .Values.global ) -}} +{{- include "common.images.image" ( dict "imageRoot" .Values.worker.image "global" .Values.global "chart" .Chart ) -}} {{- end -}} {{/* diff --git a/functions/kubernetes/charts/shuffle/values.schema.json b/functions/kubernetes/charts/shuffle/values.schema.json index 7a85b9fa..f326c08c 100644 --- a/functions/kubernetes/charts/shuffle/values.schema.json +++ b/functions/kubernetes/charts/shuffle/values.schema.json @@ -155,6 +155,11 @@ "description": "backend image repository", "default": "shuffle/shuffle-backend" }, + "tag": { + "type": "string", + "description": "backend image tag (immutable tags are recommended, defaults to appVersion)", + "default": "" + }, "digest": { "type": "string", "description": "backend image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended)", @@ -839,6 +844,11 @@ "description": "frontend image repository", "default": "shuffle/shuffle-frontend" }, + "tag": { + "type": "string", + "description": "frontend image tag (immutable tags are recommended, defaults to appVersion)", + "default": "" + }, "digest": { "type": "string", "description": "frontend image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended)", @@ -1463,6 +1473,11 @@ "description": "orborus image repository", "default": "shuffle/shuffle-orborus" }, + "tag": { + "type": "string", + "description": "orborus image tag (immutable tags are recommended, defaults to appVersion)", + "default": "" + }, "digest": { "type": "string", "description": "orborus image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended)", @@ -2092,6 +2107,11 @@ "description": "worker image repository", "default": "shuffle/shuffle-worker" }, + "tag": { + "type": "string", + "description": "worker image tag (immutable tags are recommended, defaults to appVersion)", + "default": "" + }, "digest": { "type": "string", "description": "worker image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended)", @@ -2591,6 +2611,11 @@ "description": "OS Shell + Utility image repository", "default": "bitnami/os-shell" }, + "tag": { + "type": "string", + "description": "OS Shell + Utility image tag (immutable tags are recommended)", + "default": "12-debian-12-r30" + }, "pullPolicy": { "type": "string", "description": "OS Shell + Utility image pull policy", diff --git a/functions/kubernetes/charts/shuffle/values.yaml b/functions/kubernetes/charts/shuffle/values.yaml index fb580513..8092f041 100644 --- a/functions/kubernetes/charts/shuffle/values.yaml +++ b/functions/kubernetes/charts/shuffle/values.yaml @@ -96,7 +96,7 @@ backend: ## backend image ## @param backend.image.registry backend image registry ## @param backend.image.repository backend image repository - ## @skip backend.image.tag backend image tag (immutable tags are recommended) + ## @param backend.image.tag backend image tag (immutable tags are recommended, defaults to appVersion) ## @param backend.image.digest backend image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended) ## @param backend.image.pullPolicy backend image pull policy ## @param backend.image.pullSecrets backend image pull secrets @@ -104,7 +104,7 @@ backend: image: registry: ghcr.io repository: shuffle/shuffle-backend - tag: 2.0.0 + tag: "" digest: "" ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' @@ -565,7 +565,7 @@ frontend: ## frontend image ## @param frontend.image.registry frontend image registry ## @param frontend.image.repository frontend image repository - ## @skip frontend.image.tag frontend image tag (immutable tags are recommended) + ## @param frontend.image.tag frontend image tag (immutable tags are recommended, defaults to appVersion) ## @param frontend.image.digest frontend image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended) ## @param frontend.image.pullPolicy frontend image pull policy ## @param frontend.image.pullSecrets frontend image pull secrets @@ -573,7 +573,7 @@ frontend: image: registry: ghcr.io repository: shuffle/shuffle-frontend - tag: 2.0.0 + tag: "" digest: "" ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' @@ -970,7 +970,7 @@ orborus: ## orborus image ## @param orborus.image.registry orborus image registry ## @param orborus.image.repository orborus image repository - ## @skip orborus.image.tag orborus image tag (immutable tags are recommended) + ## @param orborus.image.tag orborus image tag (immutable tags are recommended, defaults to appVersion) ## @param orborus.image.digest orborus image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended) ## @param orborus.image.pullPolicy orborus image pull policy ## @param orborus.image.pullSecrets orborus image pull secrets @@ -978,7 +978,7 @@ orborus: image: registry: ghcr.io repository: shuffle/shuffle-orborus - tag: 2.0.0 + tag: "" digest: "" ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' @@ -1377,13 +1377,13 @@ worker: ## worker image ## @param worker.image.registry worker image registry ## @param worker.image.repository worker image repository - ## @skip worker.image.tag worker image tag (immutable tags are recommended) + ## @param worker.image.tag worker image tag (immutable tags are recommended, defaults to appVersion) ## @param worker.image.digest worker image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended) ## image: registry: ghcr.io repository: shuffle/shuffle-worker - tag: 2.0.0 + tag: "" digest: "" ## ServiceAccount configuration @@ -1748,7 +1748,7 @@ volumePermissions: ## ref: https://hub.docker.com/r/bitnami/os-shell/tags/ ## @param volumePermissions.image.registry OS Shell + Utility image registry ## @param volumePermissions.image.repository OS Shell + Utility image repository - ## @skip volumePermissions.image.tag OS Shell + Utility image tag (immutable tags are recommended) + ## @param volumePermissions.image.tag OS Shell + Utility image tag (immutable tags are recommended) ## @param volumePermissions.image.pullPolicy OS Shell + Utility image pull policy ## @param volumePermissions.image.pullSecrets OS Shell + Utility image pull secrets ## From a2e4d37eeaa81fece73aaa804bd0456e7dc4938d Mon Sep 17 00:00:00 2001 From: Frikky Date: Mon, 24 Mar 2025 20:59:44 +0100 Subject: [PATCH 17/33] Allowed workflow reruns from a specific node --- backend/go-app/docker.go | 10 +++++++++- backend/go-app/walkoff.go | 11 +++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index 11bbe14d..e56455eb 100755 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -647,7 +647,15 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { alternativeName = strings.Join(alternativeNameSplit[1:3], "/") } - log.Printf("[INFO] Trying to download image: %s. Alt: %s", version.Name, alternativeName) + if len(version.Name) == 0 { + log.Printf("[ERROR] No image name provided for download: %s", version.Name) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "No image name"}`))) + return + + } + + log.Printf("[INFO] Trying to download image: '%s'. Alt name: %#v", version.Name, alternativeName) for _, image := range images { for _, tag := range image.RepoTags { diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 434b5a42..93540aa8 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -3041,9 +3041,16 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) { return } - time.Sleep(2 * time.Second) - log.Printf("[INFO] Starting validation of execution %s", workflowExecution.ExecutionId) + if shouldRerun { + log.Printf("[DEBUG] Returning single action execution ID for rerun: %s", workflowExecution.ExecutionId) + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s", "authorization": "%s"}`, workflowExecution.ExecutionId, workflowExecution.Authorization))) + return + } + + log.Printf("[INFO] Starting validation of execution %s", workflowExecution.ExecutionId) + time.Sleep(2 * time.Second) returnBody := shuffle.HandleRetValidation(ctx, workflowExecution, 1) returnBytes, err := json.Marshal(returnBody) if err != nil { From a3b0093cf51f2f0b13007eb4c935cc2ddd201e54 Mon Sep 17 00:00:00 2001 From: Frikky Date: Tue, 25 Mar 2025 00:54:32 +0100 Subject: [PATCH 18/33] Workflow Action Rerun optimization --- frontend/src/components/EditWorkflow.jsx | 2 + frontend/src/components/Navbar.jsx | 3 +- frontend/src/components/ParsedAction.jsx | 59 +++++- .../src/components/ShuffleCodeEditor1.jsx | 55 ++++- frontend/src/theme.jsx | 22 +- frontend/src/views/AngularWorkflow.jsx | 188 ++++++++++++++++-- frontend/src/views/AppExplorer.jsx | 9 +- frontend/src/views/Docs.jsx | 7 +- frontend/src/views/SettingsPage.jsx | 11 +- 9 files changed, 301 insertions(+), 55 deletions(-) diff --git a/frontend/src/components/EditWorkflow.jsx b/frontend/src/components/EditWorkflow.jsx index 5f571110..dcfa3057 100644 --- a/frontend/src/components/EditWorkflow.jsx +++ b/frontend/src/components/EditWorkflow.jsx @@ -545,6 +545,8 @@ const EditWorkflow = (props) => { }} > } label="Test" /> + } label="Staging" /> + } label="Pre-production" /> } label="Production" /> diff --git a/frontend/src/components/Navbar.jsx b/frontend/src/components/Navbar.jsx index 21321b94..04e67ac7 100644 --- a/frontend/src/components/Navbar.jsx +++ b/frontend/src/components/Navbar.jsx @@ -1363,6 +1363,7 @@ const Navbar = (props) => { sx={buttonStyles} onClick={() => { if(isCloud) { + // navigate("/new-pricing"); navigate("/pricing"); ReactGA.event({ category: "navbar", @@ -1370,7 +1371,7 @@ const Navbar = (props) => { label: "go_to_pricing", }) } else { - window.open("https://shuffler.io/pricing", '_blank'); + window.open("https://shuffler.io/pricing?env=Self-hosted", '_blank'); return; } }} diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index 35bc0ea4..5c35ec56 100755 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -179,6 +179,7 @@ const ParsedAction = (props) => { suborgWorkflows, originalWorkflow, + runFromHere, } = props; let navigate = useNavigate() @@ -233,11 +234,16 @@ const ParsedAction = (props) => { // auth, required, optional var changed = false if (selectedActionParameters === undefined || selectedActionParameters === null || selectedActionParameters.length === 0) { + console.log("Returning because no params") return } if (selectedApp !== undefined && selectedApp !== null && selectedApp.generated !== true) { - return + + if (isAgent || isIntegration) { + } else { + return + } } // Fixing required fields with a shitty structure :) @@ -346,13 +352,23 @@ const ParsedAction = (props) => { // 3. generated fields & all else - const newparams = auth + var newparams = auth .concat(bodyfield) .concat(required) .concat(generated_optional) .concat(special_optional) .concat(optional) + const dedupedParams = [] + for (var paramKey in newparams) { + const param = newparams[paramKey] + if (dedupedParams.find(item => item.name === param.name) === undefined) { + dedupedParams.push(param) + } + } + + newparams = dedupedParams + var newkeyorder = [] for (let paramkey in newparams) { //console.log("Param: ", newparams[paramkey]) @@ -1814,6 +1830,29 @@ const ParsedAction = (props) => { + + {(selectedAction?.generated === true && selectedAction?.app_version === "1.0.0") || (selectedAction?.app_name === "Shuffle Tools" && selectedAction?.app_version !== "1.2.0") ?
@@ -4737,6 +4775,7 @@ const ParsedAction = (props) => { showDropdownNumber === count && data.variant === "STATIC_VALUE" && jsonList.length > 0 ? ( + { setMainVariables(tmpVariables) } + const handleKeyDown = (event) => { + if ((event.metaKey || event.ctrlKey) && event.key === 'Enter') { + event.preventDefault() + const tryItButton = document.getElementById("try-it-button") + if (tryItButton !== undefined && tryItButton !== null) { + tryItButton.click() + } + } + } + // Remove the original useEffect for actionlist since we'll update on action/trigger changes useEffect(() => { + document.addEventListener("keydown", handleKeyDown) updateAvailableVariables(actionlist) }, []) @@ -1721,13 +1736,13 @@ const CodeEditor = (props) => { color="secondary" style={{ textTransform: "none", - width: 120, + width: 145, }} onClick={(event) => { setAnchorEl3(event.currentTarget); }} > - Python Code + Python Examples diff --git a/frontend/src/theme.jsx b/frontend/src/theme.jsx index 4897e2ca..1621dc42 100644 --- a/frontend/src/theme.jsx +++ b/frontend/src/theme.jsx @@ -27,7 +27,7 @@ const theme = createTheme(adaptV4Theme({ distributionColor: "#40E0D0", green: "#5cc879", - borderRadius: 10, + borderRadius: 8, defaultBorder: "1px solid rgba(255,255,255,0.3)", //jsonTheme: "brewer", @@ -112,39 +112,39 @@ const theme = createTheme(adaptV4Theme({ MuiCssBaseline: { styleOverrides: ` @font-face { - font-family: 'Roboto'; + font-family: 'Inter'; font-style: normal; font-display: swap; font-weight: 300; - src: local('Roboto Light'), local('Roboto-Light'); + src: local('Inter Light'), local('Inter-Light'); } @font-face { - font-family: 'Roboto'; + font-family: 'Inter'; font-style: normal; font-display: swap; font-weight: 400; - src: local('Roboto'), local('Roboto-Regular'); + src: local('Inter Regular'), local('Inter-Regular'); } @font-face { - font-family: 'Roboto'; + font-family: 'Inter'; font-style: normal; font-display: swap; font-weight: 500; - src: local('Roboto Medium'), local('Roboto-Medium'); + src: local('Inter Medium'), local('Inter-Medium'); } @font-face { - font-family: 'Roboto'; + font-family: 'Inter'; font-style: normal; font-display: swap; font-weight: 600; - src: local('Roboto SemiBold'), local('Roboto-SemiBold'); + src: local('Inter SemiBold'), local('Inter-SemiBold'); } @font-face { - font-family: 'Roboto'; + font-family: 'Inter'; font-style: normal; font-display: swap; font-weight: 700; - src: local('Roboto Bold'), local('Roboto-Bold'); + src: local('Inter Bold'), local('Inter-Bold'); } `, }, diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 1f32c6ec..3394dfd9 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -652,6 +652,31 @@ const AngularWorkflow = (defaultprops) => { ], "multiselect": true, }, + { + "name": "memory", + "value": "", + "required": true, + "description": "Whether to store the conversation in memory", + "options": [ + "Nothing", + "Shuffle Datastore", + ], + "multiselect": false, + "disabled": true, + }, + { + "name": "knowledge", + "value": "", + "required": true, + "description": "The knowledge we should inject into the context window", + "options": [ + "Nothing", + "Shuffle Files", + ], + "multiselect": false, + "disabled": true, + }, + ] }], large_image: theme.palette.singulBlackWhite, @@ -772,7 +797,7 @@ const AngularWorkflow = (defaultprops) => { const [loadedApps, setLoadedApps] = React.useState([]) - const loadAppConfig = (appId, select) => { + const loadAppConfig = (appId, select, skipAppLoad) => { if (appId === undefined || appId === null || appId.length === 0) { console.log("No appId to load") return @@ -868,7 +893,7 @@ const AngularWorkflow = (defaultprops) => { setSelectedApp(foundapp) } - if (apps === undefined || apps === null || apps.length === 0) { + if ((apps === undefined || apps === null || apps.length === 0) && skipAppLoad !== true) { console.log("No apps to update :(") getApps() return @@ -2785,6 +2810,94 @@ const AngularWorkflow = (defaultprops) => { return true; }; + const runFromHere = (curAction) => { + if (curAction.app_id === undefined || curAction.app_id === null || curAction.app_id.length === 0) { + toast.error("No app id found for action. Please contact support@shuffler.io if this persists") + return + } + + if (workflow.id !== undefined && workflow.id !== null && workflow.id.length > 0) { + curAction.source_workflow = workflow.id + } + + // Based on the previous execution id + console.log("WORKFLOW EXEC: ", workflowExecutions) + // Look for the "execution_id" parameter + const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; + const execFound = new URLSearchParams(cursearch).get("execution_id"); + if (execFound !== undefined && execFound !== null && execFound.length > 0) { + toast.info("Rerunning based on previously watched execution id") + curAction.source_execution = execFound + } else if (workflowExecutions !== undefined && workflowExecutions !== null && workflowExecutions.length > 0) { + curAction.source_execution = workflowExecutions[0].execution_id + } else { + toast.error("No previous execution found. Please run the workflow first.") + return + } + + setExecutionRunning(true) + setExecutionRequestStarted(true) + var headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + } + + if (workflow.org_id !== undefined && workflow.org_id !== null && workflow.org_id.length > 0) { + headers["Org-Id"] = workflow.org_id + } + + // Rerun makes it return execution_id + authorization + const appRunUrl = `${globalUrl}/api/v1/apps/${curAction.app_id}/run?rerun=true` + fetch(appRunUrl, { + method: 'POST', + headers: headers, + body: JSON.stringify(curAction), + credentials: "include", + }) + .then((response) => { + setExecutionRunning(false) + setExecutionRequestStarted(false) + + if (response.status !== 200) { + console.log("Status not 200 for stream results :O!") + } + + return response.json() + }) + .then((responseJson) => { + setExecutionRequestStarted(false) + + if (responseJson?.success === false) { + setExecutionRunning(false) + + if (responseJson?.reason !== undefined && responseJson?.reason !== null && responseJson?.reason.length > 0) { + toast.error(responseJson.reason) + } else { + toast.error("Failed to run the action. Please try again or contact support@shuffler.io") + } + + return + } else if (responseJson?.success === true && responseJson?.execution_id !== undefined && responseJson?.execution_id !== null && responseJson?.execution_id.length > 0) { + navigate(`?execution_id=${responseJson.execution_id}`) + setExecutionRequest({ + execution_id: responseJson.execution_id, + authorization: responseJson.authorization, + }) + + setExecutionData({}) + setExecutionModalOpen(true) + setExecutionModalView(1) + start() + } + }) + .catch((error) => { + toast.error("Failed to run the action. "+error.toString()) + + setExecutionRunning(false) + setExecutionRequestStarted(false) + }) + } + const executeWorkflow = (executionArgument, startNode, hasSaved, skip_popup) => { if (hasSaved === false) { @@ -3203,6 +3316,13 @@ const AngularWorkflow = (defaultprops) => { return } + for (var key in responseJson) { + const curapp = responseJson[key] + if (curapp?.actions === undefined || curapp?.actions === null || curapp?.actions?.length === 0 || curapp?.actions?.length === 1) { + loadAppConfig(curapp?.id, false, true) + } + } + // Find app with ID "794e51c3c1a8b24b89ccc573a3defc47" (gmail) to force-break it, // Find app with ID "3e2bdf9d5069fe3f4746c29d68785a6a" (shuffle tools) to force-break it, // as to ensure the autocorrect works. @@ -13580,14 +13700,18 @@ const AngularWorkflow = (defaultprops) => { setSourceValue({ ...sourceValue, value: value - }); + }) + + setUpdate(Math.random()) } else if (fieldType === "destination") { setDestinationValue({ ...destinationValue, value: value - }); - } - }; + }) + + setUpdate(Math.random()) + } + } const conditionsModal = ( @@ -14763,6 +14887,7 @@ const AngularWorkflow = (defaultprops) => { ] const handleSubflowParamChange = (triggerId, triggerField, newData) => { + var updateFail = "" if (workflow !== undefined && workflow !== null) { // Find the trigger with matching id @@ -14778,9 +14903,20 @@ const AngularWorkflow = (defaultprops) => { setWorkflow(workflow); setSelectedTriggerValue(newData) setLastSaved(false); - } - } - } + setUpdate(Math.random()) + } else { + updateFail = "Parameter is undefined or null" + } + } else { + updateFail = "Trigger is undefined or null" + } + } else { + updateFail = "Workflow is undefined or null" + } + + if (updateFail !== "") { + toast.error(updateFail + " - Failed to update subflow parameter value. Please try again.") + } } const SubflowSidebar = Object.getOwnPropertyNames(selectedTrigger).length === 0 || workflow.triggers[selectedTriggerIndex] === undefined || selectedTrigger.trigger_type !== "SUBFLOW" ? null : @@ -20801,9 +20937,9 @@ const AngularWorkflow = (defaultprops) => { onClose={() => { setExecutionModalOpen(false) - const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; - const newitem = removeParam("execution_id", cursearch); - navigate(curpath + newitem) + //const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; + //const newitem = removeParam("execution_id", cursearch); + //navigate(curpath + newitem) }} style={{ resize: "both", @@ -21256,7 +21392,11 @@ const AngularWorkflow = (defaultprops) => {

Details

+ Rerun workflow. Uses same startnode as the original. Runs from scratch. + + } placement="top" style={{ zIndex: 50000 }} > @@ -21558,7 +21698,7 @@ const AngularWorkflow = (defaultprops) => {
: null} - {userdata.support === true && executionData.workflow !== undefined && executionData.workflow !== null && executionData.status !== "EXECUTING" ? + {userdata.support === true && executionData.workflow !== undefined && executionData.workflow !== null && executionData.status !== "EXECUTING" && executionData.status !== "ABORTED" ?
{ suborgWorkflows={suborgWorkflows} originalWorkflow={originalWorkflow} + runFromHere={runFromHere} />
@@ -24731,6 +24872,8 @@ const AngularWorkflow = (defaultprops) => { } const handleActionParamChange = (actionId, fieldName, newData) => { + var updateFail = "" + if (workflow !== undefined) { // Find the action with matching id const actionIndex = workflow?.actions.findIndex(action => action.id === actionId); @@ -24745,9 +24888,20 @@ const AngularWorkflow = (defaultprops) => { // Update workflow state to trigger re-render setWorkflow({...workflow}); setLastSaved(false); - } - } - } + setUpdate(Math.random()) + } else { + updateFail = "Parameter is undefined or null" + } + } else { + updateFail = "Trigger is undefined or null" + } + } else { + updateFail = "Workflow is undefined or null" + } + + if (updateFail !== "") { + toast.error(updateFail + " - Failed to update subflow parameter value. Please try again.") + } } /* var foundusecase = {} diff --git a/frontend/src/views/AppExplorer.jsx b/frontend/src/views/AppExplorer.jsx index d37709ee..b19e5cdd 100644 --- a/frontend/src/views/AppExplorer.jsx +++ b/frontend/src/views/AppExplorer.jsx @@ -1088,14 +1088,13 @@ const AppExplorer = (props) => { setNewWorkflowTags(newWorkflowTags); } - // This is annoying (: - var securitySchemes = data.components.securityDefinitions; + var securitySchemes = data?.components?.securityDefinitions; if (securitySchemes === undefined) { - securitySchemes = data.securitySchemes; + securitySchemes = data?.securitySchemes; } if (securitySchemes === undefined) { - securitySchemes = data.components.securitySchemes; + securitySchemes = data?.components?.securitySchemes; } const allowedfunctions = [ @@ -1681,7 +1680,7 @@ const AppExplorer = (props) => { setExecutionResult({ valid: false, result: - "Couldn't finish execution (2). Please fill all the required fields, and validate the execution.", + "Couldn't finish execution OR no result was returned (2). Please fill all the required fields, and validate the execution.", }); } diff --git a/frontend/src/views/Docs.jsx b/frontend/src/views/Docs.jsx index 8b56b2ce..90def218 100755 --- a/frontend/src/views/Docs.jsx +++ b/frontend/src/views/Docs.jsx @@ -343,10 +343,15 @@ const Docs = (defaultprops) => { return } + console.log("PROPKEY: ", propkey) if (location.pathname.includes("/docs/")) { if (propkey === "cookie_policy" || propkey === "compliance" || propkey === "privacy_policy" || propkey === "terms_of_service") { navigate(`/legal/${propkey}`) } + + if (propkey === "app_creation") { + navigate('/docs/apps#app-creation-introduction') + } } }, [location]); @@ -932,7 +937,7 @@ const Docs = (defaultprops) => { } if (responseJson.reason !== undefined && responseJson.reason !== null && responseJson.reason.includes("404: Not Found") && !isArticlePage) { - navigate("/docs") + //navigate("/docs") return } diff --git a/frontend/src/views/SettingsPage.jsx b/frontend/src/views/SettingsPage.jsx index a507857f..c19cd987 100755 --- a/frontend/src/views/SettingsPage.jsx +++ b/frontend/src/views/SettingsPage.jsx @@ -527,6 +527,8 @@ const Settings = (props) => { }; const generateApikey = () => { + toast.info("Generating new API key. This may take a bit."); + fetch(globalUrl + "/api/v1/generateapikey", { method: "GET", headers: { @@ -538,7 +540,7 @@ const Settings = (props) => { .then((response) => { if (response.status !== 200) { console.log("Status not 200 for WORKFLOW EXECUTION :O!"); - } + } return response.json(); }) @@ -836,7 +838,12 @@ const Settings = (props) => { variant="outlined" /> - : null} + : null} */} ) } @@ -2021,7 +2021,7 @@ const Billing = memo((props) => {
- {isCloud && + {/* {isCloud && selectedOrganization.subscriptions !== undefined && selectedOrganization.subscriptions !== null && selectedOrganization.subscriptions.length > 0 && @@ -2043,7 +2043,7 @@ const Billing = memo((props) => { /> ) }) - : null} + : null} */}
{isCloud && billingInfo.subscription !== undefined && billingInfo.subscription !== null ? isChildOrg ? null : diff --git a/frontend/src/components/CacheView.jsx b/frontend/src/components/CacheView.jsx index 34b4608f..f4b8b9cb 100644 --- a/frontend/src/components/CacheView.jsx +++ b/frontend/src/components/CacheView.jsx @@ -151,13 +151,16 @@ const CacheView = memo((props) => { if (fileCategories.length === 1 && fileCategories[0] === "default") { var newcategories = ["default"] for (var key in responseJson.keys) { - if (responseJson.keys[key].category !== undefined && responseJson.keys[key].category !== null && responseJson.keys[key].category !== "" && !fileCategories.includes(responseJson.keys[key].category)) { - newcategories.push(responseJson.keys[key].category); + var category = responseJson.keys[key].category + if (category !== undefined && category !== null && category !== ""){ + category = category.replaceAll(" ", "_") + + if (!newcategories.includes(category)) { + newcategories.push(category) + } } } - console.log("CATEGORIES: ", newcategories) - setFileCategories(newcategories) } } @@ -357,6 +360,7 @@ const CacheView = memo((props) => { { editCache ? "Edit Key" : "Add Key"}{selectedCategory === "" || selectedCategory === "default" ? "" : ` in category '${selectedCategory}'`} +
Key @@ -1041,13 +1045,13 @@ const CacheView = memo((props) => { { deleteCache(orgId, data.key); //deleteFile(orgId); diff --git a/frontend/src/components/LeftSideBar.jsx b/frontend/src/components/LeftSideBar.jsx index baf60f06..92aeabe4 100644 --- a/frontend/src/components/LeftSideBar.jsx +++ b/frontend/src/components/LeftSideBar.jsx @@ -774,8 +774,8 @@ const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, }) => { }, [window?.location?.pathname]); const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); - - + const showPartnerLogo = userdata?.org_status?.includes("integration_partner") && userdata?.active_org?.image !== undefined && userdata?.active_org?.image !== null && userdata?.active_org?.image.length > 0 + return (
{ } }} > - + Shuffle Logo @@ -1611,7 +1613,7 @@ const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, }) => { }} > - {expandLeftNav && + {userdata?.licensed !== true && !userdata?.org_status?.includes("integration_partner") && expandLeftNav &&
- {subscription.active === true && !isScale && } + {subscription.active === true && !isScale && }
{top_text === "Base Cloud Access" && userdata.has_card_available === true && !isScale ? { color="primary" /> : null} - + {top_text} @@ -396,7 +453,7 @@ const LicencePopup = (props) => { }} /> : null} - {isCloud && highlight === true && top_text !== "Base Cloud Access" ? + {isCloud && highlight === true && top_text !== "Starter Plan" ? { {subscription.currency_text}{subscription.price} - / {subscription.interval} + {subscription.interval.length > 0 ? `/ ${subscription.interval}` : ""}
: null} @@ -517,23 +574,17 @@ const LicencePopup = (props) => { : null} - {subscription.name.includes("Scale") ? - "" - : - - userdata.has_card_available === true ? - "While you have a card attached to your account, Shuffle will no longer prevent workflows from running. Billing will occur at the start of each month." - : + { isCloud ? - userdata?.app_execution_limit && userdata?.app_execution_limit >= 300000 ? - "You have subscribed to the Enterprise plan, which includes " + (userdata?.app_execution_limit/1000) + "K app runs/month. You can increase the limit by upgrading current plan. Contact support@shuffler.io for more information." : - `You are not subscribed to any plan and are using the free plan with max 10,000 app runs per month. Upgrade to deactivate this limit.` + userdata?.app_execution_limit && userdata?.app_execution_limit !== 10000 ? + "You have already subscribed to the Scale plan, which includes " + (userdata?.app_execution_limit/1000) + "K app runs/month. You can increase the limit by upgrading current plan. Contact support@shuffler.io for more information." : + `You are using free Starter plan with max ${userdata?.app_execution_limit === 10000 ? "10,000" : "2,000"} runs per month. Upgrade to increase this limit.` : `You are not subscribed to any plan and are using the free, open source plan. This plan has no enforced limits, but scale issues may occur due to CPU congestion.` } - {isCloud && (userdata.has_card_available === true || selectedOrganization?.Billing?.Email?.length > 0 )? + {/* {isCloud && (userdata.has_card_available === true || selectedOrganization?.Billing?.Email?.length > 0 )?
Billing email: {BillingEmail}
- : null} + : null} */}
{isCloud ? ( ) : null} } - + color="primary">Recommended + + { + billingCycle === "annual" && + ( + + + 10% OFF + + + ) + } +
+
+ + {scaleValue > 300 ? "Enterprise Plan" : "Scale Plan"} + + + + + Monthly + + + Annual + + +
- {shuffleVariant === 1 ? "Scale" : "Enterprise"} - {shuffleVariant === 0 ? - "SaaS / Cloud - Per Month" - : - "Open Source + Scale License" - } + App Runs Units - { - - if (calculatedCores === "Get A Quote") { - console.log("Clicked on get a quote") - if (window.drift !== undefined) { - window.drift.api.startInteraction({ interactionId: 340785 }) - } - } - }}>{calculatedCost} - For {shuffleVariant === 1 ? `${selectedValue} CPU cores` : `${selectedValue}k App Runs`}: -
- - { - handleChange(event, newValue) +
+ + {scaleValue > 300 ? "Let's Talk" : `$${getPrice(32) * (scaleValue / 10)}`} + + 300 ? 1 : 0, }} - marks - value={selectedValue} - step={shuffleVariant === 0 ? 100 : 4} - min={shuffleVariant === 0 ? 100 : 8} - max={shuffleVariant === 0 ? 1000 : 32} - valueLabelDisplay="auto" - /> + > + {scaleValue > 300 ? `for ${scaleValue > 500 ? "500k+" : `${scaleValue}k`} App Runs` : `/month for ${scaleValue}k App Runs`} +
+ + { + if(value === 510){ + return "500k+" + } + return `${value}k` + }} + step={10} + min={10} + max={510} + marks + sx={{ + color: "#ff8544", + "& .MuiSlider-thumb": { + width: 15, + height: 15, + }, + "& .MuiSlider-valueLabel": { + backgroundColor: "rgba(33, 33, 33, 1)", + color: "rgba(241, 241, 241, 1)", + fontSize: 14, + borderRadius: "4px", + border: "1px solid rgba(73, 73, 73, 1)", + fontFamily: theme?.typography?.fontFamily, + }, + }} + /> +
{defaultTaskIcon} - Priority Support + Standard Email Support
@@ -1117,7 +1357,7 @@ const LicencePopup = (props) => {
{defaultTaskIcon} - Help with Workflow and App development + 30 Days workflow run history
@@ -1136,7 +1376,7 @@ const LicencePopup = (props) => { navigate("/pricing") } else { - window.open("https://shuffler.io/pricing?tab=onprem", "_blank") + window.open("https://shuffler.io/pricing?tab=Self-Hosted", "_blank") } }} color="primary" @@ -1149,6 +1389,10 @@ const LicencePopup = (props) => { style={{ borderRadius: 4, textTransform: "capitalize", color: "#1a1a1a", backgroundColor: "#ff8544", width: "100%", fontSize: 16}} onClick={() => { if (isCloud) { + if(scaleValue > 300){ + navigate("/contact?category=cloud_enterprise_plan") + return; + } ReactGA.event({ category: "header", action: "upgread_clicks_popup", @@ -1170,7 +1414,7 @@ const LicencePopup = (props) => { }} color="primary" > - Upgrade + {scaleValue > 300 ? "Let's Talk" : "Upgrade"}
diff --git a/frontend/src/components/Navbar.jsx b/frontend/src/components/Navbar.jsx index 04e67ac7..54362e86 100644 --- a/frontend/src/components/Navbar.jsx +++ b/frontend/src/components/Navbar.jsx @@ -1363,7 +1363,6 @@ const Navbar = (props) => { sx={buttonStyles} onClick={() => { if(isCloud) { - // navigate("/new-pricing"); navigate("/pricing"); ReactGA.event({ category: "navbar", @@ -1371,7 +1370,7 @@ const Navbar = (props) => { label: "go_to_pricing", }) } else { - window.open("https://shuffler.io/pricing?env=Self-hosted", '_blank'); + window.open("https://shuffler.io/pricing?env=Self-Hosted", '_blank'); return; } }} diff --git a/frontend/src/components/OrgHeaderexpandedNew.jsx b/frontend/src/components/OrgHeaderexpandedNew.jsx index e5317da3..8d95b046 100644 --- a/frontend/src/components/OrgHeaderexpandedNew.jsx +++ b/frontend/src/components/OrgHeaderexpandedNew.jsx @@ -538,7 +538,7 @@ const OrgHeaderexpandedNew = (props) => { renderValue={(selected) => selected.join(', ')} MenuProps={MenuProps} > - {["contacted", "lead", "demo done", "pov", "customer", "open source", "student", "internal", "creator", "tech partner", "old customer", "old lead"].map((name) => ( + {["contacted", "lead", "demo done", "pov", "customer", "open source", "student", "internal", "creator", "tech partner", "integration partner", "distribution partner", "service partner", "old customer", "old lead"].map((name) => ( -1} /> diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index 5c35ec56..e2e7e14b 100755 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -682,11 +682,14 @@ const ParsedAction = (props) => { // Process workflowExecutions if (workflowExecutions.length > 0) { + var appended = false + var foundvalue = "" for (let execution of workflowExecutions) { const execArg = execution.execution_argument; if (execArg && execArg.length > 0) { const valid = validateJson(execArg); if (valid.valid) { + appended = true newActionList.push({ type: "Runtime Argument", name: "Runtime Argument", @@ -697,9 +700,23 @@ const ParsedAction = (props) => { }) break + } else { + foundvalue = execArg } } } + + if (!appended && foundvalue !== undefined && foundvalue !== "") { + newActionList.push({ + type: "Runtime Argument", + name: "Runtime Argument", + highlight: "exec", + autocomplete: "exec", + + value: foundvalue, + example: foundvalue, + }) + } } // Add default Runtime Argument if none were added @@ -788,6 +805,8 @@ const ParsedAction = (props) => { } labels.push(parentNode.label); + + var secondaryExample = "" let exampleData = parentNode.example ?? ""; if (parentNode?.app_name === "http") { exampleData = "" @@ -798,9 +817,13 @@ const ParsedAction = (props) => { const foundResult = exec.results?.find(result => result?.action?.id === parentNode?.id); if (foundResult) { const valid = validateJson(foundResult.result); - if (valid.valid && valid.result.success !== false) { - exampleData = valid.result - break + if (valid.valid) { + if (valid.result.success !== false) { + exampleData = valid.result + break + } + } else { + secondaryExample = foundResult.result } } } @@ -834,6 +857,10 @@ const ParsedAction = (props) => { } } } + } + + if (exampleData === "" && secondaryExample !== "") { + exampleData = secondaryExample } if (parentNode.label === undefined) { @@ -1178,7 +1205,9 @@ const ParsedAction = (props) => { selectedAction.parameters[1].value = splitparsed[1] if (splitparsed.length > 2) { - toast.warn("Filter list only supports filtering at the first level. If you want multi-level filtering, please use the 'execute python' action with the 'filter a list' function in the code editor.") + toast.warn("Filter list only supports filtering on the first list. If you want multi-level filtering, please use the 'execute python' action with the 'filter a list' function in the code editor.", { + autoClose: 10000, + }) } else if (selectedAction.parameters[1].value.includes(".#")) { toast.warn("This filter may not work due to using .# indexing. Please use the 'execute python' action and try the 'filter a list' function in the code editor.") } @@ -1830,28 +1859,37 @@ const ParsedAction = (props) => { - + + {(selectedAction?.generated === true && selectedAction?.app_version === "1.0.0") || (selectedAction?.app_name === "Shuffle Tools" && selectedAction?.app_version !== "1.2.0") ?
} style={{ display: "table-cell", padding: 8, verticalAlign: "middle" }} diff --git a/frontend/src/components/UserManagmentTab.jsx b/frontend/src/components/UserManagmentTab.jsx index 8aff24a5..bb6870ff 100644 --- a/frontend/src/components/UserManagmentTab.jsx +++ b/frontend/src/components/UserManagmentTab.jsx @@ -248,6 +248,14 @@ const UserManagmentTab = memo((props) => { return; } + if (event.target.value.includes("ALL")) { + toast.info("Adding to available all sub-organizations. This may take a minute.") + event.target.value = selectedOrganization.child_orgs.map((org) => org.id) + } else if (event.target.value.includes("None")) { + toast.info("Removing from all sub-organizations. This may take a minute") + event.target.value = [] + } + console.log("event: ", event.target.value); setMatchingOrganizations(event.target.value); // Workaround for empty orgs @@ -286,6 +294,14 @@ const UserManagmentTab = memo((props) => { }} MenuProps={MenuProps} > + + + + + + + + {selectedOrganization.child_orgs.map((org, index) => ( -1} /> diff --git a/frontend/src/components/WorkflowValidationTimeline.jsx b/frontend/src/components/WorkflowValidationTimeline.jsx index 78e6b766..2d724cfc 100644 --- a/frontend/src/components/WorkflowValidationTimeline.jsx +++ b/frontend/src/components/WorkflowValidationTimeline.jsx @@ -608,8 +608,6 @@ const WorkflowValidationTimeline = (props) => { const ballsize = 8 const topMargin = 20 - console.log("CHIP: ", index, chipColor, chipBackground) - const chipStyle = { height: 40, minWidth: 125, diff --git a/frontend/src/views/Admin2.jsx b/frontend/src/views/Admin2.jsx index 45d933b9..30a356e5 100644 --- a/frontend/src/views/Admin2.jsx +++ b/frontend/src/views/Admin2.jsx @@ -76,6 +76,18 @@ const Admin2 = (props) => { leads.push("tech partner"); } + if (responseJson.lead_info.integration_partner) { + leads.push("integration partner"); + } + + if (responseJson.lead_info.distribution_partner) { + leads.push("distribution partner"); + } + + if (responseJson.lead_info.service_partner) { + leads.push("service partner"); + } + if (responseJson.lead_info.creator) { leads.push("creator"); } diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 3394dfd9..2d800178 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -475,6 +475,7 @@ const AngularWorkflow = (defaultprops) => { const [authGroups, setAuthGroups] = React.useState([]) const curpath = typeof window === "undefined" || window.location === undefined ? "" : window.location.pathname; + const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; // 0 = normal, 1 = just done, 2 = normal @@ -913,7 +914,7 @@ const AngularWorkflow = (defaultprops) => { break } } else { - console.log("Found app, but no actions: ", foundapp) + //console.log("Found app, but no actions: ", foundapp) } if (cy !== undefined && cy !== null) { @@ -1794,7 +1795,6 @@ const AngularWorkflow = (defaultprops) => { const newkeys = sortByKey(responseJson.executions, "-started_at"); setWorkflowExecutions(newkeys); - const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; var tmpView = new URLSearchParams(cursearch).get("execution_id"); if (execution_id !== undefined && execution_id !== null && execution_id.length > 0 && (tmpView === undefined || tmpView === null || tmpView.length === 0)) { tmpView = execution_id; @@ -1850,7 +1850,6 @@ const AngularWorkflow = (defaultprops) => { } } } else { - const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; var tmpView = new URLSearchParams(cursearch).get("execution_id"); if (tmpView === undefined || tmpView === null || tmpView.length === 0) { const execution_id = tmpView; @@ -1893,7 +1892,6 @@ const AngularWorkflow = (defaultprops) => { //toast("Failed loading the workflow run") console.log("Status not 200 for stream results :O!"); - //const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; //const newitem = removeParam("execution_id", cursearch); //navigate(curpath + newitem) } @@ -2821,9 +2819,7 @@ const AngularWorkflow = (defaultprops) => { } // Based on the previous execution id - console.log("WORKFLOW EXEC: ", workflowExecutions) // Look for the "execution_id" parameter - const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; const execFound = new URLSearchParams(cursearch).get("execution_id"); if (execFound !== undefined && execFound !== null && execFound.length > 0) { toast.info("Rerunning based on previously watched execution id") @@ -2878,7 +2874,7 @@ const AngularWorkflow = (defaultprops) => { return } else if (responseJson?.success === true && responseJson?.execution_id !== undefined && responseJson?.execution_id !== null && responseJson?.execution_id.length > 0) { - navigate(`?execution_id=${responseJson.execution_id}`) + navigate(`?execution_id=${responseJson.execution_id}&node=${curAction.id}&rerun=true`) setExecutionRequest({ execution_id: responseJson.execution_id, authorization: responseJson.authorization, @@ -4316,7 +4312,6 @@ const AngularWorkflow = (defaultprops) => { // Check for execution_id in URL // don't redirect if it exists - const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; var execFound = new URLSearchParams(cursearch).get("execution_id"); var sessionToken = new URLSearchParams(cursearch).get("session_token"); if (execFound === null && sessionToken === null) { @@ -4978,7 +4973,7 @@ const AngularWorkflow = (defaultprops) => { } } - if (nodedata.app_name === "Webhook" || nodedata.app_name === "Schedule" || nodedata.app_name === "Gmail" || nodedata.app_name === "Office365") { + if (nodedata.app_name === "Webhook" || nodedata.app_name === "Schedule") { if (!found) { //console.log("Find amount of executions for the specific nodetype: ", nodedata.app_name, "Executions: ", workflowExecutions) // Find how many executions it has @@ -5007,6 +5002,7 @@ const AngularWorkflow = (defaultprops) => { } } else { // Readding the icon after moving the node + /* if (!found) { const iconInfo = GetIconInfo(nodedata); const svg_pin = ``; @@ -5034,6 +5030,7 @@ const AngularWorkflow = (defaultprops) => { } else { //console.log("Node already exists - don't add descriptor node"); } + */ } } @@ -7636,25 +7633,9 @@ const AngularWorkflow = (defaultprops) => { } break; case 86: - if (event.ctrlKey) { - //console.log("CTRL+V") - // The below parts are handled in the function handlePaste() - /* - const clipboard = navigator.clipboard - if (clipboard === undefined || window === undefined || window === null) { - toast("Can only use cliboard over HTTPS (port 3443)") - return - } - - console.log("CLIPBOARD: ", window.clipboardData) - const pastedData = window.clipboardData.getData('Text'); - console.log("PASTED: ", pastedData) - - - //var tmpAuth = JSON.parse(JSON.stringify(appAuthentication)) - var jsonvalid = true - var parsedjson = [] - */ + console.log("CTRL+V? ctrl: ", event.ctrlKey) + if (event.ctrlKey) { + // Paste is handled in the handlePaste() function. } break; case 88: @@ -7678,30 +7659,28 @@ const AngularWorkflow = (defaultprops) => { }; const handlePaste = (event) => { - if ( - event.path !== undefined && - event.path !== null && - event.path.length > 0 - ) { + console.log("PASTE EVENT: ", event) + if (event.path !== undefined && event.path !== null && event.path.length > 0) { if (event.path[0].localName !== "body") { - return; + console.log("Skipping paste because body is not targeted") + return; } } - if ( - event.target !== undefined && - event.target !== null - ) { + console.log("Paste target: ", event?.target) + /* + if (event.target !== undefined && event.target !== null) { if (event.target.localName !== "body") { + console.log("Skipping paste because body is not targeted (2). Target: ", event?.target?.localName) return; } } + */ - - event.preventDefault(); - const clipboard = (event.originalEvent || event).clipboardData.getData( - "text/plain" - ); + // Does this stop things? + //event.preventDefault() + const clipboard = (event.originalEvent || event).clipboardData.getData("text/plain") + console.log("CLIPBOARD TO PASTE: ", clipboard) try { const allnodes = cy.nodes().jsons() @@ -9710,8 +9689,6 @@ const AngularWorkflow = (defaultprops) => { setLeftSideBarOpenByClick(false) localStorage.setItem("expandLeftNav", false) - const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; - // FIXME: Don't check specific one here const tmpExec = new URLSearchParams(cursearch).get("execution_highlight"); if ( @@ -11553,7 +11530,7 @@ const AngularWorkflow = (defaultprops) => { const positionInfo = document.activeElement.getBoundingClientRect() const outerlistitemStyle = { - width: "100%", + width: "90%", overflowX: "hidden", overflowY: "hidden", borderBottom: "1px solid rgba(255,255,255,0.4)", @@ -15106,28 +15083,6 @@ const AngularWorkflow = (defaultprops) => { Select a workflow to run
- - {workflow.triggers[selectedTriggerIndex].parameters[0].value - .length === 0 ? null : workflow.triggers[selectedTriggerIndex] - .parameters[0].value === props.match.params.key ? - null - : ( -
- - - -
- )}
{workflows === undefined || @@ -15239,13 +15194,36 @@ const AngularWorkflow = (defaultprops) => { }} renderInput={(params) => { return ( - - ); +
+ + {workflow.triggers[selectedTriggerIndex].parameters[0].value + .length === 0 ? null : workflow.triggers[selectedTriggerIndex] + .parameters[0].value === props.match.params.key ? + null + : ( +
+ + + +
+ )} +
+ ) }} /> )} @@ -17465,12 +17443,32 @@ const AngularWorkflow = (defaultprops) => { }} renderInput={(params) => { return ( - +
+ + + {subworkflow === null || subworkflow === undefined || subworkflow?.id === undefined || subworkflow?.id === null || subworkflow?.id.length === 0 ? null : + + + + + + } +
); }} /> @@ -17542,7 +17540,6 @@ const AngularWorkflow = (defaultprops) => { workflow?.triggers[selectedTriggerIndex].parameters[2] && workflow?.triggers[selectedTriggerIndex].parameters[2].value && ( - workflow?.triggers[selectedTriggerIndex].parameters[2].value.includes("email") || workflow?.triggers[selectedTriggerIndex].parameters[2].value.includes("sms") ) ? ( { if (!workflow.public && executionModalOpen) { setExecutionRunning(false); stop() - const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; const newitem = removeParam("execution_id", cursearch); navigate(curpath + newitem) setExecutionModalView(0); @@ -19359,7 +19355,6 @@ const AngularWorkflow = (defaultprops) => { if (!workflow.public && executionModalOpen) { setExecutionRunning(false); stop() - const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; const newitem = removeParam("execution_id", cursearch); navigate(curpath + newitem) setExecutionModalView(0); @@ -20424,7 +20419,7 @@ const AngularWorkflow = (defaultprops) => { > @@ -20930,6 +20925,7 @@ const AngularWorkflow = (defaultprops) => { const envStatus = !(executionData.workflow !== undefined && executionData.workflow !== null && executionData.workflow.actions !== undefined && executionData.workflow.actions !== null && executionData.workflow.actions.length > 0) ? "loading" : "success" var executionDelay = -75 + const executionModal = ( { onClose={() => { setExecutionModalOpen(false) - //const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; //const newitem = removeParam("execution_id", cursearch); //navigate(curpath + newitem) }} @@ -21369,7 +21364,6 @@ const AngularWorkflow = (defaultprops) => {

{ - const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; const newitem = removeParam("execution_id", cursearch); navigate(curpath + newitem) setExecutionRunning(false); @@ -21397,7 +21391,7 @@ const AngularWorkflow = (defaultprops) => { Rerun workflow. Uses same startnode as the original. Runs from scratch. } - placement="top" + placement="left" style={{ zIndex: 50000 }} > @@ -21752,6 +21746,7 @@ const AngularWorkflow = (defaultprops) => { } /> ) : null} +
{executionData.status !== undefined && @@ -21774,6 +21769,7 @@ const AngularWorkflow = (defaultprops) => { ) : null}
+ { executionData.results === undefined || executionData.results === null || @@ -21794,6 +21790,14 @@ const AngularWorkflow = (defaultprops) => { return null; } + const showRerun = new URLSearchParams(cursearch).get("rerun") + if (showRerun === "true") { + const showNode = new URLSearchParams(cursearch).get("node") + if (data.action.id !== showNode) { + return null + } + } + // FIXME: The latter replace doens't really work if ' is used in a string var showResult = data.result.trim(); const validate = validateJson(showResult); @@ -21873,10 +21877,10 @@ const AngularWorkflow = (defaultprops) => { ); } - if (data.action.app_name === "User Input") { + if (data?.action?.app_name === "User Input" || data?.action?.name === "run_userinput") { actionimg = ( {"Shuffle { } } - const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; const chosenNodeId = new URLSearchParams(cursearch).get("node"); const highlightNode = chosenNodeId !== null && chosenNodeId !== undefined && chosenNodeId !== "" && chosenNodeId === data.action.id var relevant_errors = [] @@ -22081,7 +22084,7 @@ const AngularWorkflow = (defaultprops) => { color="primary" title={ - Expand result window. Errors: {relevant_errors.length} + Expand debug window. Errors: {relevant_errors.length} } placement="top" @@ -22720,7 +22723,7 @@ const AngularWorkflow = (defaultprops) => { {curapp === null ? null : ( {selectedResult.action.app_name} { // Automatically mapping fields that already exist (predefined). // Warning if fields are NOT filled for (let paramkey in selectedApp.authentication.parameters) { - if ( - authenticationOption.fields[ - selectedApp.authentication.parameters[paramkey].name - ].length === 0 - ) { + if (authenticationOption.fields[selectedApp.authentication.parameters[paramkey].name].length === 0) { + if ( selectedApp.authentication.parameters[paramkey].value !== undefined && selectedApp.authentication.parameters[paramkey].value !== null && @@ -23538,8 +23538,30 @@ const AngularWorkflow = (defaultprops) => { var newAuthOption = JSON.parse(JSON.stringify(authenticationOption)); var newFields = []; + + var warningsent = false for (let authkey in newAuthOption.fields) { - const value = newAuthOption.fields[authkey]; + var value = newAuthOption.fields[authkey]; + + if (value?.toLowerCase().includes("secret. replace")) { + value = "" + + if (authkey === "url") { + // Use default value of the url + const urlparam = selectedApp.authentication.parameters.find((data) => data.name === "url") + if (urlparam !== undefined && urlparam !== null) { + if (urlparam.example !== undefined && urlparam.example !== null && urlparam.example.length > 0) { + value = urlparam.example + } + } + } else { + if (!warningsent) { + warningsent = true + toast("Warning: As you didn't fill in all fields, be aware that the authentication may fail.") + } + } + } + newFields.push({ "key": authkey, "value": value, @@ -23692,18 +23714,18 @@ const AngularWorkflow = (defaultprops) => { }} fullWidth type={ - data.example !== undefined && data.example.includes("***") + data.example !== undefined && data.example.includes("**") ? "password" : "text" } color="primary" defaultValue={ - data.value !== undefined && data.value !== null && !data.value.includes("Secret. Replace") ? data.value : "" + data.value !== undefined && data.value !== null && !data.value.includes("Secret. Replace") ? data.value : + data?.example !== undefined && data?.example !== null && data?.example !== "" && !data?.example.includes("*") ? data.example : "" } placeholder={data.example} onChange={(event) => { - authenticationOption.fields[data.name] = - event.target.value; + authenticationOption.fields[data.name] = event.target.value; }} id={`${data.name}_auth`} /> diff --git a/frontend/src/views/RunWorkflow.jsx b/frontend/src/views/RunWorkflow.jsx index 919d3551..6a9e7442 100644 --- a/frontend/src/views/RunWorkflow.jsx +++ b/frontend/src/views/RunWorkflow.jsx @@ -19,6 +19,7 @@ import RecentWorkflow from "../components/RecentWorkflow.jsx"; import { Tooltip, + Fade, Select, IconButton, CircularProgress, @@ -462,6 +463,8 @@ const RunWorkflow = (defaultprops) => { } else { console.log("Started execution") + start() + setExecutionRunning(true); if (answer !== undefined && answer !== null) { console.log("Skipping start") } else { @@ -1055,7 +1058,7 @@ const RunWorkflow = (defaultprops) => { if (parsedresult.click_info !== undefined && parsedresult.click_info !== null) { if (parsedresult.click_info.user !== undefined && parsedresult.click_info.user !== null && parsedresult.click_info.user.length > 0) { - setMessage("Already answered by " + parsedresult.click_info.user) + setMessage("Answered by " + parsedresult.click_info.user) } } else { @@ -1325,7 +1328,8 @@ const RunWorkflow = (defaultprops) => { })} : - answer !== undefined && answer !== null ? null : + (answer !== undefined && answer !== null) || message !== "" ? null : + Runtime Argument
@@ -1334,7 +1338,13 @@ const RunWorkflow = (defaultprops) => { style={{backgroundColor: theme.palette.inputColor, marginTop: 5, }} multiLine maxRows={2} + type="text" + autoComplete="off" InputProps={{ + autocomplete: "off", + form: { + autocomplete: "off", + }, style:{ height: "50px", color: "white", @@ -1375,9 +1385,11 @@ const RunWorkflow = (defaultprops) => { {message}. You may close this window. : - - {disabledButtons ? "Answered. You may close this window." : ""} - + + + {disabledButtons ? "Answered. You may close this window." : ""} + + } {disabledButtons ? null : @@ -1387,14 +1399,22 @@ const RunWorkflow = (defaultprops) => { }
- + onSubmit(null, execution_id, authorization, true) + }}> + Continue  or  diff --git a/frontend/src/views/Workflows2.jsx b/frontend/src/views/Workflows2.jsx index 45d598ee..e59507f5 100644 --- a/frontend/src/views/Workflows2.jsx +++ b/frontend/src/views/Workflows2.jsx @@ -661,7 +661,7 @@ const Workflows2 = (props) => { const [mouseHoverIndex, setMouseHoverIndex] = useState(-1); const [isLoadingWorkflow, setIsLoadingWorkflow] = useState(false); const [isLoadingPublicWorkflow, setIsLoadingPublicWorkflow] = useState(false); - const [view, setView] = useState("grid"); + const [view, setView] = useState(localStorage?.getItem("workflowView") || "grid"); const classes = useStyles(theme) const imgSize = 60; @@ -2074,55 +2074,6 @@ const Workflows2 = (props) => { }; const hasWorkflows = workflows === undefined || workflows === null || workflows.length === 0 - const NewWorkflowPaper = () => { - const [hover, setHover] = React.useState(false); - - const innerColor = "rgba(255,255,255,0.3)" - - const setupPaperStyle = { - minHeight: paperAppStyle.minHeight, - maxWidth: "100%", - minWidth: paperAppStyle.width, - color: innerColor, - padding: paperAppStyle.padding, - display: "flex", - boxSizing: "border-box", - position: "relative", - border: hasWorkflows ? `2px solid #f85a3e` : `2px solid ${innerColor}`, - cursor: "pointer", - backgroundColor: hover ? "rgba(39,41,45,0.5)" : "rgba(39,41,45,1)", - borderRadius: paperAppStyle.borderRadius, - } - - return ( - - { - setModalOpen(true) - setIsEditing(false) - }} - onMouseOver={() => { - setHover(true); - }} - onMouseOut={() => { - setHover(false); - }} - > - - - - - New Workflow - - - - - - ); - }; - const getWorkflowAppgroup = (data) => { if (currTab !== 2) { if (data.actions === undefined || data.actions === null) { @@ -2517,6 +2468,7 @@ const Workflows2 = (props) => { /> : null} + { to={ type === "public" ? parsedUrl : data.workflow_as_code ? `/workflows/${data.id}/code` : `/workflows/${data.id}` } - style={{ textDecoration: "none", color: "inherit" }} - > + style={{ textDecoration: "none", color: "inherit", overflow: "hidden", textOverflow: "ellipsis", whiteSpace: "nowrap", maxWidth: "90%", display: "block" }} > {parsedName} diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 0b35fe90..8e581980 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -4,7 +4,7 @@ go 1.23.0 toolchain go1.23.6 -//replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared +replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared require ( github.com/docker/docker v27.5.0+incompatible diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 18f3b9aa..da9b51d1 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -787,15 +787,19 @@ func handleBackendImageDownload(ctx context.Context, images string) error { newImages = append(newImages, curimage) // Force remove the current image to avoid cached layers - _, err := dockercli.ImageRemove(ctx, curimage, image.RemoveOptions{ - Force: true, - PruneChildren: true, - }) + if swarmConfig == "run" || swarmConfig == "swarm" { + _, err := dockercli.ImageRemove(ctx, curimage, image.RemoveOptions{ + Force: true, + PruneChildren: true, + }) - if err != nil { - log.Printf("[ERROR] Failed removing image for re-download: %s", err) + if err != nil { + log.Printf("[ERROR] Failed removing image for re-download: %s", err) + } else { + log.Printf("[DEBUG] Removed image: %s", curimage) + } } else { - log.Printf("[DEBUG] Removed image: %s", curimage) + log.Printf("[DEBUG] Skipping image removal for %s as swarmConfig is not set to run or swarm. Value: %#v", curimage, swarmConfig) } err = shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, curimage) @@ -2261,9 +2265,6 @@ func main() { log.Printf("[INFO] Re-downloading new image(s): %#v", incRequest.ExecutionArgument) if len(incRequest.ExecutionArgument) > 0 { - // FIXME: Wait X seconds before running this as the image build may not be done yet. This is shitty, but may be ok to do in Orborus. Easy fix for the future: Just let it run through jobs 5-10 times before actually picking it up - - // Run after 25 seconds in the goroutine instead go handleBackendImageDownload(ctx, incRequest.ExecutionArgument) } else { log.Printf("[ERROR] No image name provided for download. Removing job from queue.") From ff6b970aef1265fcf793921a3a1abf5b4b35c346 Mon Sep 17 00:00:00 2001 From: Frikky Date: Tue, 1 Apr 2025 22:33:25 +0200 Subject: [PATCH 22/33] Minor fixes everywhere --- backend/go-app/go.mod | 2 +- backend/go-app/go.sum | 4 +- backend/go-app/walkoff.go | 5 + frontend/src/components/AdminNavBar.jsx | 3 + frontend/src/components/Navbar.jsx | 2 +- frontend/src/components/OrganizationTab.jsx | 4 +- frontend/src/components/ParsedAction.jsx | 15 +- .../src/components/ShuffleCodeEditor1.jsx | 56 +- frontend/src/components/TenantsTab.jsx | 880 ++++++++++-------- frontend/src/views/AngularWorkflow.jsx | 25 +- frontend/src/views/Docs.jsx | 11 +- frontend/src/views/RunWorkflow.jsx | 176 +++- frontend/src/views/Workflows2.jsx | 58 +- functions/onprem/worker/go.mod | 2 +- functions/onprem/worker/go.sum | 4 +- 15 files changed, 791 insertions(+), 456 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 1a07e775..2b8f075b 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -20,7 +20,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.19 + github.com/shuffle/shuffle-shared v0.8.32 golang.org/x/crypto v0.36.0 google.golang.org/api v0.176.1 google.golang.org/grpc v1.68.1 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 1214c85c..5610d4db 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -333,8 +333,8 @@ github.com/sendgrid/sendgrid-go v3.14.0+incompatible h1:KDSasSTktAqMJCYClHVE94Fc github.com/sendgrid/sendgrid-go v3.14.0+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.19 h1:HXqU62sPhVzv9MeJnA5ZpPYwwbitVz1EtMMABbY3t74= -github.com/shuffle/shuffle-shared v0.8.19/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= +github.com/shuffle/shuffle-shared v0.8.31 h1:APO/BkBxiP9Hn/Fa7SsESRD6ws9E5dBfIzS57pd3MvA= +github.com/shuffle/shuffle-shared v0.8.31/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 93540aa8..5b44d73a 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -3041,6 +3041,11 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) { return } + shouldRerun := false + rerun, rerunOk := query["rerun"] + if rerunOk && len(rerun) > 0 && rerun[0] == "true" { + shouldRerun = true + } if shouldRerun { log.Printf("[DEBUG] Returning single action execution ID for rerun: %s", workflowExecution.ExecutionId) diff --git a/frontend/src/components/AdminNavBar.jsx b/frontend/src/components/AdminNavBar.jsx index c6f70357..e0faec6e 100644 --- a/frontend/src/components/AdminNavBar.jsx +++ b/frontend/src/components/AdminNavBar.jsx @@ -33,6 +33,9 @@ const AdminNavBar = (props) => { const navigate = useNavigate(); + //const leadinfo = selectedOrganization.lead_info === undefined || selectedOrganization.lead_info === null || selectedOrganization.lead_info === "" ? "" : JSON.stringify(selectedOrganization.lead_info) + //const isPartner = leadinfo.includes("partner") + useEffect(() => { const queryParams = new URLSearchParams(location.search); const tabName = queryParams.get('tab'); diff --git a/frontend/src/components/Navbar.jsx b/frontend/src/components/Navbar.jsx index 54362e86..2c437a8d 100644 --- a/frontend/src/components/Navbar.jsx +++ b/frontend/src/components/Navbar.jsx @@ -61,7 +61,7 @@ const menuData = { description: "Connect and run actions seamlessly between different platforms.", icon: "/images/logos/singul.svg", - path: "https://singul.io/", + path: "https://singul-docs.gitbook.io/singul/getting-started", gaData: { category: "navbar", action: "products_click", diff --git a/frontend/src/components/OrganizationTab.jsx b/frontend/src/components/OrganizationTab.jsx index 1911bfad..56a1531d 100644 --- a/frontend/src/components/OrganizationTab.jsx +++ b/frontend/src/components/OrganizationTab.jsx @@ -120,7 +120,7 @@ const OrganizationTab = (props) => { isLoaded={isLoaded} /> ); - case 'branding(beta)': + case 'branding': return { return (
- {['Org Configuration', "sso", "Notifications", 'Billing & Stats', 'Branding (Beta)'].map((tabName, index) => ( + {['Org Configuration', "sso", "Notifications", 'Billing & Stats', 'Branding'].map((tabName, index) => ( { if (foundResult) { const valid = validateJson(foundResult.result); if (valid.valid) { - if (valid.result.success !== false) { - exampleData = valid.result - break + + // Check if array, and if first item is object + success + if (Array.isArray(valid.result) && valid.result.length > 0 && typeof valid.result[0] === "object") { + if (valid.result[0].success !== false) { + exampleData = valid.result[0] + break + } + } else { + if (valid.result.success !== false) { + exampleData = valid.result + break + } } } else { secondaryExample = foundResult.result diff --git a/frontend/src/components/ShuffleCodeEditor1.jsx b/frontend/src/components/ShuffleCodeEditor1.jsx index 196d2b5e..dea674f5 100644 --- a/frontend/src/components/ShuffleCodeEditor1.jsx +++ b/frontend/src/components/ShuffleCodeEditor1.jsx @@ -902,6 +902,8 @@ const CodeEditor = (props) => { // Whelp this is inefficient af. Single loop pls // When the found array is empty. if (found !== null && found !== undefined) { + + //console.log("FOUND: ", found) try { for (var i = 0; i < found.length; i++) { try { @@ -936,28 +938,72 @@ const CodeEditor = (props) => { } } } + + // Find the location to ensure replacements happen correctly + var foundlocation = -1 + for (var j = 0; j < input.length; j++) { + const foundStringSize = fixedVariable.length + const foundslice = input.slice(j, j + foundStringSize) + //console.log("FOUNDSLICE: ", foundslice) + if (fixedVariable !== foundslice) { + continue + } + + // Check if it matches EXACTLY or not, as there may be more AFTER the found[i] + const nextchar = input.slice(j + foundStringSize, j + foundStringSize + 1) + if (nextchar === ".") { + continue + } + + foundlocation = j + break + } + // FIXME: There is something wrong here with: + // $variable.# + // vs + // $variable.#.subvalue + // if you put both of those lines in the same editor, then it will replace both (somehow). Make sure $variable.#.subvalue exists while testing. + console.log("FOUNDLOC: ", fixedVariable, foundlocation) for (var j = 0; j < actionlist.length; j++) { if (fixedVariable.slice(1,).toLowerCase() !== actionlist[j].autocomplete.toLowerCase()) { continue } + // Look for the location of found[i] in the input, as to make sure to skip parts of the input in the replace. Find ALL spots for it valuefound = true + var newvalue = "" try { - if (typeof actionlist[j].example === "object") { - input = input.replace(found[i], JSON.stringify(actionlist[j].example), -1); + if (typeof actionlist[j].example === "object") { + newvalue = JSON.stringify(actionlist[j].example) } else if (actionlist[j].example.trim().startsWith("{") || actionlist[j].example.trim().startsWith("[")) { - input = input.replace(found[i], JSON.stringify(actionlist[j].example), -1); + + newvalue = JSON.stringify(actionlist[j].example) } else { const newExample = fixStringInput(actionlist[j].example) - input = input.replace(found[i], newExample, -1) + + newvalue = newExample } } catch (e) { - input = input.replace(found[i], actionlist[j].example, -1) + newvalue = actionlist[j].example } + try { + console.log("REPLACE: ", foundlocation, fixedVariable, newvalue) + if (newvalue !== "") { + if (foundlocation === -1) { + input = input.replace(fixedVariable, newvalue, 1) + } else { + // Ensures we don't just randomly replace the first value we find + const replacedSlice = input.slice(foundlocation, input.length).replace(fixedVariable, newvalue, 1) + input = input.slice(0, foundlocation) + replacedSlice + } + } + } catch (e) { + console.log("Replace error: ", e) + } } if (!valuefound) { diff --git a/frontend/src/components/TenantsTab.jsx b/frontend/src/components/TenantsTab.jsx index ed94ebde..e5552e3e 100644 --- a/frontend/src/components/TenantsTab.jsx +++ b/frontend/src/components/TenantsTab.jsx @@ -30,6 +30,8 @@ import { Apps as AppsIcon, Business as BusinessIcon, Flag, + ArrowDropDown as ArrowDropDownIcon, + } from "@mui/icons-material"; import { toast } from 'react-toastify'; @@ -58,6 +60,8 @@ const TenantsTab = memo((props) => { const [parentOrgRegionName, setParentOrgRegionName] = React.useState("UK"); const [loadOrgs, setLoadOrgs] = React.useState(true); const [, forceUpdate] = React.useState(); + const [suborglistOpen, setSuborglistOpen] = React.useState(false); + const [allTenantsOpen, setAllTenantsOpen] = React.useState(false); const itemColor = "black"; useEffect(() => { @@ -482,8 +486,8 @@ const TenantsTab = memo((props) => { const createSubOrg = (currentOrgId, name) => { const data = { name: name, org_id: currentOrgId }; - console.log(data); const url = globalUrl + `/api/v1/orgs/${currentOrgId}/create_sub_org`; + setSuborglistOpen(true) fetch(url, { mode: "cors", @@ -791,7 +795,7 @@ const TenantsTab = memo((props) => { Create, manage and change to sub-organizations (tenants)! {" "} {isCloud ? "You can only make a sub organization if you are a customer of shuffle or running a POC of the platform. Please contact support@shuffler.io to try it out." - : ''} + : ''}  { overflowX: "auto", paddingBottom: 0, }}> - - - - {isCloud && ( - - )} - - - - {subOrgs.map((data, index) => { - let regiontag = "UK"; - let regionCode = "gb"; + {!suborglistOpen ? + 0) { - const regionsplit = data.region_url.split("."); - if (regionsplit.length > 2 && !regionsplit[0].includes("shuffler")) { - const namesplit = regionsplit[0].split("/"); - regiontag = namesplit[namesplit.length - 1]; - if (regiontag === "california") { - regiontag = "US"; - regionCode = "us"; - } else if (regiontag === "frankfurt") { - regiontag = "EU-2"; - regionCode = "eu"; - } else if (regiontag === "ca") { - regiontag = "CA"; - regionCode = "ca"; + }} + > + setSuborglistOpen(true)} + > + Show Sub-Organizations + + } + style={{ + width: 100, + minWidth: 100, + maxWidth: 100, + paddingLeft: 20, + display: "table-cell", + padding: "0px 8px 8px 8px", + textAlign: "center", + borderBottom: "1px solid #494949", + verticalAlign: "middle", + }} + /> + + : + + + + + {isCloud && ( + + )} + + + + {subOrgs.map((data, index) => { + let regiontag = "UK"; + let regionCode = "gb"; + + if (data.region_url?.length > 0) { + const regionsplit = data.region_url.split("."); + if (regionsplit.length > 2 && !regionsplit[0].includes("shuffler")) { + const namesplit = regionsplit[0].split("/"); + regiontag = namesplit[namesplit.length - 1]; + if (regiontag === "california") { + regiontag = "US"; + regionCode = "us"; + } else if (regiontag === "frankfurt") { + regiontag = "EU-2"; + regionCode = "eu"; + } else if (regiontag === "ca") { + regiontag = "CA"; + regionCode = "ca"; + } } } - } - return ( - - } style={{ width: 100, - minWidth: 100, - maxWidth: 100, - display: "table-cell", - padding: "8px 8px 8px 20px", - textAlign: "center", }} /> - + return ( + + } style={{ width: 100, + minWidth: 100, + maxWidth: 100, + display: "table-cell", + padding: "8px 8px 8px 20px", + textAlign: "center", }} /> + - {isCloud && ( - - {regiontag} - -
- } - style={{ display: "table-cell", padding: 8, verticalAlign: "middle" }} - /> - )} - + {isCloud && ( + + {regiontag} + +
+ } + style={{ display: "table-cell", padding: 8, verticalAlign: "middle" }} + /> + )} + + + + + + } + style={{ display: "table-cell", verticalAlign: "middle" }} + /> + + )})} + + } - - - - } - style={{ display: "table-cell", verticalAlign: "middle" }} - /> - - )})}
@@ -1366,247 +1414,289 @@ const TenantsTab = memo((props) => { paddingBottom: 0, }} > - - - - {isCloud && ( - - )} - - - + {!allTenantsOpen ? + ( - - {Array(7) - .fill() - .map((_, colIndex) => ( - - - - ))} - - )) - ) : ( - userdata?.orgs?.length > 0 && - userdata.orgs.map((data, index) => { - let regiontag = "UK"; - let regionCode = "gb"; - - if (data.region_url?.length > 0) { - const regionsplit = data.region_url.split("."); - if (regionsplit.length > 2 && !regionsplit[0].includes("shuffler")) { - const namesplit = regionsplit[0].split("/"); - regiontag = namesplit[namesplit.length - 1]; - - if (regiontag === "california") { - regiontag = "US"; - regionCode = "us"; - } else if (regiontag === "frankfurt") { - regiontag = "EU-2"; - regionCode = "eu"; - } else if (regiontag === "ca") { - regiontag = "CA"; - regionCode = "ca"; + }} + > + setAllTenantsOpen(true)} + > + Show ALL your tenants + } - } - } + style={{ + width: 100, + minWidth: 100, + maxWidth: 100, + paddingLeft: 20, + display: "table-cell", + padding: "0px 8px 8px 8px", + textAlign: "center", + borderBottom: "1px solid #494949", + verticalAlign: "middle", + }} + /> + + : + + + + + {isCloud && ( + + )} + + + - return ( - - - } - style={{ - width: 100, - minWidth: 100, - maxWidth: 100, - display: "table-cell", - padding: "8px 8px 8px 20px", - textAlign: "center", - }} - /> - - {isCloud ? ( - - {regiontag} + {userdata?.orgs?.length <= 0 ? ( + [...Array(6)].map((_, rowIndex) => ( + + {Array(7) + .fill() + .map((_, colIndex) => ( + + + + ))} + + )) + ) : ( + userdata?.orgs?.length > 0 && + userdata.orgs.map((data, index) => { + let regiontag = "UK"; + let regionCode = "gb"; - - - } - style={{ - display: "table-cell", - padding: 8, - verticalAlign: "middle", - }} - > - ) : null} - - { - handleClickChangeOrg(data?.id); - }} - > - Change Active Org - - } - style={{ - display: "table-cell", - padding: 8, - verticalAlign: "middle", - }} - > - - ); - }) - )} + if (data.region_url?.length > 0) { + const regionsplit = data.region_url.split("."); + if (regionsplit.length > 2 && !regionsplit[0].includes("shuffler")) { + const namesplit = regionsplit[0].split("/"); + regiontag = namesplit[namesplit.length - 1]; + + if (regiontag === "california") { + regiontag = "US"; + regionCode = "us"; + } else if (regiontag === "frankfurt") { + regiontag = "EU-2"; + regionCode = "eu"; + } else if (regiontag === "ca") { + regiontag = "CA"; + regionCode = "ca"; + } + } + } + + return ( + + + } + style={{ + width: 100, + minWidth: 100, + maxWidth: 100, + display: "table-cell", + padding: "8px 8px 8px 20px", + textAlign: "center", + }} + /> + + {isCloud ? ( + + {regiontag} + + + + } + style={{ + display: "table-cell", + padding: 8, + verticalAlign: "middle", + }} + > + ) : null} + + { + handleClickChangeOrg(data?.id); + }} + > + Change Active Org + + } + style={{ + display: "table-cell", + padding: 8, + verticalAlign: "middle", + }} + > + + ); + }) + )} + } diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 2d800178..64c7d800 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -4316,9 +4316,15 @@ const AngularWorkflow = (defaultprops) => { var sessionToken = new URLSearchParams(cursearch).get("session_token"); if (execFound === null && sessionToken === null) { - toast.error(`You don't have access to this workflow or loading failed. Redirecting to workflows in a few seconds. If you recently deleted this workflow, speak with support@shuffler.io to recover it from a revision.`, { - autoClose: 10000, - }) + if (isCloud) { + toast.error(`You don't have access to this workflow or loading failed. Redirecting to workflows in a few seconds. If you recently deleted this workflow, speak with support@shuffler.io to recover it from a revision.`, { + autoClose: 10000, + }) + } else { + toast.error(`You don't have access to this workflow or loading failed. Redirecting to workflows in a few seconds. Contact support@shuffler.io if this is unexpected.`, { + autoClose: 10000, + }) + } setTimeout(() => { window.location.pathname = "/workflows"; @@ -7659,7 +7665,6 @@ const AngularWorkflow = (defaultprops) => { }; const handlePaste = (event) => { - console.log("PASTE EVENT: ", event) if (event.path !== undefined && event.path !== null && event.path.length > 0) { if (event.path[0].localName !== "body") { console.log("Skipping paste because body is not targeted") @@ -7667,15 +7672,13 @@ const AngularWorkflow = (defaultprops) => { } } - console.log("Paste target: ", event?.target) - /* if (event.target !== undefined && event.target !== null) { - if (event.target.localName !== "body") { - console.log("Skipping paste because body is not targeted (2). Target: ", event?.target?.localName) - return; - } + // If it's an input area, skip paste + if (event.target.localName === "input" || event.target.localName === "textarea") { + console.log("Skipping paste because body is not targeted (1). Target: ", event?.target?.localName) + return; + } } - */ // Does this stop things? //event.preventDefault() diff --git a/frontend/src/views/Docs.jsx b/frontend/src/views/Docs.jsx index 90def218..3e2cfe1b 100755 --- a/frontend/src/views/Docs.jsx +++ b/frontend/src/views/Docs.jsx @@ -1114,6 +1114,7 @@ const Docs = (defaultprops) => { marginTop: 25, } + const showPartnerLogo = userdata?.org_status?.includes("integration_partner") && userdata?.active_org?.image !== undefined && userdata?.active_org?.image !== null && userdata?.active_org?.image.length > 0 const mainpageInfo =
{ Documentation -
- /> - link="https://discord.gg/B2CBzUm" /> -
+ {showPartnerLogo === true ? null : +
+ /> + link="https://discord.gg/B2CBzUm" /> +
+ }
Tutorial diff --git a/frontend/src/views/RunWorkflow.jsx b/frontend/src/views/RunWorkflow.jsx index 6a9e7442..e7c617b8 100644 --- a/frontend/src/views/RunWorkflow.jsx +++ b/frontend/src/views/RunWorkflow.jsx @@ -34,6 +34,7 @@ import { DialogTitle, DialogContent, MenuItem, + Autocomplete, } from '@mui/material'; import { @@ -45,6 +46,7 @@ import { LockOpen as LockOpenIcon, OpenInNew as OpenInNewIcon, Edit as EditIcon, + Polyline as PolylineIcon, } from '@mui/icons-material'; const hrefStyle = { @@ -75,6 +77,7 @@ const RunWorkflow = (defaultprops) => { const [sharingOpen, setSharingOpen] = React.useState(false) const [realtimeMarkdown, setRealtimeMarkdown] = React.useState("") const [forms, setForms] = React.useState([]) + const [workflows, setWorkflows] = React.useState([]) const [boxWidth, setBoxWidth] = React.useState(500) const [inputQuestions, setInputQuestions] = React.useState([]) @@ -181,6 +184,37 @@ const RunWorkflow = (defaultprops) => { return true } + const getWorkflows = () => { + const url = `${globalUrl}/api/v1/workflows` + fetch(url, { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for org forms"); + } + + return response.json() + }) + .then((responseJson) => { + if (responseJson.success === false) { + toast.error("Failed saving workflow. Please try again.") + } else { + if (responseJson?.length > 0) { + setWorkflows(responseJson) + } + } + }) + .catch((error) => { + //toast.error("Load form error: " + error) + }) + } + const loadForms = (orgId) => { const url = `${globalUrl}/api/v1/orgs/${orgId}/forms` fetch(url, { @@ -878,6 +912,7 @@ const RunWorkflow = (defaultprops) => { // Just use this one? var url = execution_id !== undefined && authorization !== undefined ? `${globalUrl}/api/v1/orgs/${orgId}?reference_execution=${execution_id}&authorization=${authorization}` : `${globalUrl}/api/v1/orgs/${orgId}`; + getWorkflows() loadForms(orgId) fetch(url, { @@ -1152,10 +1187,100 @@ const RunWorkflow = (defaultprops) => { No Forms Found - Every Workflow is a form, and can be accessed by going to /forms/{`{workflow_id}`}. You can control the form by editing the workflow details in the "Forms" section. - + ALL Workflows are forms, and can be accessed by going to /forms/{`{workflow_id}`}. You can control the form by editing the workflow details in the "Forms" section. +
} + + {workflows === undefined || workflows === null || workflows.length === 0 ? null : + option.id === value.id} + getOptionLabel={(option) => { + if ( + option === undefined || + option === null || + option.name === undefined || + option.name === null + ) { + return "No Workflow Selected"; + } + + const newname = ( + option.name.charAt(0).toUpperCase() + option.name.substring(1) + ).replaceAll("_", " "); + return newname; + }} + options={workflows} + fullWidth + style={{ + backgroundColor: theme.palette.inputColor, + borderRadius: theme.palette?.borderRadius, + marginTop: 75, + }} + renderOption={(props, data, state) => { + if (data.id === workflow.id) { + data = workflow; + } + + //key={index} + return ( + + {data.image !== undefined && data.image !== null && data.image.length > 0 ? + {data.name} + : null} + + Choose Subflow '{data.name}' + + + }> + { + window.location.href = `/forms/${data.id}` + }} + value={data} + > + + {data.name} + + + ) + }} + renderInput={(params) => { + return ( +
+ +
+ ) + }} + /> + }
) } @@ -1405,7 +1530,10 @@ const RunWorkflow = (defaultprops) => { variant="contained" disabled={!handleValidateForm(executionArgument) || disabledButtons} color="primary" - style={{flex: 1,}} + style={{ + flex: 1, + textTransform: "none", + }} onClick={() => { setButtonClicked("FINISHED") setExecutionData({ @@ -1418,14 +1546,25 @@ const RunWorkflow = (defaultprops) => {  or  - + onSubmit(null, execution_id, authorization, false) + }}> + Stop + : @@ -1436,6 +1575,9 @@ const RunWorkflow = (defaultprops) => { color="primary" fullWidth disabled={!handleValidateForm(executionArgument) || executionLoading} + style={{ + textTransform: "none", + }} > {executionLoading ? @@ -1621,7 +1763,10 @@ const RunWorkflow = (defaultprops) => { disabled={workflow.id === undefined || workflow.id === null} variant={"outlined"} color={"secondary"} - style={{marginRight: 10, }} + style={{ + marginRight: 10, + textTransform: "none", + }} onClick={() => { window.open(`/workflows/${workflow.id}`, "_blank") }} @@ -1634,7 +1779,10 @@ const RunWorkflow = (defaultprops) => { disabled={workflow.id === undefined || workflow.id === null} variant={workflow.sharing === "form" ? "outlined" : "contained"} color={"secondary"} - style={{marginRight: 10, }} + style={{ + marginRight: 10, + textTransform: "none", + }} onClick={() => { setSharingOpen(true) }} @@ -1656,7 +1804,9 @@ const RunWorkflow = (defaultprops) => { disabled={workflow.id === undefined || workflow.id === null} variant={"contained"} color={"primary"} - style={{}} + style={{ + textTransform: "none", + }} onClick={() => { setEditWorkflowModalOpen(true) }} diff --git a/frontend/src/views/Workflows2.jsx b/frontend/src/views/Workflows2.jsx index e59507f5..c4b80b46 100644 --- a/frontend/src/views/Workflows2.jsx +++ b/frontend/src/views/Workflows2.jsx @@ -812,9 +812,10 @@ const Workflows2 = (props) => { } - const isCloud = - window.location.host === "localhost:3002" || - window.location.host === "shuffler.io"; + //const isCloud = + // window.location.host === "localhost:3002" || + // window.location.host === "shuffler.io"; + const isCloud = false const findWorkflow = (filters) => { console.log("Using filters: ", filters) @@ -2474,16 +2475,18 @@ const Workflows2 = (props) => { style={{ display: "flex", flexDirection: "column", width: "100%", fontFamily: theme?.typography?.fontFamily }} > - -
{ - navigate("/admin") - }} - > - {image} -
- + {currTab === 2 ? null : + +
{ + navigate("/admin") + }} + > + {image} +
+ + } { /* @@ -2562,9 +2565,19 @@ const Workflows2 = (props) => { > + style={{ + textDecoration: "none", + color: "inherit", + overflow: "hidden", + textOverflow: "ellipsis", + whiteSpace: "nowrap", + maxWidth: "90%", + display: "block" + }} + target={currTab === 2 ? "_blank" : "_self"} + > {parsedName} @@ -4261,7 +4274,7 @@ const Workflows2 = (props) => { TabIndicatorProps={{ style: { display: 'none' } }} > { ...(currTab === 2 ? tabActive : {}) }} /> + + { + navigate("/forms") + }} + style={{ + ...tabStyle, + marginRight: 0, + marginLeft: 25, + ...(currTab === 3 ? tabActive : {}) + }} + /> diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 035ac090..23f27d64 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -8,7 +8,7 @@ require ( github.com/docker/docker v27.5.0+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.19 + github.com/shuffle/shuffle-shared v0.8.31 k8s.io/api v0.30.2 k8s.io/apimachinery v0.30.2 k8s.io/client-go v0.30.2 diff --git a/functions/onprem/worker/go.sum b/functions/onprem/worker/go.sum index 028e4dd8..88c9ba70 100644 --- a/functions/onprem/worker/go.sum +++ b/functions/onprem/worker/go.sum @@ -294,8 +294,8 @@ github.com/sendgrid/sendgrid-go v3.14.0+incompatible h1:KDSasSTktAqMJCYClHVE94Fc github.com/sendgrid/sendgrid-go v3.14.0+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.19 h1:HXqU62sPhVzv9MeJnA5ZpPYwwbitVz1EtMMABbY3t74= -github.com/shuffle/shuffle-shared v0.8.19/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= +github.com/shuffle/shuffle-shared v0.8.31 h1:APO/BkBxiP9Hn/Fa7SsESRD6ws9E5dBfIzS57pd3MvA= +github.com/shuffle/shuffle-shared v0.8.31/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= From 2e40266171f6dfa154b1935337e2b5e9f37b1735 Mon Sep 17 00:00:00 2001 From: Frikky Date: Wed, 2 Apr 2025 12:20:48 +0200 Subject: [PATCH 23/33] Fixed a few minor problems like queue confirms working well for environments with spaces --- backend/go-app/go.mod | 2 +- backend/go-app/go.sum | 4 +-- backend/go-app/walkoff.go | 4 ++- frontend/src/components/Navbar.jsx | 2 +- frontend/src/views/LoginPage.jsx | 2 +- functions/onprem/orborus/orborus.go | 51 ++++++++++++++++------------- 6 files changed, 36 insertions(+), 29 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 2b8f075b..6822373e 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -4,7 +4,7 @@ go 1.23.0 toolchain go1.23.7 -//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared +replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared require ( cloud.google.com/go/datastore v1.15.0 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 5610d4db..01efec32 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -333,8 +333,8 @@ github.com/sendgrid/sendgrid-go v3.14.0+incompatible h1:KDSasSTktAqMJCYClHVE94Fc github.com/sendgrid/sendgrid-go v3.14.0+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.31 h1:APO/BkBxiP9Hn/Fa7SsESRD6ws9E5dBfIzS57pd3MvA= -github.com/shuffle/shuffle-shared v0.8.31/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= +github.com/shuffle/shuffle-shared v0.8.32 h1:RizHZ4lZg9wQj3RQvJMgh+1SAMszwhYfcco3aT716RQ= +github.com/shuffle/shuffle-shared v0.8.32/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 5b44d73a..317595c3 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -207,8 +207,10 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque return } + log.Printf("Queue confirm: %#v. Queue: %#v", removeExecutionRequests.Data, id) + // remove items from DB - parsedId := fmt.Sprintf("workflowqueue-%s", id) + parsedId := strings.ReplaceAll(fmt.Sprintf("workflowqueue-%s", id), " ", "-") ids := []string{} for _, execution := range removeExecutionRequests.Data { ids = append(ids, execution.ExecutionId) diff --git a/frontend/src/components/Navbar.jsx b/frontend/src/components/Navbar.jsx index 2c437a8d..fa038a51 100644 --- a/frontend/src/components/Navbar.jsx +++ b/frontend/src/components/Navbar.jsx @@ -1247,7 +1247,7 @@ const Navbar = (props) => { letterSpacing: '0.5px', }} > - Coming Soon + Beta: Coming Soon )} diff --git a/frontend/src/views/LoginPage.jsx b/frontend/src/views/LoginPage.jsx index 9bcc6966..9a04b959 100755 --- a/frontend/src/views/LoginPage.jsx +++ b/frontend/src/views/LoginPage.jsx @@ -375,7 +375,7 @@ const LoginPage = props => { padding: "40px", flex: 1, maxWidth: isMobile ? "100%" : 410, - minWidth: 410, + minWidth: isCloud ? 410 : 475, background: "#212121", borderRadius: "12px", display: "flex", diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 291c861d..b5a3272b 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -1,13 +1,9 @@ package main /* - Orborus exists to listen for new jobs which are deployed as workers. + Orborus exists to listen for new jobs from Shuffle. This is to run workflows, pipelines, and other tasks. */ -// Potential issues: -// Default network could be same as on the host -// Ingress network may not exist (default) - import ( "archive/zip" "bytes" @@ -88,7 +84,6 @@ var appServiceAccountName = os.Getenv("SHUFFLE_APP_SERVICE_ACCOUNT_NAME") // var baseimagename = "shuffle/shuffle" var baseimagename = os.Getenv("SHUFFLE_BASE_IMAGE_NAME") var baseimageregistry = os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY") - //var baseimagetagsuffix = os.Getenv("SHUFFLE_BASE_IMAGE_TAG_SUFFIX") // Used for cloud with auth @@ -781,7 +776,7 @@ func handleBackendImageDownload(ctx context.Context, images string) error { log.Printf("[DEBUG] Skipping image removal for %s as swarmConfig is not set to run or swarm. Value: %#v", curimage, swarmConfig) } - err = shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, curimage) + err := shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, curimage) if err != nil { log.Printf("[ERROR] Failed downloading image: %s", err) } else { @@ -1340,11 +1335,11 @@ func deployWorker(image string, identifier string, env []string, executionReques ) if err != nil { - log.Printf("[ERROR] Container create error(2): %s", err) + log.Printf("[ERROR][%s] Container create error(2): %s", executionRequest.ExecutionId, err) return err } } else { - log.Printf("[ERROR] Container create error: %s", err) + log.Printf("[ERROR][%s] Container create error: %s", executionRequest.ExecutionId, err) return err } } @@ -1369,19 +1364,19 @@ func deployWorker(image string, identifier string, env []string, executionReques identifier+"-2", ) if err != nil { - log.Printf("[ERROR] Failed to CREATE container (2): %s", err) + log.Printf("[ERROR][%s] Failed to CREATE container (2): %s", executionRequest.ExecutionId, err) } err = dockercli.ContainerStart(context.Background(), cont.ID, containerStartOptions) if err != nil { - log.Printf("[ERROR] Failed to start container (2): %s", err) + log.Printf("[ERROR][%s] Failed to start container (2): %s", executionRequest.ExecutionId, err) } } else { - log.Printf("[ERROR] Failed initial container start. Quitting as this is NOT a simple network issue. Err: %s", err) + log.Printf("[ERROR][%s] Failed initial container start. Quitting as this is NOT a simple network issue. Err: %s", executionRequest.ExecutionId, err) } if err != nil { - log.Printf("[ERROR] Failed to start worker container in environment '%s': %s", environment, err) + log.Printf("[ERROR][%s] Failed to start worker container in environment '%s': %s", executionRequest.ExecutionId, environment, err) return err } else { log.Printf("[INFO][%s] Worker Container created (2). Environment %s: docker logs %s", executionRequest.ExecutionId, environment, cont.ID) @@ -1389,13 +1384,13 @@ func deployWorker(image string, identifier string, env []string, executionReques stats, err := dockercli.ContainerInspect(ctx, cont.ID) if err != nil { - log.Printf("[WARNING] Failed checking worker '%s': %s", cont.ID, err) + log.Printf("[WARNING][%s] Failed checking worker '%s': %s", executionRequest.ExecutionId, cont.ID, err) return nil } containerStatus := stats.ContainerJSONBase.State.Status if containerStatus != "running" { - log.Printf("[ERROR] Status of %s is %s. Should be running. Contact support@shuffler.io if this persists.", cont.ID, containerStatus) + log.Printf("[ERROR][%s] Status of %s is %s. Should be running. Contact support@shuffler.io if this persists.", executionRequest.ExecutionId, cont.ID, containerStatus) } /* err = stopWorker(containerName) @@ -1459,17 +1454,30 @@ func initializeImages() { if baseimageregistry == "" { baseimageregistry = "docker.io" // Dockerhub baseimageregistry = "ghcr.io" // Github - log.Printf("[DEBUG] Setting baseimageregistry to %#v", baseimageregistry) + + if len(os.Getenv("REGISTRY_URL")) > 0 { + baseimageregistry = os.Getenv("REGISTRY_URL") + } else { + os.Setenv("REGISTRY_URL", baseimageregistry) + } + + os.Setenv("SHUFFLE_BASE_IMAGE_REGISTRY", baseimageregistry) + + log.Printf("[WARNING] Setting baseimageregistry to %#v", baseimageregistry) } if baseimagename == "" { + // FIXME: This is probably the problem for image names tbh baseimagename = "frikky/shuffle" // Dockerhub baseimagename = "shuffle" // Github (ghcr.io) - log.Printf("[DEBUG] Setting baseimagename to %#v", baseimagename) + + os.Setenv("SHUFFLE_BASE_IMAGE_NAME", baseimagename) + log.Printf("[WARNING] Setting baseimagename to %#v", baseimagename) } log.Printf("[DEBUG] Setting swarm config to %#v. Default is empty.", swarmConfig) + // FIXME: Shuffle Worker vs Apps != same newWorker := fmt.Sprintf("%s/%s/shuffle-worker:%s", baseimageregistry, baseimagename, workerVersion) if len(newWorkerImage) > 0 { newWorker = newWorkerImage @@ -2412,11 +2420,11 @@ func main() { } if execution.Status == "ABORT" || execution.Status == "FAILED" { - log.Printf("[INFO] Executionstatus issue: ", execution.Status) + log.Printf("[INFO][%s] Executionstatus issue: ", execution.ExecutionId, execution.Status) } if shuffle.ArrayContains(executionIds, execution.ExecutionId) { - log.Printf("[INFO] Execution already handled (rerun of old executions?): %s", execution.ExecutionId) + log.Printf("[INFO][%s] Execution already handled (rerunning old execution)", execution.ExecutionId) toBeRemoved.Data = append(toBeRemoved.Data, execution) // Should check when last this was ran, and if it's more than 10 minutes ago and it's not finished, we should run it again? @@ -2506,7 +2514,7 @@ func main() { toBeRemoved.Data = append(toBeRemoved.Data, execution) executionIds = append(executionIds, execution.ExecutionId) } else { - log.Printf("[WARNING] Execution ID '%s' failed to deploy: %s", execution.ExecutionId, err) + log.Printf("[WARNING][%s] Failed to deploy: %s", execution.ExecutionId, err) if strings.Contains(err.Error(), "already exists") { toBeRemoved.Data = append(toBeRemoved.Data, execution) executionIds = append(executionIds, execution.ExecutionId) @@ -3638,15 +3646,12 @@ func zombiecheck(ctx context.Context, workerTimeout int) error { All: true, }) - //log.Printf("Len: %d", len(containers)) - if err != nil { log.Printf("[ERROR] Failed creating Containerlist: %s", err) return err } containerNames := map[string]string{} - stopContainers := []string{} removeContainers := []string{} log.Printf("[INFO] Baseimage: %s, Workertimeout: %d", baseimagename, int64(workerTimeout)) From abbdca95672e88e5c84170c6431ae3fbb734b234 Mon Sep 17 00:00:00 2001 From: Frikky Date: Wed, 2 Apr 2025 14:24:35 +0200 Subject: [PATCH 24/33] Sane defaults for comment position --- frontend/src/defaultCytoscapeStyle.jsx | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/frontend/src/defaultCytoscapeStyle.jsx b/frontend/src/defaultCytoscapeStyle.jsx index 723ad938..8a2623f6 100644 --- a/frontend/src/defaultCytoscapeStyle.jsx +++ b/frontend/src/defaultCytoscapeStyle.jsx @@ -114,8 +114,22 @@ const data = [ "background-opacity": "0.5", "text-wrap": "wrap", "text-max-width": "data(width)", - "text-halign": "data(textHalign)", - "text-valign": "data(textValign)" + "text-halign": function(element) { + const align = element?.data("textHalign") + if (align === null || align === undefined || align === "") { + return "center" + } + + return align + }, + "text-valign": function(element) { + const align = element?.data("textValign") + if (align === null || align === undefined || align === "") { + return "center" + } + + return align + } }, }, { From 80c568e5461845461ddd8c967cdb966e2e542291 Mon Sep 17 00:00:00 2001 From: Frikky Date: Wed, 2 Apr 2025 23:07:09 +0200 Subject: [PATCH 25/33] Added multiple updates to Orborus to ensure image downloads happen properly, even with bad defaults --- backend/go-app/go.mod | 4 +- backend/go-app/go.sum | 4 +- backend/go-app/main.go | 11 +- backend/go-app/walkoff.go | 13 --- frontend/src/views/AngularWorkflow.jsx | 30 +++--- functions/onprem/orborus/go.mod | 4 +- functions/onprem/orborus/go.sum | 4 +- functions/onprem/orborus/orborus.go | 138 +++++++++++++++---------- functions/onprem/worker/go.mod | 2 +- functions/onprem/worker/go.sum | 4 +- functions/onprem/worker/worker.go | 21 ++-- 11 files changed, 122 insertions(+), 113 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 6822373e..6aa6d2e0 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -4,7 +4,7 @@ go 1.23.0 toolchain go1.23.7 -replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared +//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared require ( cloud.google.com/go/datastore v1.15.0 @@ -20,7 +20,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.32 + github.com/shuffle/shuffle-shared v0.8.33 golang.org/x/crypto v0.36.0 google.golang.org/api v0.176.1 google.golang.org/grpc v1.68.1 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 01efec32..a011691e 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -333,8 +333,8 @@ github.com/sendgrid/sendgrid-go v3.14.0+incompatible h1:KDSasSTktAqMJCYClHVE94Fc github.com/sendgrid/sendgrid-go v3.14.0+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.32 h1:RizHZ4lZg9wQj3RQvJMgh+1SAMszwhYfcco3aT716RQ= -github.com/shuffle/shuffle-shared v0.8.32/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= +github.com/shuffle/shuffle-shared v0.8.33 h1:XEiaUf2ykAPku9lX7JUXFDRBSropqiT0L2p4RdqVawE= +github.com/shuffle/shuffle-shared v0.8.33/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 6df286b9..dda3cfd2 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3543,7 +3543,8 @@ func handleCloudJob(job shuffle.CloudSyncJob) error { return err } - redirectDomain := "localhost:5001" + backendPort := os.Getenv("BACKEND_PORT") + redirectDomain := fmt.Sprintf("localhost:%s", backendPort) redirectUrl := fmt.Sprintf("http://%s/api/v1/triggers/outlook/register", redirectDomain) outlookClient, _, err := shuffle.GetOutlookClient(ctx, "", hook.OauthToken, redirectUrl) if err != nil { @@ -4199,12 +4200,13 @@ func runInitEs(ctx context.Context) { return func() { log.Printf("[INFO] Running schedule for cleaning up or re-running unfinished workflows in %d environments.", len(environments)) + backendPort := os.Getenv("BACKEND_PORT") for _, environment := range environments { // Allowed without PROXY management as it's localhost // client := shuffle.GetExternalClient(syncUrl) httpClient := &http.Client{} - url := fmt.Sprintf("http://localhost:5001/api/v1/environments/%s/stop", environment) + url := fmt.Sprintf("http://localhost:%s/api/v1/environments/%s/stop", backendPort, environment) req, err := http.NewRequest( "GET", url, @@ -4231,7 +4233,7 @@ func runInitEs(ctx context.Context) { } log.Printf("[DEBUG] Successfully ran workflow cleanup request for %s. Body: %s", environment, string(respBody)) - url = fmt.Sprintf("http://localhost:5001/api/v1/environments/%s/rerun", environment) + url = fmt.Sprintf("http://localhost:%s/api/v1/environments/%s/rerun", backendPort, environment) req, err = http.NewRequest( "GET", url, @@ -4256,7 +4258,7 @@ func runInitEs(ctx context.Context) { log.Printf("[ERROR] Failed setting respbody %s", err) continue } - log.Printf("[DEBUG] Successfully ran workflow RERUN request for %s. Body: %s", environment, string(respBody)) + log.Printf("[DEBUG] Ran workflow RERUN request for %s with the response. Body: %s", environment, string(respBody)) } } } @@ -5373,6 +5375,7 @@ func main() { if innerPort == "" { log.Printf("[DEBUG] Running on %s:5001", hostname) log.Fatal(http.ListenAndServe(":5001", nil)) + os.Setenv("BACKEND_PORT", "5001") } else { log.Printf("[DEBUG] Running on %s:%s", hostname, innerPort) log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", innerPort), nil)) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 317595c3..b3aa3537 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -22,9 +22,6 @@ import ( dockerclient "github.com/docker/docker/client" "github.com/docker/docker/api/types/image" - - //gyaml "github.com/ghodss/yaml" - "github.com/h2non/filetype" uuid "github.com/satori/go.uuid" @@ -36,14 +33,6 @@ import ( "github.com/go-git/go-git/v5/storage/memory" "github.com/go-git/go-git/v5/plumbing" http2 "github.com/go-git/go-git/v5/plumbing/transport/http" - //http2 "gopkg.in/src-d/go-git.v5/plumbing/transport/http" - //http2 "github.com/go-git/go-git/plumbing/transport/http" - - //"github.com/gorilla/websocket" - //"google.golang.org/appengine" - //"google.golang.org/appengine/memcache" - //"cloud.google.com/go/firestore" - // "google.golang.org/api/option" gyaml "github.com/ghodss/yaml" ) @@ -207,8 +196,6 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque return } - log.Printf("Queue confirm: %#v. Queue: %#v", removeExecutionRequests.Data, id) - // remove items from DB parsedId := strings.ReplaceAll(fmt.Sprintf("workflowqueue-%s", id), " ", "-") ids := []string{} diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index db55ee92..0ff57718 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -21288,20 +21288,18 @@ const AngularWorkflow = (defaultprops) => { - - - + @@ -21641,7 +21639,6 @@ const AngularWorkflow = (defaultprops) => { > -

{ @@ -21651,9 +21648,8 @@ const AngularWorkflow = (defaultprops) => { stop() }} > - See more runs + Back to all runs

- ../../../../shuffle-shared +//replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared require ( github.com/docker/docker v27.5.0+incompatible github.com/docker/go-connections v0.5.0 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.19 + github.com/shuffle/shuffle-shared v0.8.33 k8s.io/api v0.30.2 k8s.io/apimachinery v0.30.2 ) diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index 39b13f42..6b5c6bd0 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -301,8 +301,8 @@ github.com/sendgrid/sendgrid-go v3.14.0+incompatible h1:KDSasSTktAqMJCYClHVE94Fc github.com/sendgrid/sendgrid-go v3.14.0+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.19 h1:HXqU62sPhVzv9MeJnA5ZpPYwwbitVz1EtMMABbY3t74= -github.com/shuffle/shuffle-shared v0.8.19/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= +github.com/shuffle/shuffle-shared v0.8.33 h1:XEiaUf2ykAPku9lX7JUXFDRBSropqiT0L2p4RdqVawE= +github.com/shuffle/shuffle-shared v0.8.33/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index b5a3272b..b990c032 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -56,7 +56,7 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" ) -// Starts jobs in bulk, so this could be increased +// Starts jobs in bulk, so this could be increased or decreased based on who the user is var sleepTime = 2 // Making it work on low-end machines even during busy times :) @@ -84,6 +84,7 @@ var appServiceAccountName = os.Getenv("SHUFFLE_APP_SERVICE_ACCOUNT_NAME") // var baseimagename = "shuffle/shuffle" var baseimagename = os.Getenv("SHUFFLE_BASE_IMAGE_NAME") var baseimageregistry = os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY") + //var baseimagetagsuffix = os.Getenv("SHUFFLE_BASE_IMAGE_TAG_SUFFIX") // Used for cloud with auth @@ -431,19 +432,19 @@ func deployServiceWorkers(image string) { } /* - isMemcachedRunning, err := checkMemcached(ctx, dockercli) - if err != nil { - log.Printf("[ERROR] Failed checking memcached: %s", err) - } - if isMemcachedRunning == false { - log.Printf("[ERROR] Memcached is not running. Will try to deploy it.") - deployMemcached(dockercli) - } + isMemcachedRunning, err := checkMemcached(ctx, dockercli) + if err != nil { + log.Printf("[ERROR] Failed checking memcached: %s", err) + } + if isMemcachedRunning == false { + log.Printf("[ERROR] Memcached is not running. Will try to deploy it.") + deployMemcached(dockercli) + } - ip := "shuffle-cache" - if len(os.Getenv("SHUFFLE_MEMCACHED")) == 0 { - os.Setenv("SHUFFLE_MEMCACHED", fmt.Sprintf("%s:11211", ip)) - } + ip := "shuffle-cache" + if len(os.Getenv("SHUFFLE_MEMCACHED")) == 0 { + os.Setenv("SHUFFLE_MEMCACHED", fmt.Sprintf("%s:11211", ip)) + } */ defaultNetworkAttach := false @@ -699,7 +700,6 @@ func deployServiceWorkers(image string) { if err == nil { log.Printf("[DEBUG] Successfully deployed workers with %d replica(s) on %d node(s)", replicas, cnt) - //time.Sleep(time.Duration(10) * time.Second) //log.Printf("[DEBUG] Servicecreate request: %#v %#v", service, err) } else { if !strings.Contains(fmt.Sprintf("%s", err), "Already Exists") && !strings.Contains(fmt.Sprintf("%s", err), "is already in use by service") { @@ -743,9 +743,7 @@ func handleBackendImageDownload(ctx context.Context, images string) error { // Remove the image handled := []string{} - log.Printf("[DEBUG] Removing existing image (s): %s. Waiting 30 seconds before starting to ensure backend has the latest images built and ready to distribute.", images) - //time.Sleep(time.Duration(30) * time.Second) - + log.Printf("[DEBUG] Removing existing image (s): %s. Waiting 30 seconds before starting to ensure backend has the latest images built and ready to distribute.", images) newImages := []string{} for _, curimage := range strings.Split(images, ",") { curimage = strings.TrimSpace(curimage) @@ -763,7 +761,7 @@ func handleBackendImageDownload(ctx context.Context, images string) error { // Force remove the current image to avoid cached layers if swarmConfig == "run" || swarmConfig == "swarm" { _, err := dockercli.ImageRemove(ctx, curimage, image.RemoveOptions{ - Force: true, + Force: true, PruneChildren: true, }) @@ -831,7 +829,7 @@ func handleBackendImageDownload(ctx context.Context, images string) error { if !strings.Contains(fmt.Sprintf("%s", resp), "error") { break } else { - found = true + found = true log.Printf("[ERROR] Failed updating service %s with the new image %s: %s. Resp: %#v", service.Spec.Annotations.Name, image, err, resp) } } @@ -1234,7 +1232,6 @@ func deployK8sWorker(image string, identifier string, env []string) error { func deployWorker(image string, identifier string, env []string, executionRequest shuffle.ExecutionRequest) error { - if len(os.Getenv("REGISTRY_URL")) > 0 && os.Getenv("REGISTRY_URL") != "" { env = append(env, fmt.Sprintf("REGISTRY_URL=%s", os.Getenv("REGISTRY_URL"))) } @@ -1294,7 +1291,6 @@ func deployWorker(image string, identifier string, env []string, executionReques } } - //var swarmConfig = os.Getenv("SHUFFLE_SWARM_CONFIG") parsedUuid := uuid.NewV4() @@ -1306,7 +1302,7 @@ func deployWorker(image string, identifier string, env []string, executionReques if isKubernetes != "true" { hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) - if strings.ToLower(cleanupEnv) != "false" { + if strings.ToLower(cleanupEnv) == "true" { hostConfig.AutoRemove = true } } @@ -1379,32 +1375,32 @@ func deployWorker(image string, identifier string, env []string, executionReques log.Printf("[ERROR][%s] Failed to start worker container in environment '%s': %s", executionRequest.ExecutionId, environment, err) return err } else { - log.Printf("[INFO][%s] Worker Container created (2). Environment %s: docker logs %s", executionRequest.ExecutionId, environment, cont.ID) + log.Printf("[INFO][%s] Worker Container created (2). Runtime Location '%s': docker logs -f %s", executionRequest.ExecutionId, environment, cont.ID) } stats, err := dockercli.ContainerInspect(ctx, cont.ID) if err != nil { log.Printf("[WARNING][%s] Failed checking worker '%s': %s", executionRequest.ExecutionId, cont.ID, err) - return nil + return nil } containerStatus := stats.ContainerJSONBase.State.Status if containerStatus != "running" { log.Printf("[ERROR][%s] Status of %s is %s. Should be running. Contact support@shuffler.io if this persists.", executionRequest.ExecutionId, cont.ID, containerStatus) } - /* - err = stopWorker(containerName) - if err != nil { - log.Printf("Failed stopping worker %s", execution.ExecutionId) - return nil - } + /* + err = stopWorker(containerName) + if err != nil { + log.Printf("Failed stopping worker %s", execution.ExecutionId) + return nil + } - err = deployWorker(dockercli, workerImage, containerName, env) - if err != nil { - log.Printf("Failed executing worker %s in state %s", execution.ExecutionId, containerStatus) - return nil + err = deployWorker(dockercli, workerImage, containerName, env) + if err != nil { + log.Printf("Failed executing worker %s in state %s", execution.ExecutionId, containerStatus) + return nil + } } - } */ } else { log.Printf("[INFO][%s] New Worker created. Environment %s: docker logs %s", executionRequest.ExecutionId, environment, cont.ID) @@ -1452,8 +1448,8 @@ func initializeImages() { } if baseimageregistry == "" { + //baseimageregistry = "ghcr.io" // Github baseimageregistry = "docker.io" // Dockerhub - baseimageregistry = "ghcr.io" // Github if len(os.Getenv("REGISTRY_URL")) > 0 { baseimageregistry = os.Getenv("REGISTRY_URL") @@ -1462,14 +1458,14 @@ func initializeImages() { } os.Setenv("SHUFFLE_BASE_IMAGE_REGISTRY", baseimageregistry) - + log.Printf("[WARNING] Setting baseimageregistry to %#v", baseimageregistry) } if baseimagename == "" { // FIXME: This is probably the problem for image names tbh + //baseimagename = "shuffle" // Github (ghcr.io) baseimagename = "frikky/shuffle" // Dockerhub - baseimagename = "shuffle" // Github (ghcr.io) os.Setenv("SHUFFLE_BASE_IMAGE_NAME", baseimagename) log.Printf("[WARNING] Setting baseimagename to %#v", baseimagename) @@ -1477,19 +1473,18 @@ func initializeImages() { log.Printf("[DEBUG] Setting swarm config to %#v. Default is empty.", swarmConfig) - // FIXME: Shuffle Worker vs Apps != same - newWorker := fmt.Sprintf("%s/%s/shuffle-worker:%s", baseimageregistry, baseimagename, workerVersion) + // This is now always static + newWorker := fmt.Sprintf("ghcr.io/shuffle/shuffle-worker:%s", workerVersion) if len(newWorkerImage) > 0 { newWorker = newWorkerImage } - // check whether they are the same first - - if os.Getenv("SHUFFLE_AUTO_IMAGE_DOWNLOAD") != "true" { + // Check whether they are the same first + if os.Getenv("SHUFFLE_AUTO_IMAGE_DOWNLOAD") == "false" { + log.Printf("[DEBUG] Skipping image download as SHUFFLE_AUTO_IMAGE_DOWNLOAD is set to false") + } else { images := []string{ fmt.Sprintf("frikky/shuffle:app_sdk"), - fmt.Sprintf("shuffle/shuffle:app_sdk"), - fmt.Sprintf("%s/%s/shuffle-app_sdk:%s", baseimageregistry, baseimagename, appSdkVersion), newWorker, } @@ -1510,8 +1505,6 @@ func initializeImages() { log.Printf("[DEBUG] Successfully downloaded and built %s", image) } } - } else { - log.Printf("[DEBUG] Skipping image download as SHUFFLE_AUTO_IMAGE_DOWNLOAD is set to true") } } @@ -2005,7 +1998,7 @@ func main() { log.Printf("[INFO] Setting up Docker environment. Downloading worker and App SDK!") initializeImages() - workerImage := fmt.Sprintf("%s/%s/shuffle-worker:%s", baseimageregistry, baseimagename, workerVersion) + workerImage := fmt.Sprintf("ghcr.io/shuffle/shuffle-worker:%s", workerVersion) if len(newWorkerImage) > 0 { workerImage = newWorkerImage } @@ -2116,6 +2109,7 @@ func main() { // Marshal and set body orborusStats := getOrborusStats(ctx) + pipelinePayload, pipelineerr := sendPipelineHealthStatus() if pipelineerr != nil { @@ -2515,9 +2509,23 @@ func main() { executionIds = append(executionIds, execution.ExecutionId) } else { log.Printf("[WARNING][%s] Failed to deploy: %s", execution.ExecutionId, err) + if strings.Contains(err.Error(), "already exists") { toBeRemoved.Data = append(toBeRemoved.Data, execution) executionIds = append(executionIds, execution.ExecutionId) + } else if strings.Contains(err.Error(), "No such image") { + // Download the image + + if isKubernetes == "true" { + log.Printf("[DEBUG] Skipping image pull of '%s' because Kubernetes does it in realtime instead", workerImage) + } else { + log.Printf("[DEBUG] Re-pulling image %s as it doesn't exist, and is necessary for worker to run (autofix)", workerImage) + pullOptions := image.PullOptions{} + _, err = dockercli.ImagePull(ctx, workerImage, pullOptions) + if err != nil { + log.Printf("[ERROR] Failed to pull image %s: %s", workerImage, err) + } + } } } } @@ -3208,10 +3216,28 @@ func deletePipeline(pipelineId string) error { func listPipelines() ([]shuffle.PipelineInfo, error) { responseData := shuffle.PipelineInfoWrapper{} + if tenzirDisabled { + return responseData.Pipelines, errors.New("Tenzir is disabled") + } + var reqBody []byte url := fmt.Sprintf("%s/api/v0/pipeline/list", pipelineUrl) - resp, err := http.Post(url, "application/json", bytes.NewBuffer(reqBody)) + client := http.Client{ + Timeout: 2 * time.Second, + } + req, err := http.NewRequest( + "POST", + url, + bytes.NewBuffer(reqBody), + ) + + if err != nil { + return responseData.Pipelines, err + } + + req.Header.Set("Content-Type", "application/json") + resp, err := client.Do(req) if err != nil { return responseData.Pipelines, err } @@ -3438,6 +3464,10 @@ func sendPipelineHealthStatus() (shuffle.LakeConfig, error) { Pipelines: []shuffle.PipelineInfoMini{}, } + if tenzirDisabled { + return pipelinePayload, nil + } + // To not spam down the list API too much randint := rand.Intn(5) if len(pipelines) == 0 || randint == 0 { @@ -3460,10 +3490,6 @@ func sendPipelineHealthStatus() (shuffle.LakeConfig, error) { pipelinePayload.Pipelines = pipelines } - if tenzirDisabled { - return pipelinePayload, nil - } - err := deployTenzirNode() if err != nil { if (!strings.Contains(err.Error(), "SHUFFLE_SKIP_PIPELINES") && !strings.Contains(err.Error(), "Kubernetes not implemented for Tenzir node")) && !strings.Contains(err.Error(), "Tenzir Node is already running") && !strings.Contains(err.Error(), "docker daemon") { @@ -3800,8 +3826,7 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest, image string, if err != nil { log.Printf("[ERROR] Failed creating worker request: %s", err) if strings.Contains(fmt.Sprintf("%s", err), "connection refused") || strings.Contains(fmt.Sprintf("%s", err), "EOF") { - workerImage := fmt.Sprintf("%s/%s/shuffle-worker:%s", baseimageregistry, baseimagename, workerVersion) - + workerImage := fmt.Sprintf("ghcr.io/shuffle/shuffle-worker:%s", workerVersion) if len(newWorkerImage) > 0 { workerImage = newWorkerImage } @@ -3825,8 +3850,7 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest, image string, log.Printf("[ERROR] Error running worker request to %s (1): %s", streamUrl, err) if strings.Contains(fmt.Sprintf("%s", err), "connection refused") || strings.Contains(fmt.Sprintf("%s", err), "EOF") { - workerImage := fmt.Sprintf("%s/%s/shuffle-worker:%s", baseimageregistry, baseimagename, workerVersion) - + workerImage := fmt.Sprintf("ghcr.io/shuffle/shuffle-worker:%s", workerVersion) if len(newWorkerImage) > 0 { workerImage = newWorkerImage } diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 23f27d64..eacebda6 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -8,7 +8,7 @@ require ( github.com/docker/docker v27.5.0+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.31 + github.com/shuffle/shuffle-shared v0.8.33 k8s.io/api v0.30.2 k8s.io/apimachinery v0.30.2 k8s.io/client-go v0.30.2 diff --git a/functions/onprem/worker/go.sum b/functions/onprem/worker/go.sum index 88c9ba70..360e7230 100644 --- a/functions/onprem/worker/go.sum +++ b/functions/onprem/worker/go.sum @@ -294,8 +294,8 @@ github.com/sendgrid/sendgrid-go v3.14.0+incompatible h1:KDSasSTktAqMJCYClHVE94Fc github.com/sendgrid/sendgrid-go v3.14.0+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.31 h1:APO/BkBxiP9Hn/Fa7SsESRD6ws9E5dBfIzS57pd3MvA= -github.com/shuffle/shuffle-shared v0.8.31/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= +github.com/shuffle/shuffle-shared v0.8.33 h1:XEiaUf2ykAPku9lX7JUXFDRBSropqiT0L2p4RdqVawE= +github.com/shuffle/shuffle-shared v0.8.33/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 5dc288f8..2a3a4301 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -105,9 +105,9 @@ var window = shuffle.NewTimeWindow(10 * time.Second) // Images to be autodeployed in the latest version of Shuffle. var autoDeploy = map[string]string{ - "http:1.4.0": "frikky/shuffle:http_1.4.0", - "shuffle-tools:1.2.0": "frikky/shuffle:shuffle-tools_1.2.0", - "shuffle-subflow:1.1.0": "frikky/shuffle:shuffle-subflow_1.1.0", + "http:1.4.0": "frikky/shuffle:http_1.4.0", + "shuffle-tools:1.2.0": "frikky/shuffle:shuffle-tools_1.2.0", + "shuffle-subflow:1.1.0": "frikky/shuffle:shuffle-subflow_1.1.0", // "shuffle-tools-fork:1.0.0": "frikky/shuffle:shuffle-tools-fork_1.0.0", } @@ -299,7 +299,7 @@ func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason } // Might not be necessary because of cleanupEnv hostconfig autoremoval - if cleanupEnv == "true" && (os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "swarm") { + if strings.ToLower(cleanupEnv) == "true" && (os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "swarm") { /* ctx := context.Background() dockercli, err := dockerclient.NewEnvClient() @@ -884,7 +884,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] } // Removing because log extraction should happen first - if cleanupEnv == "true" { + if strings.ToLower(cleanupEnv) == "true" { hostConfig.AutoRemove = true } @@ -1247,6 +1247,7 @@ func getWorkerURLs() ([]string, error) { } func askOtherWorkersToDownloadImage(image string) { + // Why wouldn't it happen on swarm? Hmm if os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "swarm" { return } @@ -1489,10 +1490,6 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { env = append(env, fmt.Sprintf("SHUFFLE_APP_SDK_TIMEOUT=%s", os.Getenv("SHUFFLE_APP_SDK_TIMEOUT"))) } - // Fixes issue: - // standard_go init_linux.go:185: exec user process caused "argument list too long" - // https://devblogs.microsoft.com/oldnewthing/20100203-00/?p=15083 - // FIXME: Ensure to NEVER do this anymore // This potentially breaks too much stuff. Better to have the app poll the data. _ = executionData @@ -1516,9 +1513,11 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { fmt.Sprintf("%s:%s_%s", baseimagename, parsedAppname, action.AppVersion), } - // If cleanup is set, it should run for efficiency + // This is the weirdest shit ever looking back at + // Needs optimization lol + pullOptions := image.PullOptions{} - if cleanupEnv == "true" { + if strings.ToLower(cleanupEnv) == "true" { err = deployApp(dockercli, images[0], identifier, env, workflowExecution, action) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { if strings.Contains(err.Error(), "exited prematurely") { From d213cc3b9825ee8c4e00ad6f8dff99156a9e694f Mon Sep 17 00:00:00 2001 From: Frikky Date: Wed, 2 Apr 2025 23:07:51 +0200 Subject: [PATCH 26/33] Also made requests not stupid slow due to waiting for Tenzir for no reason --- functions/onprem/orborus/orborus.go | 1 - 1 file changed, 1 deletion(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index b990c032..a945dd80 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -3,7 +3,6 @@ package main /* Orborus exists to listen for new jobs from Shuffle. This is to run workflows, pipelines, and other tasks. */ - import ( "archive/zip" "bytes" From 7c87def694b5fe4142e0f918d2500462a4e5d9c1 Mon Sep 17 00:00:00 2001 From: Frikky Date: Wed, 2 Apr 2025 23:20:27 +0200 Subject: [PATCH 27/33] Added dockerbuild nightly file to separate from prod --- .github/workflows/dockerbuild-nightly.yaml | 82 ++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 .github/workflows/dockerbuild-nightly.yaml diff --git a/.github/workflows/dockerbuild-nightly.yaml b/.github/workflows/dockerbuild-nightly.yaml new file mode 100644 index 00000000..a04e87bd --- /dev/null +++ b/.github/workflows/dockerbuild-nightly.yaml @@ -0,0 +1,82 @@ +name: dockerbuild + +on: + workflow_dispatch: + push: + branches: + - nightly + paths: + - "**" + - "!.github/**" + - "!**.md" + - "!docker-compose.yml" +jobs: + main: + runs-on: ubuntu-latest + continue-on-error: ${{ matrix.experimental }} + strategy: + fail-fast: false + matrix: + include: + - app: frontend + path: frontend + version: nightly + experimental: true + - app: backend + path: backend + version: nightly + experimental: true + - app: orborus + path: functions/onprem/orborus + version: nightly + experimental: true + - app: worker + path: functions/onprem/worker + version: nightly + experimental: true + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: "amd64,arm64,arm" + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to Ghcr + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Ghcr Build and push + id: docker_build + uses: docker/build-push-action@v4 + env: + BUILDX_NO_DEFAULT_LOAD: true + with: + logout: false + context: ${{ matrix.path }}/ + file: ${{ matrix.path }}/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + tags: | + ghcr.io/shuffle/shuffle-${{ matrix.app }}:${{ matrix.version }} + ${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:${{ matrix.version }} + frikky/shuffle-${{ matrix.app }}:${{ matrix.version }} + frikky/shuffle:${{ matrix.app }} + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} From bc8b1c2c6ec7a646442b5a0fe47c9b7981de215a Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 3 Apr 2025 01:53:55 +0200 Subject: [PATCH 28/33] Improved the image download mechanism for self-built apps onprem --- backend/go-app/docker.go | 64 +++++++++++++++++++++-------- backend/go-app/go.mod | 2 +- backend/go-app/walkoff.go | 2 +- functions/onprem/orborus/go.mod | 2 +- functions/onprem/orborus/go.sum | 4 +- functions/onprem/orborus/orborus.go | 4 +- functions/onprem/worker/go.mod | 2 +- functions/onprem/worker/go.sum | 4 +- 8 files changed, 56 insertions(+), 28 deletions(-) diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index e56455eb..9f0c72e2 100755 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -604,11 +604,27 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { // return //} - body, err := ioutil.ReadAll(request.Body) - if err != nil { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`)) - return + var err error + body := []byte{} + //log.Printf("IMAGE REQUEST BODY: %#v", request.Body) + if request.Body == nil || request.Body == http.NoBody { + // Check for the image query, otherwise we skip everything + imageQuery := request.URL.Query().Get("image") + if len(imageQuery) == 0 { + resp.WriteHeader(400) + resp.Write([]byte(`{"success": false, "reason": "No image query found"}`)) + return + } + + body = []byte(fmt.Sprintf(`{"name": "%s"}`, imageQuery)) + + } else { + body, err = ioutil.ReadAll(request.Body) + if err != nil { + resp.WriteHeader(400) + resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`)) + return + } } // This has to be done in a weird way because Datastore doesn't @@ -630,21 +646,24 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { return } - ctx := context.Background() - images, err := dockercli.ImageList(ctx, image.ListOptions{ - All: true, - }) - img := image.Summary{} - tagFound := "" - img2 := image.Summary{} + tagFound := "" tagFound2 := "" - alternativeNameSplit := strings.Split(version.Name, "/") - alternativeName := version.Name - if len(alternativeNameSplit) == 3 { - alternativeName = strings.Join(alternativeNameSplit[1:3], "/") + // Old way of doing it + //alternativeNameSplit := strings.Split(version.Name, "/") + //alternativeName := version.Name + //if len(alternativeNameSplit) == 3 { + // alternativeName = strings.Join(alternativeNameSplit[1:3], "/") + //} + + appname, baseAppname, appnameSplit2, err := shuffle.GetAppNameSplit(version) + if err != nil { + log.Printf("[ERROR] Failed getting appname split: %s", err) + resp.WriteHeader(500) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "Couldn't get the right docker image name"}`))) + return } if len(version.Name) == 0 { @@ -655,11 +674,20 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { } - log.Printf("[INFO] Trying to download image: '%s'. Alt name: %#v", version.Name, alternativeName) + log.Printf("[INFO] Trying to download image: '%s'. Appname: '%s'. BaseAppname: '%s', Split2: %s", version.Name, appname, baseAppname, appnameSplit2) + + alternativeName := appname + ctx := context.Background() + images, err := dockercli.ImageList(ctx, image.ListOptions{ + All: true, + }) for _, image := range images { for _, tag := range image.RepoTags { - //log.Printf("[DEBUG] Tag: %s", tag) + if strings.Contains(tag, "") { + continue + } + if strings.ToLower(tag) == strings.ToLower(version.Name) { img = image tagFound = tag diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 6aa6d2e0..e697498e 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -20,7 +20,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.33 + github.com/shuffle/shuffle-shared v0.8.34 golang.org/x/crypto v0.36.0 google.golang.org/api v0.176.1 google.golang.org/grpc v1.68.1 diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index b3aa3537..8b8fb878 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -974,7 +974,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { if err != nil { log.Printf("[ERROR] Failed to list child workflows: %s", err) } else { - log.Printf("\n\n[DEBUG] Found %d child workflows for workflow %s\n\n", len(childWorkflows), workflow.ID) + //log.Printf("\n\n[DEBUG] Found %d child workflows for workflow %s\n\n", len(childWorkflows), workflow.ID) // Find cookies and append them to request.Header to replicate current request as closely as possible for _, childWorkflow := range childWorkflows { diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 0c3ada48..cd7ec3c5 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -10,7 +10,7 @@ require ( github.com/docker/docker v27.5.0+incompatible github.com/docker/go-connections v0.5.0 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.33 + github.com/shuffle/shuffle-shared v0.8.34 k8s.io/api v0.30.2 k8s.io/apimachinery v0.30.2 ) diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index 6b5c6bd0..7b2f951d 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -301,8 +301,8 @@ github.com/sendgrid/sendgrid-go v3.14.0+incompatible h1:KDSasSTktAqMJCYClHVE94Fc github.com/sendgrid/sendgrid-go v3.14.0+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.33 h1:XEiaUf2ykAPku9lX7JUXFDRBSropqiT0L2p4RdqVawE= -github.com/shuffle/shuffle-shared v0.8.33/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= +github.com/shuffle/shuffle-shared v0.8.34 h1:yITGbBEaORKNo6yybrLMoHdHnZsnvq2tAZiileaGa0U= +github.com/shuffle/shuffle-shared v0.8.34/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index a945dd80..7c976475 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -742,7 +742,7 @@ func handleBackendImageDownload(ctx context.Context, images string) error { // Remove the image handled := []string{} - log.Printf("[DEBUG] Removing existing image (s): %s. Waiting 30 seconds before starting to ensure backend has the latest images built and ready to distribute.", images) + //log.Printf("[DEBUG] Removing existing image (s): %s", images) newImages := []string{} for _, curimage := range strings.Split(images, ",") { curimage = strings.TrimSpace(curimage) @@ -770,7 +770,7 @@ func handleBackendImageDownload(ctx context.Context, images string) error { log.Printf("[DEBUG] Removed image: %s", curimage) } } else { - log.Printf("[DEBUG] Skipping image removal for %s as swarmConfig is not set to run or swarm. Value: %#v", curimage, swarmConfig) + //log.Printf("[DEBUG] Skipping image removal for %s as swarmConfig is not set to run or swarm. Value: %#v", curimage, swarmConfig) } err := shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, curimage) diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index eacebda6..7fda9ba2 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -8,7 +8,7 @@ require ( github.com/docker/docker v27.5.0+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.33 + github.com/shuffle/shuffle-shared v0.8.34 k8s.io/api v0.30.2 k8s.io/apimachinery v0.30.2 k8s.io/client-go v0.30.2 diff --git a/functions/onprem/worker/go.sum b/functions/onprem/worker/go.sum index 360e7230..6878ebfb 100644 --- a/functions/onprem/worker/go.sum +++ b/functions/onprem/worker/go.sum @@ -294,8 +294,8 @@ github.com/sendgrid/sendgrid-go v3.14.0+incompatible h1:KDSasSTktAqMJCYClHVE94Fc github.com/sendgrid/sendgrid-go v3.14.0+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.33 h1:XEiaUf2ykAPku9lX7JUXFDRBSropqiT0L2p4RdqVawE= -github.com/shuffle/shuffle-shared v0.8.33/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= +github.com/shuffle/shuffle-shared v0.8.34 h1:yITGbBEaORKNo6yybrLMoHdHnZsnvq2tAZiileaGa0U= +github.com/shuffle/shuffle-shared v0.8.34/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= From 3ce9a75ef6f0643d8b26ba1afd05d7077f934ec6 Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 3 Apr 2025 01:55:23 +0200 Subject: [PATCH 29/33] Minor workflow change --- .github/workflows/dockerbuild-nightly.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerbuild-nightly.yaml b/.github/workflows/dockerbuild-nightly.yaml index a04e87bd..cb7907c6 100644 --- a/.github/workflows/dockerbuild-nightly.yaml +++ b/.github/workflows/dockerbuild-nightly.yaml @@ -1,4 +1,4 @@ -name: dockerbuild +name: nightly-dockerbuild on: workflow_dispatch: From d1d06df2b086a84973147272362785340fd9dbed Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 3 Apr 2025 18:23:34 +0200 Subject: [PATCH 30/33] Removed a problem with duplicate binds in worker --- backend/go-app/go.mod | 2 +- frontend/src/views/LoginPage.jsx | 12 ++++++++++-- functions/onprem/orborus/orborus.go | 5 ++--- functions/onprem/worker/worker.go | 5 ++++- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index e697498e..3bccf326 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -4,7 +4,7 @@ go 1.23.0 toolchain go1.23.7 -//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared +replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared require ( cloud.google.com/go/datastore v1.15.0 diff --git a/frontend/src/views/LoginPage.jsx b/frontend/src/views/LoginPage.jsx index 9a04b959..6291e6a3 100755 --- a/frontend/src/views/LoginPage.jsx +++ b/frontend/src/views/LoginPage.jsx @@ -323,9 +323,17 @@ const LoginPage = props => { if (document !== undefined) { if (register) { - document.title = "Login to Shuffle SaaS" + if (isCloud) { + document.title = "Login to Shuffle SaaS" + } else { + document.title = "Login to Shuffle" + } } else { - document.title = "Register to Shuffle SaaS" + if (isCloud) { + document.title = "Register to Shuffle SaaS" + } else { + document.title = "Register to Shuffle" + } } } diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 7c976475..f1fc5f29 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -1260,18 +1260,17 @@ func deployWorker(image string, identifier string, env []string, executionReques Resources: container.Resources{}, } - certPath := "/certs" // This is just to test the mounting locally so // I can control from what source I'm mounting // the certs to. Default behaviour is: // /certs:/certs. + certPath := "/certs" if os.Getenv("SHUFFLE_CERT_PATH") != "" { certPath = os.Getenv("SHUFFLE_CERT_PATH") } _, err := os.ReadDir(certPath) - if certPath != "" && err == nil { certVol := mount.Mount{ Type: mount.TypeBind, @@ -2477,7 +2476,7 @@ func main() { // Look for volume binds if len(os.Getenv("SHUFFLE_VOLUME_BINDS")) > 0 { - log.Printf("[DEBUG] Added volume binds: %s", os.Getenv("SHUFFLE_VOLUME_BINDS")) + //log.Printf("[DEBUG] Added volume binds: %s", os.Getenv("SHUFFLE_VOLUME_BINDS")) env = append(env, fmt.Sprintf("SHUFFLE_VOLUME_BINDS=%s", os.Getenv("SHUFFLE_VOLUME_BINDS"))) } diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 2a3a4301..d20e3578 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -906,7 +906,10 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] if len(volumeBinds) > 0 { log.Printf("[DEBUG] Setting up binds for container. Got %d volume binds.", len(volumeBinds)) - hostConfig.Binds = volumeBinds + //hostConfig.Binds = volumeBinds + + // Only use mounts, not direct binds + hostConfig.Binds = []string{} hostConfig.Mounts = []mount.Mount{} for _, bind := range volumeBinds { if !strings.Contains(bind, ":") || strings.Contains(bind, "..") || strings.HasPrefix(bind, "~") { From b204296e719ff1ffdde79b2cde0f08ee88eae388 Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 3 Apr 2025 19:19:40 +0200 Subject: [PATCH 31/33] Fixed a problem with docker mounts always being read-write and not mounting properly into containers --- backend/go-app/main.go | 8 ++++ frontend/src/views/AppCreator.jsx | 4 +- functions/onprem/orborus/go.mod | 2 +- functions/onprem/orborus/orborus.go | 15 +++++-- functions/onprem/worker/worker.go | 65 ++++++++++++++++++++--------- 5 files changed, 69 insertions(+), 25 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index dda3cfd2..dbef20e4 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3544,6 +3544,10 @@ func handleCloudJob(job shuffle.CloudSyncJob) error { } backendPort := os.Getenv("BACKEND_PORT") + if backendPort == "" { + backendPort = "5001" + } + redirectDomain := fmt.Sprintf("localhost:%s", backendPort) redirectUrl := fmt.Sprintf("http://%s/api/v1/triggers/outlook/register", redirectDomain) outlookClient, _, err := shuffle.GetOutlookClient(ctx, "", hook.OauthToken, redirectUrl) @@ -4201,6 +4205,10 @@ func runInitEs(ctx context.Context) { log.Printf("[INFO] Running schedule for cleaning up or re-running unfinished workflows in %d environments.", len(environments)) backendPort := os.Getenv("BACKEND_PORT") + if backendPort == "" { + backendPort = "5001" + } + for _, environment := range environments { // Allowed without PROXY management as it's localhost // client := shuffle.GetExternalClient(syncUrl) diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 50106831..74fe9ae6 100755 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -2037,7 +2037,7 @@ const AppCreator = (defaultprops) => { for (let actionkey in actions) { var item = JSON.parse(JSON.stringify(actions[actionkey])) if (item.errors.length > 0) { - toast("Saving with error in action " + item.name); + //toast("Saving with error in action " + item.name); } if (item.name === undefined && item.description !== undefined) { @@ -3858,7 +3858,7 @@ const AppCreator = (defaultprops) => { if (currentAction.url === "" && actions !== undefined && actions !== null && actions.length > 0) { for (var i = 0; i < actions.length; i++) { if (actions[i].name.toLowerCase() === e.target.value.toLowerCase()) { - toast("Action with name " + e.target.value + " already exists. If you keep this, it will be overwritten.") + //toast("Action with name " + e.target.value + " already exists. If you keep this, it will be overwritten.") break } } diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index cd7ec3c5..0ffdc2a6 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -4,7 +4,7 @@ go 1.23.0 toolchain go1.23.6 -//replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared +replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared require ( github.com/docker/docker v27.5.0+incompatible diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index f1fc5f29..e773b1fd 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -744,6 +744,8 @@ func handleBackendImageDownload(ctx context.Context, images string) error { handled := []string{} //log.Printf("[DEBUG] Removing existing image (s): %s", images) newImages := []string{} + + successful := []string{} for _, curimage := range strings.Split(images, ",") { curimage = strings.TrimSpace(curimage) if shuffle.ArrayContains(handled, curimage) { @@ -775,12 +777,19 @@ func handleBackendImageDownload(ctx context.Context, images string) error { err := shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, curimage) if err != nil { - log.Printf("[ERROR] Failed downloading image: %s", err) + //log.Printf("[ERROR] Failed downloading image: %s", err) } else { - log.Printf("[DEBUG] Downloaded image: %s", curimage) + //log.Printf("[DEBUG] Downloaded image: %s", curimage) + successful = append(successful, curimage) } } + if len(successful) == 0 { + log.Printf("[ERROR] Failed downloading image copies: %s. This means the app may not have been updated.", strings.Join(handled, ", ")) + } else { + log.Printf("[DEBUG] Successfully downloaded image copies: %s", strings.Join(successful, ", ")) + } + if swarmConfig == "run" || swarmConfig == "swarm" { log.Printf("[DEBUG] Should update service with new image after updating(s): %s. \n\nBETA REPLACEMENT IMPLEMENTATION: Contact support@shuffler.io for support.", strings.Join(newImages, "\n")) @@ -1919,7 +1928,7 @@ func main() { } // Handle Cleanup - made it cleanup by default - if strings.ToLower(os.Getenv("SHUFFLE_CONTAINER_AUTO_CLEANUP")) != "false" { + if strings.ToLower(os.Getenv("SHUFFLE_CONTAINER_AUTO_CLEANUP")) != "false" && os.Getenv("CLEANUP") == "" { cleanupEnv = "true" } diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index d20e3578..d52275d1 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -894,19 +894,27 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] if len(volumeBindString) > 0 { volumeBindSplit := strings.Split(volumeBindString, ",") for _, volumeBind := range volumeBindSplit { - if strings.Contains(volumeBind, ":") { - volumeBinds = append(volumeBinds, volumeBind) - } else { - log.Printf("[ERROR] Volume bind '%s' is invalid.", volumeBind) + if volumeBind == "srcfolder=dstfolder" || volumeBind == "srcfolder:dstfolder" || volumeBind == "/srcfolder:/dstfolder" { + log.Printf("[DEBUG] Volume bind '%s' is invalid and is used for visualization.", volumeBind) + continue } + + if !strings.HasPrefix(volumeBind, "/") { + log.Printf("[ERROR] Volume bind '%s' is invalid. Use absolute paths.", volumeBind) + continue + } + + if !strings.Contains(volumeBind, ":") { + log.Printf("[ERROR] Volume bind '%s' is invalid. Use absolute paths with colon inbetween them (/srcpath:dstpath/", volumeBind) + continue + } + + volumeBinds = append(volumeBinds, volumeBind) } } // Add more volume binds if possible if len(volumeBinds) > 0 { - log.Printf("[DEBUG] Setting up binds for container. Got %d volume binds.", len(volumeBinds)) - - //hostConfig.Binds = volumeBinds // Only use mounts, not direct binds hostConfig.Binds = []string{} @@ -917,15 +925,27 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] continue } - log.Printf("[DEBUG] Appending bind %s to app container", bind) + log.Printf("[DEBUG] Appending bind %s to App container", bind) bindSplit := strings.Split(bind, ":") sourceFolder := bindSplit[0] destinationFolder := bindSplit[1] - hostConfig.Mounts = append(hostConfig.Mounts, mount.Mount{ + + readOnly := false + if len(bindSplit) > 2 { + mode := bindSplit[2] + if mode == "ro" { + readOnly = true + } + } + + builtMount := mount.Mount{ Type: mount.TypeBind, Source: sourceFolder, Target: destinationFolder, - }) + ReadOnly: readOnly, + } + + hostConfig.Mounts = append(hostConfig.Mounts, builtMount) } } @@ -1053,17 +1073,24 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont err = cli.ContainerStart(ctx, cont.ID, container.StartOptions{}) if err != nil { if strings.Contains(fmt.Sprintf("%s", err), "cannot join network") || strings.Contains(fmt.Sprintf("%s", err), "No such container") { + // Remove the "CREATED" one from the previous: + removeErr := cli.ContainerRemove(ctx, cont.ID, container.RemoveOptions{}) + if removeErr != nil { + log.Printf("[ERROR] Failed to remove container %s: %s", cont.ID, removeErr) + } + + log.Printf("[WARNING] Failed deploying App on first attempt: %s. Removing some HostConfig configs.", err) parsedUuid := uuid.NewV4() identifier = fmt.Sprintf("%s-%s-nonetwork", identifier, parsedUuid) - hostConfig = &container.HostConfig{ - LogConfig: container.LogConfig{ - Type: "json-file", - Config: map[string]string{ - "max-size": "10m", - }, + + hostConfig.NetworkMode = container.NetworkMode("") + hostConfig.LogConfig = container.LogConfig{ + Type: "json-file", + Config: map[string]string{ + "max-size": "10m", }, - Resources: container.Resources{}, } + hostConfig.Resources = container.Resources{} cont, err = cli.ContainerCreate( context.Background(), @@ -1085,12 +1112,12 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont return err } - log.Printf("[DEBUG] Running secondary check without network with worker") + //log.Printf("[DEBUG] Running secondary check without network with worker") err = cli.ContainerStart(ctx, cont.ID, container.StartOptions{}) } if err != nil { - log.Printf("[ERROR] Failed to start container in environment %s: %s", environment, err) + log.Printf("[ERROR] Failed to start container (2) in runtime location %s: %s", environment, err) cacheErr := shuffle.DeleteCache(ctx, actionExecId) if cacheErr != nil { From 9dbe9b807744665c627ac173f76ee517b3525e90 Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 3 Apr 2025 22:48:51 +0200 Subject: [PATCH 32/33] Fixed minor issues with app deployment etc --- backend/go-app/go.mod | 4 ++-- backend/go-app/go.sum | 4 ++-- frontend/src/components/Priorities.jsx | 3 ++- frontend/src/views/AngularWorkflow.jsx | 2 +- frontend/src/views/AppCreator.jsx | 5 ++--- functions/onprem/orborus/go.mod | 4 ++-- functions/onprem/orborus/go.sum | 4 ++-- functions/onprem/orborus/orborus.go | 2 +- functions/onprem/worker/go.mod | 2 +- functions/onprem/worker/go.sum | 4 ++-- functions/onprem/worker/worker.go | 2 +- 11 files changed, 18 insertions(+), 18 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 3bccf326..3db9ac92 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -4,7 +4,7 @@ go 1.23.0 toolchain go1.23.7 -replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared +//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared require ( cloud.google.com/go/datastore v1.15.0 @@ -20,7 +20,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.34 + github.com/shuffle/shuffle-shared v0.8.35 golang.org/x/crypto v0.36.0 google.golang.org/api v0.176.1 google.golang.org/grpc v1.68.1 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index a011691e..5d37f78b 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -333,8 +333,8 @@ github.com/sendgrid/sendgrid-go v3.14.0+incompatible h1:KDSasSTktAqMJCYClHVE94Fc github.com/sendgrid/sendgrid-go v3.14.0+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.33 h1:XEiaUf2ykAPku9lX7JUXFDRBSropqiT0L2p4RdqVawE= -github.com/shuffle/shuffle-shared v0.8.33/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= +github.com/shuffle/shuffle-shared v0.8.35 h1:3awc0TrsLLZiQeWD2XGIkTnFbczAG0cMfy1+cB/P7zg= +github.com/shuffle/shuffle-shared v0.8.35/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/frontend/src/components/Priorities.jsx b/frontend/src/components/Priorities.jsx index 5102b72b..0362a59f 100644 --- a/frontend/src/components/Priorities.jsx +++ b/frontend/src/components/Priorities.jsx @@ -837,12 +837,13 @@ const NotificationItem = memo((props) => { color="secondary" style={{ height: 50, + textTransform: "none", }} onClick={() => { dismissNotification(data.id); }} > - Mark Read + Mark as Read ) : null} diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 0ff57718..df92a1cd 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -21552,7 +21552,7 @@ const AngularWorkflow = (defaultprops) => { onClick={(e) => { e.preventDefault() e.stopPropagation() - window.open(`/admin?admin_tab=notifications&workflow=${data.workflow.id}&execution_id=${data.execution_id}`, "_blank") + window.open(`/admin?org_id=${workflow.org_id}&admin_tab=notifications&workflow=${data.workflow.id}&execution_id=${data.execution_id}`, "_blank") }} /> diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 74fe9ae6..ae234e9e 100755 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -943,8 +943,8 @@ const AppCreator = (defaultprops) => { if (newaction.url !== undefined && newaction.url !== null && newaction.url.includes("_shuffle_replace_")) { //const regex = /_shuffle_replace_\d/i; const regex = /_shuffle_replace_\d+/i - - newaction.url = newaction.url.replaceAll(new RegExp(regex, 'g'), "") + const newurl = newaction.url.replaceAll(new RegExp(regex, 'g'), "") + newaction.url = newurl } // Finding category @@ -956,7 +956,6 @@ const AppCreator = (defaultprops) => { if (pathsplit[splitkey].includes("_shuffle_replace_")) { //const regex = /_shuffle_replace_\d/i; const regex = /_shuffle_replace_\d+/i - //console.log("NEW: ", pathsplit[splitkey] = pathsplit[splitkey].replaceAll(new RegExp(regex, 'g'), "") } diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 0ffdc2a6..cd796147 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -4,13 +4,13 @@ go 1.23.0 toolchain go1.23.6 -replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared +//replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared require ( github.com/docker/docker v27.5.0+incompatible github.com/docker/go-connections v0.5.0 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.34 + github.com/shuffle/shuffle-shared v0.8.35 k8s.io/api v0.30.2 k8s.io/apimachinery v0.30.2 ) diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index 7b2f951d..cbbfad08 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -301,8 +301,8 @@ github.com/sendgrid/sendgrid-go v3.14.0+incompatible h1:KDSasSTktAqMJCYClHVE94Fc github.com/sendgrid/sendgrid-go v3.14.0+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.34 h1:yITGbBEaORKNo6yybrLMoHdHnZsnvq2tAZiileaGa0U= -github.com/shuffle/shuffle-shared v0.8.34/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= +github.com/shuffle/shuffle-shared v0.8.35 h1:3awc0TrsLLZiQeWD2XGIkTnFbczAG0cMfy1+cB/P7zg= +github.com/shuffle/shuffle-shared v0.8.35/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index e773b1fd..6e031d4a 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -2256,7 +2256,7 @@ func main() { toBeRemoved.Data = append(toBeRemoved.Data, incRequest) } else if incRequest.Type == "DOCKER_IMAGE_DOWNLOAD" { - log.Printf("[INFO] Re-downloading new image(s): %#v", incRequest.ExecutionArgument) + log.Printf("[INFO] Re-downloading new image(s) due to backend request: %#v", incRequest.ExecutionArgument) if len(incRequest.ExecutionArgument) > 0 { go handleBackendImageDownload(ctx, incRequest.ExecutionArgument) diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 7fda9ba2..83de43e0 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -8,7 +8,7 @@ require ( github.com/docker/docker v27.5.0+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.8.34 + github.com/shuffle/shuffle-shared v0.8.35 k8s.io/api v0.30.2 k8s.io/apimachinery v0.30.2 k8s.io/client-go v0.30.2 diff --git a/functions/onprem/worker/go.sum b/functions/onprem/worker/go.sum index 6878ebfb..49b14df1 100644 --- a/functions/onprem/worker/go.sum +++ b/functions/onprem/worker/go.sum @@ -294,8 +294,8 @@ github.com/sendgrid/sendgrid-go v3.14.0+incompatible h1:KDSasSTktAqMJCYClHVE94Fc github.com/sendgrid/sendgrid-go v3.14.0+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.8.34 h1:yITGbBEaORKNo6yybrLMoHdHnZsnvq2tAZiileaGa0U= -github.com/shuffle/shuffle-shared v0.8.34/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= +github.com/shuffle/shuffle-shared v0.8.35 h1:3awc0TrsLLZiQeWD2XGIkTnFbczAG0cMfy1+cB/P7zg= +github.com/shuffle/shuffle-shared v0.8.35/go.mod h1:NruHSAscDsW595wpK2r7MeHPGspUEKRNvBpcN1iGbHI= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index d52275d1..803aaac2 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -895,7 +895,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] volumeBindSplit := strings.Split(volumeBindString, ",") for _, volumeBind := range volumeBindSplit { if volumeBind == "srcfolder=dstfolder" || volumeBind == "srcfolder:dstfolder" || volumeBind == "/srcfolder:/dstfolder" { - log.Printf("[DEBUG] Volume bind '%s' is invalid and is used for visualization.", volumeBind) + log.Printf("[DEBUG] Volume bind '%s' is invalid.", volumeBind) continue } From 8d0939a94107a2288a6f39be6e14a98f3ee79778 Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 3 Apr 2025 22:51:36 +0200 Subject: [PATCH 33/33] Minor print fixes --- backend/go-app/main.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index dbef20e4..746904aa 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -4202,7 +4202,7 @@ func runInitEs(ctx context.Context) { cleanupJob := func() func() { return func() { - log.Printf("[INFO] Running schedule for cleaning up or re-running unfinished workflows in %d environments.", len(environments)) + //log.Printf("[INFO] Running schedule for cleaning up or re-running unfinished workflows in %d environments.", len(environments)) backendPort := os.Getenv("BACKEND_PORT") if backendPort == "" { @@ -4221,6 +4221,7 @@ func runInitEs(ctx context.Context) { nil, ) + // FIXME: This will stop working of the user rotates their key lol req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, parsedApikey)) if err != nil { log.Printf("[ERROR] Failed CREATING environment request for %s: %s", environment, err) @@ -4266,7 +4267,8 @@ func runInitEs(ctx context.Context) { log.Printf("[ERROR] Failed setting respbody %s", err) continue } - log.Printf("[DEBUG] Ran workflow RERUN request for %s with the response. Body: %s", environment, string(respBody)) + + //log.Printf("[DEBUG] Ran workflow RERUN request for %s with the response. Body: %s", environment, string(respBody)) } } }