From 923a7df0e1dab553a73c400bfda488bce49fbb55 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Thu, 16 Oct 2025 10:29:29 +0000 Subject: [PATCH 01/16] fix: backend/Dockerfile to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-ALPINE322-OPENSSL-13174131 - https://snyk.io/vuln/SNYK-ALPINE322-OPENSSL-13174131 - https://snyk.io/vuln/SNYK-ALPINE322-OPENSSL-13174132 - https://snyk.io/vuln/SNYK-ALPINE322-OPENSSL-13174132 - https://snyk.io/vuln/SNYK-ALPINE322-OPENSSL-13174133 --- backend/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100755 => 100644 backend/Dockerfile diff --git a/backend/Dockerfile b/backend/Dockerfile old mode 100755 new mode 100644 index 647f74cf..5ccc3de8 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -27,7 +27,7 @@ FROM alpine:latest as certs RUN apk add --update ca-certificates # Sets up the final image -FROM alpine:3.22.1 +FROM alpine:3.22.2 # FIXME: Install cgo because CGO_ENABLED=1 during build RUN apk add --no-cache libc6-compat From bc4b84908fe66432e4ec9705afb22a3950ac5e42 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Thu, 23 Oct 2025 01:22:47 +0530 Subject: [PATCH 02/16] defaulting app replicas to 3 --- .env | 2 ++ docker-compose.yml | 1 + 2 files changed, 3 insertions(+) diff --git a/.env b/.env index 16a4f419..5f507244 100755 --- a/.env +++ b/.env @@ -96,6 +96,8 @@ SHUFFLE_WORKER_SERVER_URL= SHUFFLE_ORBORUS_PULL_TIME= # Max recursion depth for subflows SHUFFLE_MAX_EXECUTION_DEPTH= +# Amount of app replicas +SHUFFLE_APP_REPLICAS=3 # DATABASE CONFIGURATIONS DATASTORE_EMULATOR_HOST=shuffle-database:8000 diff --git a/docker-compose.yml b/docker-compose.yml index 8ab7e3b9..75638774 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,6 +52,7 @@ services: - HTTPS_PROXY=${HTTPS_PROXY} - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY} - SHUFFLE_PASS_APP_PROXY=${SHUFFLE_PASS_APP_PROXY} + - SHUFFLE_APP_REPLICAS=${SHUFFLE_APP_REPLICAS} - SHUFFLE_STATS_DISABLED=true - SHUFFLE_LOGS_DISABLED=true - SHUFFLE_SWARM_CONFIG=run From 97e264904dcb7317a5ec63a4583a8551ceecdcf5 Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 24 Oct 2025 16:40:32 +0200 Subject: [PATCH 03/16] Set default app replicas to 3 --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 75638774..70b3bd5d 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -56,6 +56,7 @@ services: - SHUFFLE_STATS_DISABLED=true - SHUFFLE_LOGS_DISABLED=true - SHUFFLE_SWARM_CONFIG=run + - SHUFFLE_APP_REPLICAS=3 - CLEANUP=false - SHUFFLE_WORKER_IMAGE=ghcr.io/shuffle/shuffle-worker:latest env_file: .env From 073c51ca3877d47e8bb067e59bd6d496d33df43f Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 24 Oct 2025 16:42:57 +0200 Subject: [PATCH 04/16] Remove unused environment variables from docker-compose Removed SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY environment variable and commented out DOCKER_HOST. --- docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 70b3bd5d..8a8d2cd2 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -42,8 +42,6 @@ services: - /var/run/docker.sock:/var/run/docker.sock environment: - SHUFFLE_APP_SDK_TIMEOUT=300 - - SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY=7 # The amount of concurrent executions Orborus can handle. - #- DOCKER_HOST=tcp://docker-socket-proxy:2375 - ENVIRONMENT_NAME=Shuffle - ORG_ID=Shuffle - BASE_URL=http://${OUTER_HOSTNAME}:5001 From 392a05100ec7a2b2c2569d1f141a3cb07d0b5e9c Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Fri, 31 Oct 2025 19:58:38 +0530 Subject: [PATCH 05/16] log info over what we got from backend --- functions/onprem/worker/worker.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 5a858888..11c7b524 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -2555,6 +2555,11 @@ func getWorkerBackendExecution(auth string, executionId string) (*shuffle.Workfl return workflowExecution, err } + if debug { + log.Printf("[INFO] Here is the result we got back from backend: %s", workflowExecution.Results) + } + + setWorkflowExecution(context.Background(), *workflowExecution, false) return workflowExecution, nil } From 70937213a1c69e48c4963c5e7077653178f5fddc Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Fri, 31 Oct 2025 21:39:47 +0530 Subject: [PATCH 06/16] fix: memcache does not consider env --- docker-compose.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8ab7e3b9..461acf9a 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,6 +31,9 @@ services: #- DOCKER_HOST=tcp://docker-socket-proxy:2375 - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps - SHUFFLE_FILE_LOCATION=/shuffle-files + - HTTP_PROXY=http://192.168.29.215:5000 + - HTTPS_PROXY=http://192.168.29.215:5000 + - NO_PROXY=192.168.29.215:3001, shuffler.io, 192.168.29.215:9200 restart: unless-stopped orborus: image: ghcr.io/shuffle/shuffle-orborus:latest @@ -122,11 +125,10 @@ services: # - MEMCACHED_MAX_CONNECTIONS=2500 # ports: # - 11211:11211 - # deploy: - # resources: - # limits: - # memory: 2048M - + # networks: + # - shuffle + # - swarm_executions + #docker-socket-proxy: # image: tecnativa/docker-socket-proxy # container_name: docker-socket-proxy From 61f499807bba50b4adc53f5c19046d7bd3e713b0 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Fri, 31 Oct 2025 21:40:49 +0530 Subject: [PATCH 07/16] remove test data --- docker-compose.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 461acf9a..0d96bcdf 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,9 +31,6 @@ services: #- DOCKER_HOST=tcp://docker-socket-proxy:2375 - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps - SHUFFLE_FILE_LOCATION=/shuffle-files - - HTTP_PROXY=http://192.168.29.215:5000 - - HTTPS_PROXY=http://192.168.29.215:5000 - - NO_PROXY=192.168.29.215:3001, shuffler.io, 192.168.29.215:9200 restart: unless-stopped orborus: image: ghcr.io/shuffle/shuffle-orborus:latest From 5564814ada701fd4660d9e37de67cb677706f27f Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Sun, 2 Nov 2025 10:25:58 +0000 Subject: [PATCH 08/16] fix: frontend/Dockerfile to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-DEBIAN12-LIBXML2-10350658 - https://snyk.io/vuln/SNYK-DEBIAN12-LIBXML2-10350668 - https://snyk.io/vuln/SNYK-DEBIAN12-EXPAT-6227597 - https://snyk.io/vuln/SNYK-DEBIAN12-EXPAT-9407503 - https://snyk.io/vuln/SNYK-DEBIAN12-TIFF-13019601 --- frontend/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100755 => 100644 frontend/Dockerfile diff --git a/frontend/Dockerfile b/frontend/Dockerfile old mode 100755 new mode 100644 index 20051475..5df910ee --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -27,7 +27,7 @@ COPY ./*.json /usr/src/app/ RUN npm run build --loglevel verbose 2>&1 # Production environment -FROM nginx:1.29.0 +FROM nginx:1.29.3 RUN mkdir -p /usr/share/nginx/html/build RUN mkdir -p /usr/share/nginx/html/css From 2dc1966d08df1b4aa7c2469ec01df89c5bed9841 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Mon, 10 Nov 2025 13:54:14 +0530 Subject: [PATCH 09/16] change the order of stopSchedule --- backend/go-app/walkoff.go | 41 ++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 032af73a..0c015470 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -1505,30 +1505,39 @@ func stopScheduleGCP(resp http.ResponseWriter, request *http.Request) { return } -func deleteSchedule(ctx context.Context, id string) error { - log.Printf("[DEBUG] Should stop schedule %s!", id) +func deleteKeySchedule(ctx context.Context, id string) error { err := shuffle.DeleteKey(ctx, "schedules", id) if err != nil { - log.Printf("[ERROR] Failed to delete schedule: %s", err) return err + } + return nil +} + +func deleteSchedule(ctx context.Context, id string) error { + log.Printf("[DEBUG] Should stop schedule %s!", id) + if value, exists := scheduledJobs[id]; exists { + // Stops the schedule properly + value.Lock() } else { - if value, exists := scheduledJobs[id]; exists { - // Stops the schedule properly - value.Lock() - } else { - // FIXME - allow it to kind of stop anyway? - if j, ok := cronJobs[id]; ok { - err := CronScheduler.RemoveByID(j) - if err != nil { - log.Printf("[ERROR] Failed to remove the scheduler %s", err) - return err - } - } else { - return errors.New("Can't find the schedule.") + // FIXME - allow it to kind of stop anyway? + if j, ok := cronJobs[id]; ok { + err := CronScheduler.RemoveByID(j) + if err != nil { + log.Printf("[ERROR] Failed to remove the scheduler %s", err) + return err } + } else { + // Just stop and delete anyway if not in memory + deleteKeySchedule(ctx, id) + return errors.New("Can't find the schedule.") } } + err := deleteKeySchedule(ctx, id) + if err != nil { + log.Printf("[ERROR] Failed to stop schedule in db %s: %s", id, err) + return err + } return nil } From ff0f0149c24dd2aec082323d76960a1ff9529088 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Mon, 10 Nov 2025 13:58:59 +0530 Subject: [PATCH 10/16] change order of deleteSchedule --- backend/go-app/walkoff.go | 41 ++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 032af73a..0c015470 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -1505,30 +1505,39 @@ func stopScheduleGCP(resp http.ResponseWriter, request *http.Request) { return } -func deleteSchedule(ctx context.Context, id string) error { - log.Printf("[DEBUG] Should stop schedule %s!", id) +func deleteKeySchedule(ctx context.Context, id string) error { err := shuffle.DeleteKey(ctx, "schedules", id) if err != nil { - log.Printf("[ERROR] Failed to delete schedule: %s", err) return err + } + return nil +} + +func deleteSchedule(ctx context.Context, id string) error { + log.Printf("[DEBUG] Should stop schedule %s!", id) + if value, exists := scheduledJobs[id]; exists { + // Stops the schedule properly + value.Lock() } else { - if value, exists := scheduledJobs[id]; exists { - // Stops the schedule properly - value.Lock() - } else { - // FIXME - allow it to kind of stop anyway? - if j, ok := cronJobs[id]; ok { - err := CronScheduler.RemoveByID(j) - if err != nil { - log.Printf("[ERROR] Failed to remove the scheduler %s", err) - return err - } - } else { - return errors.New("Can't find the schedule.") + // FIXME - allow it to kind of stop anyway? + if j, ok := cronJobs[id]; ok { + err := CronScheduler.RemoveByID(j) + if err != nil { + log.Printf("[ERROR] Failed to remove the scheduler %s", err) + return err } + } else { + // Just stop and delete anyway if not in memory + deleteKeySchedule(ctx, id) + return errors.New("Can't find the schedule.") } } + err := deleteKeySchedule(ctx, id) + if err != nil { + log.Printf("[ERROR] Failed to stop schedule in db %s: %s", id, err) + return err + } return nil } From 52fed7105e42e1d6ae8f15b51210beaef574b668 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Tue, 11 Nov 2025 22:43:57 +0530 Subject: [PATCH 11/16] fix: tenzir disable + networkID focused --- functions/onprem/orborus/orborus.go | 57 +++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 1d9d8fed..a0f78845 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -441,12 +441,32 @@ func deployServiceWorkers(image string) { if err != nil { if strings.Contains(fmt.Sprintf("%s", err), "already exists") { // Try patching for attachable - + if debug { + log.Printf("[DEBUG] Network %s already exists", networkName) + } } else { log.Printf("[DEBUG] Failed to create network %s for workers: %s. This is not critical, and containers will still be added", networkName, err) } } + networkID := "" + + // find network ID + networks, err := dockercli.NetworkList(ctx, network.ListOptions{}) + if err == nil { + for _, net := range networks { + if net.Name == networkName { + if net.Scope == "swarm" { + log.Printf("[DEBUG] Found swarm-scoped network: %s (%s)", networkName, net.ID) + networkID = net.ID + } else { + log.Printf("[WARNING] Network %s exists but is not swarm scoped (scope=%s)", networkName, net.Scope) + } + break + } + } + } + /* isMemcachedRunning, err := checkMemcached(ctx, dockercli) if err != nil { @@ -463,12 +483,17 @@ func deployServiceWorkers(image string) { } */ + if networkID == "" { + log.Printf("[ERROR] Network %s does not exist", networkName) + networkID = networkName + } + defaultNetworkAttach := false if containerId != "" { log.Printf("[DEBUG] Should connect orborus container to worker network as it's running in Docker with name %#v!", containerId) // https://pkg.go.dev/github.com/docker/docker@v20.10.12+incompatible/api/types/network#EndpointSettings networkConfig := &network.EndpointSettings{} - err := dockercli.NetworkConnect(ctx, networkName, containerId, networkConfig) + err := dockercli.NetworkConnect(ctx, networkID, containerId, networkConfig) if err != nil { log.Printf("[ERROR] Failed connecting Orborus to docker network %s: %s", networkName, err) } @@ -491,7 +516,7 @@ func deployServiceWorkers(image string) { for _, container := range containers { if strings.Contains(strings.ToLower(container.Image), "docker-socket-proxy") { networkConfig := &network.EndpointSettings{} - err := dockercli.NetworkConnect(ctx, networkName, container.ID, networkConfig) + err := dockercli.NetworkConnect(ctx, networkID, container.ID, networkConfig) if err != nil { log.Printf("[ERROR] Failed connecting Docker socket proxy to docker network %s: %s", networkName, err) } else { @@ -571,7 +596,7 @@ func deployServiceWorkers(image string) { }, Networks: []swarm.NetworkAttachmentConfig{ swarm.NetworkAttachmentConfig{ - Target: networkName, + Target: networkID, }, swarm.NetworkAttachmentConfig{ Target: "ingress", @@ -2109,12 +2134,12 @@ func cleanup() { func StartAgent() { log.Printf("[INFO] Starting Orborus agent mode") - + auditLogEnabled := os.Getenv("SHUFFLE_AUDIT_LOG_ENABLED") == "true" - + if auditLogEnabled { log.Printf("[INFO] Audit log monitoring is enabled") - + // Initialize telemetry configuration telemetryConfig := shuffle.TelemetryConfig{ Enabled: true, @@ -2130,7 +2155,7 @@ func StartAgent() { Exclude: patterns, }) } - + if includePatterns := os.Getenv("SHUFFLE_AUDIT_LOG_INCLUDE"); includePatterns != "" { patterns := strings.Split(includePatterns, ",") telemetryConfig.Filters = append(telemetryConfig.Filters, shuffle.TelemetryFilter{ @@ -2151,7 +2176,7 @@ func StartAgent() { sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) - + go func() { <-sigChan log.Printf("[INFO] Received shutdown signal, stopping audit log collector...") @@ -2184,7 +2209,7 @@ func main() { if os.Getenv("SHUFFLE_PIPELINE_STANDALONE") == "true" { log.Printf("[INFO] Allowing use of standalone pipeline (tenzir). URL: %s", pipelineUrl) - + //if os.Getenv("SHUFFLE_SKIP_PIPELINES") == "false" { // os.Setenv("SHUFFLE_SKIP_PIPELINES", "true") //} @@ -3016,10 +3041,7 @@ func handlePipeline(incRequest shuffle.ExecutionRequest) error { func deployTenzirNode() error { // Disabled all pipeline features - if os.Getenv("SHUFFLE_SKIP_PIPELINES") == "false" || os.Getenv("SHUFFLE_PIPELINE_ENABLED") == "true" { - // return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES") - //log.Printf("[INFO] Pipelines are enabled by user") - } else { + if os.Getenv("SHUFFLE_SKIP_PIPELINES") != "true" { return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES") } @@ -3249,7 +3271,7 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri EndpointsConfig: map[string]*network.EndpointSettings{ "tenzir-network": { IPAMConfig: nil, - Aliases: []string{"tenzir-node"}, + Aliases: []string{"tenzir-node"}, }, }, } @@ -3909,7 +3931,10 @@ func sendPipelineHealthStatus() (shuffle.LakeConfig, error) { } else { //tenzirDisabled = true - log.Printf("[WARNING] Disabling pipelines: %s. You will need to restart the Orborus to fix this.", err) + if debug { + log.Printf("[WARNING] Disabling pipelines: %s. You will need to restart the Orborus to fix this.", err) + } + } return pipelinePayload, err From 697632c916efea079ee758d00a1f765ddc272e6e Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Wed, 12 Nov 2025 02:13:16 +0530 Subject: [PATCH 12/16] fix: patch network on the fly for workers --- functions/onprem/orborus/orborus.go | 30 ++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index a0f78845..b5bcf486 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -765,7 +765,36 @@ func deployServiceWorkers(image string) { if err == nil { log.Printf("[DEBUG] Successfully deployed workers with %d replica(s) on %d node(s)", replicas, cnt) + // wait for service to be ready + time.Sleep(time.Duration(rand.Intn(4)+1) * time.Second) + //log.Printf("[DEBUG] Servicecreate request: %#v %#v", service, err) + // patch service network + // this is an edgecase that we noticed on docker version 29 + // and API version 1.44 + services, serr := dockercli.ServiceList(ctx, types.ServiceListOptions{}) + if serr == nil { + for _, svc := range services { + if svc.Spec.Annotations.Name == innerContainerName { + log.Printf("[DEBUG] Found service %s (%s) — patching network attach", innerContainerName, svc.ID) + + spec := svc.Spec + spec.TaskTemplate.Networks = append(spec.TaskTemplate.Networks, swarm.NetworkAttachmentConfig{ + Target: networkID, + }) + + _, uerr := dockercli.ServiceUpdate(ctx, svc.ID, svc.Version, spec, types.ServiceUpdateOptions{}) + if uerr != nil { + log.Printf("[WARNING] Failed to patch service %s with network %s: %v", innerContainerName, networkID, uerr) + } else { + log.Printf("[INFO] Successfully attached network %s to service %s", networkID, innerContainerName) + } + break + } + } + } else { + log.Printf("[WARNING] Failed to list services for patching network attach: %v", serr) + } } else { if !strings.Contains(fmt.Sprintf("%s", err), "Already Exists") && !strings.Contains(fmt.Sprintf("%s", err), "is already in use by service") { log.Printf("[ERROR] Failed making service: %s", err) @@ -3926,7 +3955,6 @@ func sendPipelineHealthStatus() (shuffle.LakeConfig, error) { err := deployTenzirNode() if err != nil { if (!strings.Contains(err.Error(), "SHUFFLE_SKIP_PIPELINES") && !strings.Contains(err.Error(), "Kubernetes not implemented for Tenzir node")) && !strings.Contains(err.Error(), "Tenzir Node is already running") && !strings.Contains(err.Error(), "docker daemon") { - log.Printf("[ERROR] Tenzir node connection problem: %s", err) } else { From 7be4e6adf5d5dd05efca58114c195ab9a707188d Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Wed, 12 Nov 2025 02:48:32 +0530 Subject: [PATCH 13/16] fix: shuffle swarm network bug where it wasn't getting attached in swarm --- functions/onprem/worker/worker.go | 58 ++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 5a858888..c0596a8a 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -3454,7 +3454,6 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl _ = service if err != nil { - if strings.Contains(fmt.Sprintf("%s", err), "network") && strings.Contains(fmt.Sprintf("%s", err), "not found") { log.Printf("[DEBUG] Network %s not found. Trying to initialize it.", networkName) networkErr := initSwarmNetwork() @@ -3481,6 +3480,63 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl log.Printf("[DEBUG] Failed deploying %s with image %s: %s", name, image, err) return err + } else { + // wait for service to be ready + time.Sleep(time.Duration(rand.Intn(4)+1) * time.Second) + //log.Printf("[DEBUG] Servicecreate request: %#v %#v", service, err) + // patch service network + // this is an edgecase that we noticed on docker version 29 + // and API version 1.44 + + // get networkID of swarmNetworkName + networkID := "" + + ctx := context.Background() + + // find network ID + networks, err := dockercli.NetworkList(ctx, network.ListOptions{}) + if err == nil { + for _, net := range networks { + if net.Name == networkName { + if net.Scope == "swarm" { + log.Printf("[DEBUG] Found swarm-scoped network: %s (%s)", networkName, net.ID) + networkID = net.ID + } else { + log.Printf("[WARNING] Network %s exists but is not swarm scoped (scope=%s)", networkName, net.Scope) + } + break + } + } + } + + if networkID == "" { + log.Printf("[ERROR] Network %s not found", networkName) + networkID = networkName + } + + services, serr := dockercli.ServiceList(ctx, types.ServiceListOptions{}) + if serr == nil { + for _, svc := range services { + if svc.ID == service.ID { + log.Printf("[DEBUG] Found service %s (%s) — patching network attach", service.ID, svc.ID) + + spec := svc.Spec + spec.TaskTemplate.Networks = append(spec.TaskTemplate.Networks, swarm.NetworkAttachmentConfig{ + Target: networkID, + }) + + _, uerr := dockercli.ServiceUpdate(ctx, svc.ID, svc.Version, spec, types.ServiceUpdateOptions{}) + if uerr != nil { + log.Printf("[WARNING] Failed to patch service %s with network %s: %v", service.ID, networkID, uerr) + } else { + log.Printf("[INFO] Successfully attached network %s to service %s", networkID, service.ID) + } + break + } + } + } else { + log.Printf("[WARNING] Failed to list services for patching network attach: %v", serr) + } } log.Printf("[DEBUG] Successfully deployed service %s with image %s on port %d", name, image, deployport) From ff6cad8049d10248264750f6e5749e5dad48102a Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Wed, 12 Nov 2025 10:09:33 +0100 Subject: [PATCH 14/16] k8s: remove SHUFFLE_SWARM_CONFIG env variable from orborus --- .../charts/shuffle/templates/orborus/orborus-dpl.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml index 05ca1ac3..a786dcd2 100644 --- a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml +++ b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml @@ -86,8 +86,6 @@ spec: value: kubernetes - name: IS_KUBERNETES value: "true" - - name: SHUFFLE_SWARM_CONFIG - value: run {{- if .Values.orborus.extraEnvVars }} {{- include "common.tplvalues.render" (dict "value" .Values.orborus.extraEnvVars "context" $) | nindent 12 }} {{- end }} From 182a6a363fb3d6c6dd6fbe6248caf75dabd6cf88 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Wed, 12 Nov 2025 15:03:15 +0530 Subject: [PATCH 15/16] chore: bump shuffle-shared version --- backend/go-app/go.mod | 2 +- functions/onprem/orborus/go.mod | 2 +- functions/onprem/worker/go.mod | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 02239000..3e8a4c0d 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -26,7 +26,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.51 + github.com/shuffle/shuffle-shared v0.9.52 github.com/shuffle/singul v0.0.20 golang.org/x/crypto v0.40.0 google.golang.org/api v0.236.0 diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 36ff423a..67e428c7 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -10,7 +10,7 @@ require ( github.com/docker/docker v28.3.3+incompatible github.com/docker/go-connections v0.5.0 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.51 + github.com/shuffle/shuffle-shared v0.9.52 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 ) diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 085375c5..1227cfba 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -11,7 +11,7 @@ require ( github.com/docker/docker v28.3.3+incompatible github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.39 + github.com/shuffle/shuffle-shared v0.9.52 github.com/shuffle/singul v0.0.19 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 @@ -60,7 +60,7 @@ require ( github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/frikky/kin-openapi v0.42.0 // indirect - github.com/frikky/schemaless v0.0.23 // indirect + github.com/frikky/schemaless v0.0.24 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect From 48f19db0a6ab45efaba926ada58f38b6472f6e95 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Wed, 12 Nov 2025 15:09:25 +0530 Subject: [PATCH 16/16] singul version bump --- functions/onprem/worker/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 1227cfba..f6d2762b 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/satori/go.uuid v1.2.0 github.com/shuffle/shuffle-shared v0.9.52 - github.com/shuffle/singul v0.0.19 + github.com/shuffle/singul v0.0.20 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 k8s.io/client-go v0.33.1