Merge branch 'nightly' of https://github.com/shuffle/shuffle into nightly

.env

@@ -96,6 +96,8 @@ SHUFFLE_WORKER_SERVER_URL=
 SHUFFLE_ORBORUS_PULL_TIME=
 # Max recursion depth for subflows
 SHUFFLE_MAX_EXECUTION_DEPTH=
+# Amount of app replicas
+SHUFFLE_APP_REPLICAS=3
 
 # DATABASE CONFIGURATIONS
 DATASTORE_EMULATOR_HOST=shuffle-database:8000

backend/Dockerfile

@@ -27,7 +27,7 @@ FROM alpine:latest as certs
 RUN apk add --update ca-certificates
 
 # Sets up the final image
-FROM alpine:3.22.1
+FROM alpine:3.22.2
 
 # FIXME: Install cgo because CGO_ENABLED=1 during build
 RUN apk add --no-cache libc6-compat

backend/go-app/go.mod

@@ -26,7 +26,7 @@ require (
 	github.com/gorilla/mux v1.8.1
 	github.com/h2non/filetype v1.1.3
 	github.com/satori/go.uuid v1.2.0
-	github.com/shuffle/shuffle-shared v0.9.51
+	github.com/shuffle/shuffle-shared v0.9.52
 	github.com/shuffle/singul v0.0.20
 	golang.org/x/crypto v0.40.0
 	google.golang.org/api v0.236.0

backend/go-app/walkoff.go

@@ -1505,13 +1505,16 @@ func stopScheduleGCP(resp http.ResponseWriter, request *http.Request) {
 	return
 }
 
-func deleteSchedule(ctx context.Context, id string) error {
+func deleteKeySchedule(ctx context.Context, id string) error {
-	log.Printf("[DEBUG] Should stop schedule %s!", id)
 	err := shuffle.DeleteKey(ctx, "schedules", id)
 	if err != nil {
-		log.Printf("[ERROR] Failed to delete schedule: %s", err)
 		return err
-	} else {
+	}
+	return nil
+}
+
+func deleteSchedule(ctx context.Context, id string) error {
+	log.Printf("[DEBUG] Should stop schedule %s!", id)
 	if value, exists := scheduledJobs[id]; exists {
 		// Stops the schedule properly
 		value.Lock()
@@ -1524,11 +1527,17 @@ func deleteSchedule(ctx context.Context, id string) error {
 				return err
 			}
 		} else {
+			// Just stop and delete anyway if not in memory
+			deleteKeySchedule(ctx, id)
 			return errors.New("Can't find the schedule.")
 		}
 	}
-	}
 
+	err := deleteKeySchedule(ctx, id)
+	if err != nil {
+		log.Printf("[ERROR] Failed to stop schedule in db %s: %s", id, err)
+		return err
+	}
 	return nil
 }
 

docker-compose.yml

@@ -42,8 +42,6 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock
     environment:
       - SHUFFLE_APP_SDK_TIMEOUT=300
-      - SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY=7 # The amount of concurrent executions Orborus can handle. 
-      #- DOCKER_HOST=tcp://docker-socket-proxy:2375
       - ENVIRONMENT_NAME=Shuffle
       - ORG_ID=Shuffle
       - BASE_URL=http://${OUTER_HOSTNAME}:5001
@@ -52,9 +50,11 @@ services:
       - HTTPS_PROXY=${HTTPS_PROXY}
       - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY}
       - SHUFFLE_PASS_APP_PROXY=${SHUFFLE_PASS_APP_PROXY}
+      - SHUFFLE_APP_REPLICAS=${SHUFFLE_APP_REPLICAS}
       - SHUFFLE_STATS_DISABLED=true
       - SHUFFLE_LOGS_DISABLED=true
       - SHUFFLE_SWARM_CONFIG=run
+      - SHUFFLE_APP_REPLICAS=3
       - CLEANUP=false
       - SHUFFLE_WORKER_IMAGE=ghcr.io/shuffle/shuffle-worker:latest
     env_file: .env
@@ -122,10 +122,9 @@ services:
   #    - MEMCACHED_MAX_CONNECTIONS=2500
   #  ports:
   #    - 11211:11211
-  #  deploy:
+  #  networks:
-  #    resources:
+  #  -  shuffle
-  #      limits:
+  #  -  swarm_executions
-  #        memory: 2048M
 
   #docker-socket-proxy:
   #  image: tecnativa/docker-socket-proxy

frontend/Dockerfile

@@ -27,7 +27,7 @@ COPY ./*.json /usr/src/app/
 RUN npm run build --loglevel verbose 2>&1
 
 # Production environment
-FROM nginx:1.29.0
+FROM nginx:1.29.3
 
 RUN mkdir -p /usr/share/nginx/html/build
 RUN mkdir -p /usr/share/nginx/html/css

functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml

@@ -86,8 +86,6 @@ spec:
               value: kubernetes
             - name: IS_KUBERNETES
               value: "true"
-            - name: SHUFFLE_SWARM_CONFIG
-              value: run
             {{- if .Values.orborus.extraEnvVars }}
             {{- include "common.tplvalues.render" (dict "value" .Values.orborus.extraEnvVars "context" $) | nindent 12 }}
             {{- end }}

functions/onprem/orborus/go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/docker/docker v28.3.3+incompatible
 	github.com/docker/go-connections v0.5.0
 	github.com/satori/go.uuid v1.2.0
-	github.com/shuffle/shuffle-shared v0.9.51
+	github.com/shuffle/shuffle-shared v0.9.52
 	k8s.io/api v0.33.1
 	k8s.io/apimachinery v0.33.1
 )

functions/onprem/orborus/orborus.go

@@ -441,12 +441,32 @@ func deployServiceWorkers(image string) {
 	if err != nil {
 		if strings.Contains(fmt.Sprintf("%s", err), "already exists") {
 			// Try patching for attachable
-
+			if debug {
+				log.Printf("[DEBUG] Network %s already exists", networkName)
+			}
 		} else {
 			log.Printf("[DEBUG] Failed to create network %s for workers: %s. This is not critical, and containers will still be added", networkName, err)
 		}
 	}
 
+	networkID := ""
+
+	// find network ID
+	networks, err := dockercli.NetworkList(ctx, network.ListOptions{})
+	if err == nil {
+		for _, net := range networks {
+			if net.Name == networkName {
+				if net.Scope == "swarm" {
+					log.Printf("[DEBUG] Found swarm-scoped network: %s (%s)", networkName, net.ID)
+					networkID = net.ID
+				} else {
+					log.Printf("[WARNING] Network %s exists but is not swarm scoped (scope=%s)", networkName, net.Scope)
+				}
+				break
+			}
+		}
+	}
+
 	/*
 		isMemcachedRunning, err := checkMemcached(ctx, dockercli)
 		if err != nil {
@@ -463,12 +483,17 @@ func deployServiceWorkers(image string) {
 		}
 	*/
 
+	if networkID == "" {
+		log.Printf("[ERROR] Network %s does not exist", networkName)
+		networkID = networkName
+	}
+
 	defaultNetworkAttach := false
 	if containerId != "" {
 		log.Printf("[DEBUG] Should connect orborus container to worker network as it's running in Docker with name %#v!", containerId)
 		// https://pkg.go.dev/github.com/docker/docker@v20.10.12+incompatible/api/types/network#EndpointSettings
 		networkConfig := &network.EndpointSettings{}
-		err := dockercli.NetworkConnect(ctx, networkName, containerId, networkConfig)
+		err := dockercli.NetworkConnect(ctx, networkID, containerId, networkConfig)
 		if err != nil {
 			log.Printf("[ERROR] Failed connecting Orborus to docker network %s: %s", networkName, err)
 		}
@@ -491,7 +516,7 @@ func deployServiceWorkers(image string) {
 			for _, container := range containers {
 				if strings.Contains(strings.ToLower(container.Image), "docker-socket-proxy") {
 					networkConfig := &network.EndpointSettings{}
-					err := dockercli.NetworkConnect(ctx, networkName, container.ID, networkConfig)
+					err := dockercli.NetworkConnect(ctx, networkID, container.ID, networkConfig)
 					if err != nil {
 						log.Printf("[ERROR] Failed connecting Docker socket proxy to docker network %s: %s", networkName, err)
 					} else {
@@ -571,7 +596,7 @@ func deployServiceWorkers(image string) {
 		},
 		Networks: []swarm.NetworkAttachmentConfig{
 			swarm.NetworkAttachmentConfig{
-				Target: networkName,
+				Target: networkID,
 			},
 			swarm.NetworkAttachmentConfig{
 				Target: "ingress",
@@ -740,7 +765,36 @@ func deployServiceWorkers(image string) {
 
 	if err == nil {
 		log.Printf("[DEBUG] Successfully deployed workers with %d replica(s) on %d node(s)", replicas, cnt)
+		// wait for service to be ready
+		time.Sleep(time.Duration(rand.Intn(4)+1) * time.Second)
+
 		//log.Printf("[DEBUG] Servicecreate request: %#v %#v", service, err)
+		// patch service network
+		// this is an edgecase that we noticed on docker version 29
+		// and API version 1.44
+		services, serr := dockercli.ServiceList(ctx, types.ServiceListOptions{})
+		if serr == nil {
+			for _, svc := range services {
+				if svc.Spec.Annotations.Name == innerContainerName {
+					log.Printf("[DEBUG] Found service %s (%s) — patching network attach", innerContainerName, svc.ID)
+
+					spec := svc.Spec
+					spec.TaskTemplate.Networks = append(spec.TaskTemplate.Networks, swarm.NetworkAttachmentConfig{
+						Target: networkID,
+					})
+
+					_, uerr := dockercli.ServiceUpdate(ctx, svc.ID, svc.Version, spec, types.ServiceUpdateOptions{})
+					if uerr != nil {
+						log.Printf("[WARNING] Failed to patch service %s with network %s: %v", innerContainerName, networkID, uerr)
+					} else {
+						log.Printf("[INFO] Successfully attached network %s to service %s", networkID, innerContainerName)
+					}
+					break
+				}
+			}
+		} else {
+			log.Printf("[WARNING] Failed to list services for patching network attach: %v", serr)
+		}
 	} else {
 		if !strings.Contains(fmt.Sprintf("%s", err), "Already Exists") && !strings.Contains(fmt.Sprintf("%s", err), "is already in use by service") {
 			log.Printf("[ERROR] Failed making service: %s", err)
@@ -3016,10 +3070,7 @@ func handlePipeline(incRequest shuffle.ExecutionRequest) error {
 
 func deployTenzirNode() error {
 	// Disabled all pipeline features
-	if os.Getenv("SHUFFLE_SKIP_PIPELINES") == "false" || os.Getenv("SHUFFLE_PIPELINE_ENABLED") == "true" {
+	if os.Getenv("SHUFFLE_SKIP_PIPELINES") != "true" {
-		// return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES")
-		//log.Printf("[INFO] Pipelines are enabled by user")
-	} else {
 		return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES")
 	}
 
@@ -3904,14 +3955,16 @@ func sendPipelineHealthStatus() (shuffle.LakeConfig, error) {
 	err := deployTenzirNode()
 	if err != nil {
 		if (!strings.Contains(err.Error(), "SHUFFLE_SKIP_PIPELINES") && !strings.Contains(err.Error(), "Kubernetes not implemented for Tenzir node")) && !strings.Contains(err.Error(), "Tenzir Node is already running") && !strings.Contains(err.Error(), "docker daemon") {
-
 			log.Printf("[ERROR] Tenzir node connection problem: %s", err)
 
 		} else {
 			//tenzirDisabled = true
+			if debug {
 				log.Printf("[WARNING] Disabling pipelines: %s. You will need to restart the Orborus to fix this.", err)
 			}
 
+		}
+
 		return pipelinePayload, err
 	}
 

functions/onprem/worker/go.mod

@@ -11,8 +11,8 @@ require (
 	github.com/docker/docker v28.3.3+incompatible
 	github.com/gorilla/mux v1.8.1
 	github.com/satori/go.uuid v1.2.0
-	github.com/shuffle/shuffle-shared v0.9.39
+	github.com/shuffle/shuffle-shared v0.9.52
-	github.com/shuffle/singul v0.0.19
+	github.com/shuffle/singul v0.0.20
 	k8s.io/api v0.33.1
 	k8s.io/apimachinery v0.33.1
 	k8s.io/client-go v0.33.1
@@ -60,7 +60,7 @@ require (
 	github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/frikky/kin-openapi v0.42.0 // indirect
-	github.com/frikky/schemaless v0.0.23 // indirect
+	github.com/frikky/schemaless v0.0.24 // indirect
 	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect

functions/onprem/worker/worker.go

@@ -2555,6 +2555,11 @@ func getWorkerBackendExecution(auth string, executionId string) (*shuffle.Workfl
 		return workflowExecution, err
 	}
 
+	if debug {
+		log.Printf("[INFO] Here is the result we got back from backend: %s", workflowExecution.Results)
+	}
+
+	setWorkflowExecution(context.Background(), *workflowExecution, false)
 	return workflowExecution, nil
 }
 
@@ -3454,7 +3459,6 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl
 	_ = service
 
 	if err != nil {
-
 		if strings.Contains(fmt.Sprintf("%s", err), "network") && strings.Contains(fmt.Sprintf("%s", err), "not found") {
 			log.Printf("[DEBUG] Network %s not found. Trying to initialize it.", networkName)
 			networkErr := initSwarmNetwork()
@@ -3481,6 +3485,63 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl
 
 		log.Printf("[DEBUG] Failed deploying %s with image %s: %s", name, image, err)
 		return err
+	} else {
+		// wait for service to be ready
+		time.Sleep(time.Duration(rand.Intn(4)+1) * time.Second)
+		//log.Printf("[DEBUG] Servicecreate request: %#v %#v", service, err)
+		// patch service network
+		// this is an edgecase that we noticed on docker version 29
+		// and API version 1.44
+
+		// get networkID of swarmNetworkName
+		networkID := ""
+
+		ctx := context.Background()
+
+		// find network ID
+		networks, err := dockercli.NetworkList(ctx, network.ListOptions{})
+		if err == nil {
+			for _, net := range networks {
+				if net.Name == networkName {
+					if net.Scope == "swarm" {
+						log.Printf("[DEBUG] Found swarm-scoped network: %s (%s)", networkName, net.ID)
+						networkID = net.ID
+					} else {
+						log.Printf("[WARNING] Network %s exists but is not swarm scoped (scope=%s)", networkName, net.Scope)
+					}
+					break
+				}
+			}
+		}
+
+		if networkID == "" {
+			log.Printf("[ERROR] Network %s not found", networkName)
+			networkID = networkName
+		}
+
+		services, serr := dockercli.ServiceList(ctx, types.ServiceListOptions{})
+		if serr == nil {
+			for _, svc := range services {
+				if svc.ID == service.ID {
+					log.Printf("[DEBUG] Found service %s (%s) — patching network attach", service.ID, svc.ID)
+
+					spec := svc.Spec
+					spec.TaskTemplate.Networks = append(spec.TaskTemplate.Networks, swarm.NetworkAttachmentConfig{
+						Target: networkID,
+					})
+
+					_, uerr := dockercli.ServiceUpdate(ctx, svc.ID, svc.Version, spec, types.ServiceUpdateOptions{})
+					if uerr != nil {
+						log.Printf("[WARNING] Failed to patch service %s with network %s: %v", service.ID, networkID, uerr)
+					} else {
+						log.Printf("[INFO] Successfully attached network %s to service %s", networkID, service.ID)
+					}
+					break
+				}
+			}
+		} else {
+			log.Printf("[WARNING] Failed to list services for patching network attach: %v", serr)
+		}
 	}
 
 	log.Printf("[DEBUG] Successfully deployed service %s with image %s on port %d", name, image, deployport)