Merge pull request #1820 from Shuffle/main

Syncing new fixes in main back to nightly

.env

@@ -86,7 +86,7 @@ SHUFFLE_CONTAINER_AUTO_CLEANUP=true
 SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY=5
 SHUFFLE_HEALTHCHECK_DISABLED=false 
 SHUFFLE_ELASTIC=true
-SHUFFLE_LOGS_DISABLED=false
+SHUFFLE_LOGS_DISABLED=true
 SHUFFLE_CHAT_DISABLED=false
 SHUFFLE_DISABLE_RERUN_AND_ABORT=false
 SHUFFLE_RERUN_SCHEDULE=300
@@ -99,7 +99,6 @@ SHUFFLE_MAX_EXECUTION_DEPTH=
 
 # DATABASE CONFIGURATIONS
 DATASTORE_EMULATOR_HOST=shuffle-database:8000
-#SHUFFLE_OPENSEARCH_URL=http://shuffle-opensearch:9200
 SHUFFLE_OPENSEARCH_URL=https://shuffle-opensearch:9200
 SHUFFLE_OPENSEARCH_CERTIFICATE_FILE=
 SHUFFLE_OPENSEARCH_APIKEY=

.github/workflows/dockerbuild.yaml

@@ -20,19 +20,19 @@ jobs:
         include:
           - app: frontend
             path: frontend
-            version: 2.1.0-rc2
+            version: 2.1.0
             experimental: true
           - app: backend
             path: backend
-            version: 2.1.0-rc2
+            version: 2.1.0
             experimental: true
           - app: orborus
             path: functions/onprem/orborus
-            version: 2.1.0-rc2
+            version: 2.1.0
             experimental: true
           - app: worker
             path: functions/onprem/worker
-            version: 2.1.0-rc2
+            version: 2.1.0
             experimental: true
     steps:
       - name: Checkout

backend/Dockerfile

@@ -20,7 +20,7 @@ RUN go mod tidy
 # From November 2022, CGO is enabled due to packages
 # that we use requiring it. This is a temporary fix
 # and makes us HAVE to install libc compatibility packages farther down.
-RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o webapp .
+RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o shufflebackend .
 
 # Certificate build - gets required certs
 FROM alpine:latest as certs
@@ -40,4 +40,4 @@ COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certifica
 
 WORKDIR /app
 EXPOSE 5001
-CMD ["./webapp"]
+CMD ["./shufflebackend"]

backend/app_gen/python-lib/baseline/Dockerfile

@@ -1,26 +1,26 @@
-# Base our app image off of the WALKOFF App SDK image
 FROM frikky/shuffle:app_sdk as base
 
 # We're going to stage away all of the bloat from the build tools so lets create a builder stage
 FROM base as builder
 
 # Install all alpine build tools needed for our pip installs
-RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev 
+RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev git
 
 # Install all of our pip packages in a single directory that we can copy to our base image later
 RUN mkdir /install
 WORKDIR /install
 COPY requirements.txt /requirements.txt
-RUN pip install --prefix="/install" -r /requirements.txt
+RUN pip install --no-cache-dir --upgrade --prefix="/install" -r /requirements.txt
 
 # Switch back to our base image and copy in all of our built packages and source code
 FROM base
 COPY --from=builder /install /usr/local
 COPY src /app
 
-# Install any binary dependencies needed in our final image - this can be a lot of different stuff
+# Install any binary dependencies needed in our final image
-RUN apk --no-cache add --update libmagic
+# RUN apk --no-cache add --update my_binary_dependency
+RUN apk --no-cache add jq git curl
 
 # Finally, lets run our app!
 WORKDIR /app
-CMD python app.py --log-level DEBUG
+CMD ["python", "app.py", "--log-level", "DEBUG"]

backend/go-app/go.mod

@@ -24,7 +24,7 @@ require (
 	github.com/gorilla/mux v1.8.1
 	github.com/h2non/filetype v1.1.3
 	github.com/satori/go.uuid v1.2.0
-	github.com/shuffle/shuffle-shared v0.9.16
+	github.com/shuffle/shuffle-shared v0.9.25
 	github.com/shuffle/singul v0.0.16
 	golang.org/x/crypto v0.40.0
 	google.golang.org/api v0.236.0

backend/go-app/go.sum

@@ -363,8 +363,8 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G
 github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8=
 github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8=
 github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
-github.com/shuffle/shuffle-shared v0.9.16 h1:B3Dp3h2f62JdWmqQ1kTu7DwJY6HCOMrxxSojAeuhI9k=
+github.com/shuffle/shuffle-shared v0.9.21 h1:ZxA/g0Lk93miHJpVYAhtTQmtWLJWcCtm6kFNF772Ynk=
-github.com/shuffle/shuffle-shared v0.9.16/go.mod h1:PhDEizuz4SmJaSmy0+yrFWwD1mXVUsy8/knKlrqF1qw=
+github.com/shuffle/shuffle-shared v0.9.21/go.mod h1:PhDEizuz4SmJaSmy0+yrFWwD1mXVUsy8/knKlrqF1qw=
 github.com/shuffle/singul v0.0.16 h1:dW+0Mln9R1aUJ0fjikpWcxbjoQWqJHxe4kSxh2tQN5E=
 github.com/shuffle/singul v0.0.16/go.mod h1:LYkp320A6gsoPlYbXUM+WvEPUVAuutlSsqnVKyRy4gs=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=

backend/go-app/main.go

@@ -3335,11 +3335,16 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User, s
 	// Read and copy the baseline Dockerfile
 	dockerfileContent, err := ioutil.ReadFile(dockerfileSource)
 	if err != nil {
+		foundDockerfile := shuffle.GetBaseDockerfile() 
+		if len(foundDockerfile) > 0 {
+			dockerfileContent = foundDockerfile
+		} else {
 			log.Printf("[ERROR] Failed to read baseline Dockerfile: %s", err)
 			resp.WriteHeader(500)
 			resp.Write([]byte(`{"success": false, "reason": "Failed to read baseline Dockerfile"}`))
 			return
 		}
+	}
 
 	err = ioutil.WriteFile(dockerfileDestination, dockerfileContent, 0644)
 	if err != nil {

docker-compose.yml

@@ -55,6 +55,7 @@ services:
       - SHUFFLE_STATS_DISABLED=true
       - SHUFFLE_LOGS_DISABLED=true
       - SHUFFLE_SWARM_CONFIG=run
+      - CLEANUP=false
       - SHUFFLE_WORKER_IMAGE=ghcr.io/shuffle/shuffle-worker:latest
     env_file: .env
     restart: unless-stopped
@@ -65,7 +66,7 @@ services:
     hostname: shuffle-opensearch
     container_name: shuffle-opensearch
     environment:
-      - "OPENSEARCH_JAVA_OPTS=-Xms4096m -Xmx4096m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
+      - "OPENSEARCH_JAVA_OPTS=-Xms3072m -Xmx3072m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
       - bootstrap.memory_lock=true
       - DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI=true
       - cluster.initial_master_nodes=shuffle-opensearch
@@ -94,12 +95,16 @@ services:
   #  image: memcached:latest
   #  container_name: shuffle-cache
   #  hostname: shuffle-cache
-  #  mem_limit: 1024m
+  #  command: memcached -m 2048 -c 2500
   #  environment:
-  #    - MEMCACHED_MEMORY=1024
+  #    - MEMCACHED_MEMORY=2048
   #    - MEMCACHED_MAX_CONNECTIONS=2500
   #  ports:
   #    - 11211:11211
+  #  deploy:
+  #    resources:
+  #      limits:
+  #        memory: 2048M
   
   #docker-socket-proxy:
   #  image: tecnativa/docker-socket-proxy

functions/kubernetes/all-in-one.yaml

@@ -90,7 +90,7 @@ data:
   IS_KUBERNETES: "true"
   REGISTRY_URL: "docker-registry:5000"
   REGISTRY_AUTH: "false"
-  SHUFFLE_KUBERNETES_WORKER: "ghcr.io/shuffle/shuffle-worker:nightly"
+  SHUFFLE_KUBERNETES_WORKER: "ghcr.io/shuffle/shuffle-worker:latest"
 kind: ConfigMap
 
 ---
@@ -702,7 +702,7 @@ spec:
                 configMapKeyRef:
                   key: REGISTRY_AUTH
                   name: env
-          image: ghcr.io/shuffle/shuffle-backend:nightly
+          image: ghcr.io/shuffle/shuffle-backend:latest
           imagePullPolicy: Always
           name: shuffle-backend
           ports:
@@ -769,7 +769,7 @@ spec:
     spec:
       containers:
         - name: shuffle-frontend
-          image: ghcr.io/shuffle/shuffle-frontend:nightly
+          image: ghcr.io/shuffle/shuffle-frontend:latest
           env:
             - name: BACKEND_HOSTNAME
               valueFrom:
@@ -853,13 +853,13 @@ spec:
             - name: ORG_ID
               value: Shuffle
             - name: SHUFFLE_APP_SDK_VERSION
-              value: nightly
+              value: latest 
             - name: SHUFFLE_SCALE_REPLICAS
               value: "5"
             - name: SHUFFLE_SWARM_CONFIG
               value: run
             - name: SHUFFLE_WORKER_VERSION
-              value: nightly
+              value: latest
             - name: IS_KUBERNETES
               valueFrom:
                 configMapKeyRef:
@@ -885,7 +885,7 @@ spec:
                 configMapKeyRef:
                   key: SHUFFLE_MEMCACHED
                   name: env
-          image: ghcr.io/shuffle/shuffle-orborus:nightly
+          image: ghcr.io/shuffle/shuffle-orborus:latest
           #imagePullPolicy: Never
           name: shuffle-orborus
           resources: {}

functions/kubernetes/charts/shuffle/Chart.yaml

@@ -3,7 +3,7 @@ name: shuffle
 description: A Helm chart for deploying Shuffle on Kubernetes
 type: application
 version: 0.0.0 # Set during publishing in GitHub actions
-appVersion: nightly # Overwritten during publishing in GitHub actions
+appVersion: latest # Overwritten during publishing in GitHub actions
 dependencies:
   - name: common
     version: ^2.23.0

functions/kubernetes/orborus.yaml

@@ -72,7 +72,7 @@ spec:
             - name: AUTH 
               value: "3663a270-bb3a-4678-a365-d879601a1a0c"
 
-          image: ghcr.io/shuffle/shuffle-orborus:nightly
+          image: ghcr.io/shuffle/shuffle-orborus:latest
           #imagePullPolicy: Never
           name: shuffle-orborus
           resources: {}

functions/onprem/orborus/orborus.go

@@ -12,6 +12,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"regexp"
 	"log"
 	"math"
 	"net"
@@ -195,10 +196,10 @@ func getThisContainerId() {
 	if containerId == "" {
 		if containerName != "" {
 			containerId = containerName
-			log.Printf("[INFO] Falling back to CONTAINER_NAME as container ID")
+			log.Printf("[INFO] Falling back to ORBORUS_CONTAINER_NAME as container ID")
 		} else {
 			containerId = "shuffle-orborus"
-			log.Printf(`[WARNING] CONTAINER_NAME is not set. Falling back to default name "%s" as container ID`, containerId)
+			log.Printf(`[WARNING] ORBORUS_CONTAINER_NAME env is not set. Falling back to default name "%s" as container ID. This may cause issues on the same server`, containerId)
 		}
 	}
 
@@ -219,8 +220,8 @@ func skipCheckInCleanup(name string) bool {
 }
 
 func cleanupExistingNodes(ctx context.Context) error {
-	if cleanupEnv == "false" {
+	if cleanupEnv != "true" {
-		log.Printf("[INFO] Skipping cleanup of existing workers as CLEANUP is set to false. This should be auto-discovered during executions then instead.")
+		log.Printf("[INFO] Skipping cleanup of existing workers as CLEANUP is NOT set to true. Swarm actions are being auto-discovered during executions then instead.")
 		return nil
 	}
 
@@ -342,7 +343,10 @@ func deployServiceWorkers(image string) {
 	if len(dockerSwarmBridgeMTU) == 0 {
 		mtu, err = strconv.Atoi(dockerSwarmBridgeMTU) // by default
 		if err != nil {
+			if debug { 
 				log.Printf("[DEBUG] Failed to convert the default MTU to int: %s. Using 1500 instead. Input: %s", err, dockerSwarmBridgeMTU)
+			}
+
 			mtu = 1500
 		}
 	}
@@ -526,6 +530,7 @@ func deployServiceWorkers(image string) {
 		nodeCount = uint64(cnt)
 	}
 
+
 	appReplicas := os.Getenv("SHUFFLE_APP_REPLICAS")
 	appReplicaCnt := 2
 	if len(appReplicas) > 0 {
@@ -538,6 +543,9 @@ func deployServiceWorkers(image string) {
 	}
 
 	log.Printf("[DEBUG] Found %d node(s) to replicate over. Defaulting to 1 IF we can't auto-discover them.", cnt)
+
+	// FIXME: From September 2025 - This is set back to 1, as this doesn't really reflect how scale works at all. It is just confusing, and makes number larger/smaller "arbitrarily" instead of using default docker scale
+	nodeCount = 1
 	replicatedJobs := uint64(replicas * nodeCount)
 
 	log.Printf("[DEBUG] Deploying %d container(s) for worker with swarm to each node. Service name: %s. Image: %s", replicas, innerContainerName, image)
@@ -1050,6 +1058,13 @@ func deployK8sWorker(image string, identifier string, env []string) error {
 		env = append(env, fmt.Sprintf("SHUFFLE_BASE_IMAGE_REGISTRY=%s", os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")))
 	}
 
+	if len(os.Getenv("SHUFFLE_BASE_IMAGE_NAME")) > 0 {
+		env = append(env, fmt.Sprintf("SHUFFLE_BASE_IMAGE_NAME=%s", os.Getenv("SHUFFLE_BASE_IMAGE_NAME")))
+	} else {
+		log.Printf("[INFO] SHUFFLE_BASE_IMAGE_NAME is not set. Defaulting to %s", baseimagename)
+		env = append(env, fmt.Sprintf("SHUFFLE_BASE_IMAGE_NAME=%s", baseimagename))
+	}
+
 	if len(os.Getenv("REGISTRY_URL")) > 0 {
 		env = append(env, fmt.Sprintf("REGISTRY_URL=%s", os.Getenv("REGISTRY_URL")))
 	}
@@ -1196,10 +1211,13 @@ func deployK8sWorker(image string, identifier string, env []string) error {
 		ImagePullPolicy: corev1.PullIfNotPresent,
 	}
 
+	if len(os.Getenv("REGISTRY_URL")) > 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_NAME")) > 0 {
+		log.Printf("[INFO] Setting image pull policy to Always as private registry is used.")
+		containerAttachment.ImagePullPolicy = corev1.PullAlways
+	}
+
 	podname := shuffle.GetPodName()
-
 	ctx := context.Background()
-
 	if len(podname) > 0 {
 		_, err := shuffle.GetCurrentPodNetworkConfig(ctx, clientset, kubernetesNamespace, podname)
 		if err != nil {
@@ -1700,17 +1718,45 @@ func checkSwarmService(ctx context.Context) {
 		return
 	}
 
+	listenAddr := "0.0.0.0"
 	req := swarm.InitRequest{
-		ListenAddr:    "0.0.0.0:2377",
+		ListenAddr:    fmt.Sprintf("%s:2377", listenAddr),
 		AdvertiseAddr: fmt.Sprintf("%s:2377", ip),
 	}
 
-	ret, err := dockercli.SwarmInit(ctx, req)
+	id, err := dockercli.SwarmInit(ctx, req)
 	if err != nil {
-		log.Printf("[WARNING] Swarm init: %s", err)
+		log.Printf("[ERROR] Swarm init issue: %s. Retrying with a failover IP address from interface.", err)
+
+		// Dummy message used for testing
+		//err = errors.New("Error response from daemon: could not choose an IP address to advertise since this system has multiple addresses on different interfaces (10.52.208.221 on eno1 and 192.168.122.1 on virbr0) - specify one with --advertise-addr")
+
+		msg := err.Error()
+
+		// Extract all IPv4 addresses from the error message
+		var ipv4Re = regexp.MustCompile(`\b(?:\d{1,3}\.){3}\d{1,3}\b`)
+		candidates := ipv4Re.FindAllString(msg, -1)
+		if len(candidates) > 0 {
+			// Pick the first valid candidate (or implement your own heuristic)
+
+			for cnt, candidate := range candidates {
+				if cnt > 5 {
+					break
 				}
 
-	log.Printf("[DEBUG] Swarm info: %s\n\n", ret)
+				req.AdvertiseAddr = fmt.Sprintf("%s:2377", candidate)
+				_, err = dockercli.SwarmInit(context.Background(), req)
+				if err != nil {
+					continue
+				}
+
+				break
+			}
+
+		}
+	}
+
+	log.Printf("[INFO] Swarm init ID: '%s'. If this is empty, there is most likely an error.", id)
 }
 
 func getContainerResourceUsage(ctx context.Context, cli *dockerclient.Client, containerID string) (float64, float64, error) {
@@ -3939,10 +3985,11 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest, image string,
 
 	identifier := "shuffle-workers"
 	if isKubernetes == "true" {
-		if shuffle.IsRunningInCluster() {
+		// FIXME: Do we need this to map the cluster?
-			log.Printf("[INFO] Running in Kubernetes cluster")
+		//if shuffle.IsRunningInCluster() {
+		//log.Printf("[INFO] Running in Kubernetes cluster")
 		// try getting the k8s worker server url
-		}
+		//}
 	}
 
 	if strings.Contains(streamUrl, "shuffler.io") || strings.Contains(streamUrl, "localhost") || strings.Contains(streamUrl, "127.0.0.1") || strings.Contains(streamUrl, "shuffle-backend") {

functions/onprem/worker/worker.go

@@ -662,6 +662,14 @@ func deployk8sApp(image string, identifier string, env []string) error {
 		},
 	}
 
+	if len(os.Getenv("REGISTRY_URL")) > 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_NAME")) > 0 {
+		log.Printf("[INFO] Setting image pull policy to Always as private registry is used.")
+		//containerAttachment.ImagePullPolicy = corev1.PullAlways
+		deployment.Spec.Template.Spec.Containers[0].ImagePullPolicy = corev1.PullAlways
+	} else {
+		deployment.Spec.Template.Spec.Containers[0].ImagePullPolicy = corev1.PullIfNotPresent
+	}
+
 	_, err = clientset.AppsV1().Deployments(kubernetesNamespace).Create(context.Background(), deployment, metav1.CreateOptions{})
 	if err != nil {
 		log.Printf("[ERROR] Failed creating deployment: %v", err)
@@ -1302,8 +1310,7 @@ func getWorkerURLs() ([]string, error) {
 		// 		}
 		// 	}
 		// }
-
+		//log.Printf("[DEBUG] Worker URLs for k8s: %#v", workerUrls)
-		log.Printf("[DEBUG] Worker URLs for k8s: %#v", workerUrls)
 
 		return workerUrls, nil
 	}
@@ -3219,10 +3226,8 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl
 	}
 
 	// Apps used a lot should have 2 replicas (default)
-	replicas := uint64(1)
+	// New default to 3 (as the chance of queues piling up is lower)
-	//if (strings.Contains(strings.ToLower(name), "shuffle") && strings.Contains(strings.ToLower(name), "tools")) || strings.Contains(strings.ToLower(name), "http") {
+	replicas := uint64(3)
-	//	replicas = 2
-	//}
 
 	// Sent from Orborus
 	// Should be equal to
@@ -3239,24 +3244,28 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl
 	}
 
 	// Max scale as well
+	nodeCount := uint64(1)
 	if inputReplicas > 0 && inputReplicas < 100 { 
 		if replicas != uint64(inputReplicas) {
 			log.Printf("[DEBUG] Overwriting replicas to %d/node as inputReplicas is set to %d", inputReplicas, inputReplicas)
 		}
 
 		replicas = uint64(inputReplicas)
-	}
+	} else {
-
 		cnt, err := findActiveSwarmNodes(dockercli)
 		if err != nil {
 			log.Printf("[ERROR] Unable to find active swarm nodes: %s", err)
 		}
 
-	nodeCount := uint64(1)
 		if cnt > 0 {
 			nodeCount = uint64(cnt)
 		}
 
+		// FIXME: From September 2025 - This is set back to 1, as this doesn't really reflect how scale works at all. It is just confusing, and makes number larger/smaller "arbitrarily" instead of using default docker scale
+		nodeCount = 1
+	}
+
+
 	replicatedJobs := uint64(replicas * nodeCount)
 	log.Printf("[DEBUG] Deploying app with name %s with image %s", name, image)
 
@@ -3615,7 +3624,10 @@ func findAppInfoKubernetes(image, name string, env []string) error {
 
 	for _, deployment := range deployments.Items {
 		if deployment.Name == name {
-			log.Printf("[INFO] Found deployment %s - no need to deploy another", name)
+			if debug { 
+				log.Printf("[DEBUG] Found deployment %s - no need to deploy another", name)
+			}
+
 			return nil
 		}
 	}
@@ -4025,7 +4037,7 @@ func getStreamResultsWrapper(client *http.Client, req *http.Request, workflowExe
 	if newresp.StatusCode != 200 {
 		log.Printf("[ERROR] StatusCode (1): %d - %s", newresp.StatusCode, string(body))
 		time.Sleep(time.Duration(sleepTime) * time.Second)
-		return environments, errors.New(fmt.Sprintf("Bad status code: %d", newresp.StatusCode))
+		return environments, errors.New(fmt.Sprintf("Bad status code from backend: %d", newresp.StatusCode))
 	}
 
 	err = json.Unmarshal(body, &workflowExecution)