From 47c32c25d626527d686b4d77f8ca51e95ce902ca Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 28 Aug 2025 21:50:39 +0200 Subject: [PATCH 01/13] Minor fix default setup fix --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 980e0764..cfd67680 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -65,7 +65,7 @@ services: hostname: shuffle-opensearch container_name: shuffle-opensearch environment: - - "OPENSEARCH_JAVA_OPTS=-Xms4096m -Xmx4096m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM + - "OPENSEARCH_JAVA_OPTS=-Xms3072m -Xmx3072m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM - bootstrap.memory_lock=true - DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI=true - cluster.initial_master_nodes=shuffle-opensearch From 9ed6351a9ab13c8f764417af9145073a0ce379fa Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 28 Aug 2025 21:52:19 +0200 Subject: [PATCH 02/13] Rebuild with 2.1.0 tag --- .github/workflows/dockerbuild.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dockerbuild.yaml b/.github/workflows/dockerbuild.yaml index 9eb946cf..3aa07ae6 100644 --- a/.github/workflows/dockerbuild.yaml +++ b/.github/workflows/dockerbuild.yaml @@ -20,19 +20,19 @@ jobs: include: - app: frontend path: frontend - version: 2.1.0-rc2 + version: 2.1.0 experimental: true - app: backend path: backend - version: 2.1.0-rc2 + version: 2.1.0 experimental: true - app: orborus path: functions/onprem/orborus - version: 2.1.0-rc2 + version: 2.1.0 experimental: true - app: worker path: functions/onprem/worker - version: 2.1.0-rc2 + version: 2.1.0 experimental: true steps: - name: Checkout From 744d8d6b7fd82e4e3e0db1966f6ec77118a737b6 Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 4 Sep 2025 00:43:09 +0200 Subject: [PATCH 03/13] Fixed the issue with app builds onprem not working --- .env | 3 +-- backend/Dockerfile | 4 ++-- backend/app_gen/python-lib/baseline/Dockerfile | 12 ++++++------ backend/go-app/go.mod | 2 +- backend/go-app/main.go | 13 +++++++++---- docker-compose.yml | 1 + functions/onprem/worker/worker.go | 8 +++++--- 7 files changed, 25 insertions(+), 18 deletions(-) mode change 100755 => 100644 backend/app_gen/python-lib/baseline/Dockerfile diff --git a/.env b/.env index 0b1e8cee..16a4f419 100755 --- a/.env +++ b/.env @@ -86,7 +86,7 @@ SHUFFLE_CONTAINER_AUTO_CLEANUP=true SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY=5 SHUFFLE_HEALTHCHECK_DISABLED=false SHUFFLE_ELASTIC=true -SHUFFLE_LOGS_DISABLED=false +SHUFFLE_LOGS_DISABLED=true SHUFFLE_CHAT_DISABLED=false SHUFFLE_DISABLE_RERUN_AND_ABORT=false SHUFFLE_RERUN_SCHEDULE=300 @@ -99,7 +99,6 @@ SHUFFLE_MAX_EXECUTION_DEPTH= # DATABASE CONFIGURATIONS DATASTORE_EMULATOR_HOST=shuffle-database:8000 -#SHUFFLE_OPENSEARCH_URL=http://shuffle-opensearch:9200 SHUFFLE_OPENSEARCH_URL=https://shuffle-opensearch:9200 SHUFFLE_OPENSEARCH_CERTIFICATE_FILE= SHUFFLE_OPENSEARCH_APIKEY= diff --git a/backend/Dockerfile b/backend/Dockerfile index 16ffeff0..647f74cf 100755 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -20,7 +20,7 @@ RUN go mod tidy # From November 2022, CGO is enabled due to packages # that we use requiring it. This is a temporary fix # and makes us HAVE to install libc compatibility packages farther down. -RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o webapp . +RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o shufflebackend . # Certificate build - gets required certs FROM alpine:latest as certs @@ -40,4 +40,4 @@ COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certifica WORKDIR /app EXPOSE 5001 -CMD ["./webapp"] +CMD ["./shufflebackend"] diff --git a/backend/app_gen/python-lib/baseline/Dockerfile b/backend/app_gen/python-lib/baseline/Dockerfile old mode 100755 new mode 100644 index 740fee62..d5d7e443 --- a/backend/app_gen/python-lib/baseline/Dockerfile +++ b/backend/app_gen/python-lib/baseline/Dockerfile @@ -1,26 +1,26 @@ -# Base our app image off of the WALKOFF App SDK image FROM frikky/shuffle:app_sdk as base # We're going to stage away all of the bloat from the build tools so lets create a builder stage FROM base as builder # Install all alpine build tools needed for our pip installs -RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev +RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev git # Install all of our pip packages in a single directory that we can copy to our base image later RUN mkdir /install WORKDIR /install COPY requirements.txt /requirements.txt -RUN pip install --prefix="/install" -r /requirements.txt +RUN pip install --no-cache-dir --upgrade --prefix="/install" -r /requirements.txt # Switch back to our base image and copy in all of our built packages and source code FROM base COPY --from=builder /install /usr/local COPY src /app -# Install any binary dependencies needed in our final image - this can be a lot of different stuff -RUN apk --no-cache add --update libmagic +# Install any binary dependencies needed in our final image +# RUN apk --no-cache add --update my_binary_dependency +RUN apk --no-cache add jq git curl # Finally, lets run our app! WORKDIR /app -CMD python app.py --log-level DEBUG +CMD ["python", "app.py", "--log-level", "DEBUG"] \ No newline at end of file diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index f6d34f38..966ebb91 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -24,7 +24,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.16 + github.com/shuffle/shuffle-shared v0.9.17 github.com/shuffle/singul v0.0.16 golang.org/x/crypto v0.40.0 google.golang.org/api v0.236.0 diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 79622f95..38752409 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3333,10 +3333,15 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User, s // Read and copy the baseline Dockerfile dockerfileContent, err := ioutil.ReadFile(dockerfileSource) if err != nil { - log.Printf("[ERROR] Failed to read baseline Dockerfile: %s", err) - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false, "reason": "Failed to read baseline Dockerfile"}`)) - return + foundDockerfile := shuffle.GetBaseDockerfile() + if len(foundDockerfile) > 0 { + dockerfileContent = foundDockerfile + } else { + log.Printf("[ERROR] Failed to read baseline Dockerfile: %s", err) + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false, "reason": "Failed to read baseline Dockerfile"}`)) + return + } } err = ioutil.WriteFile(dockerfileDestination, dockerfileContent, 0644) diff --git a/docker-compose.yml b/docker-compose.yml index cfd67680..ce3de3f6 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -55,6 +55,7 @@ services: - SHUFFLE_STATS_DISABLED=true - SHUFFLE_LOGS_DISABLED=true - SHUFFLE_SWARM_CONFIG=run + - CLEANUP=false - SHUFFLE_WORKER_IMAGE=ghcr.io/shuffle/shuffle-worker:latest env_file: .env restart: unless-stopped diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index c6b82823..2ecffa73 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -1302,8 +1302,7 @@ func getWorkerURLs() ([]string, error) { // } // } // } - - log.Printf("[DEBUG] Worker URLs for k8s: %#v", workerUrls) + //log.Printf("[DEBUG] Worker URLs for k8s: %#v", workerUrls) return workerUrls, nil } @@ -3615,7 +3614,10 @@ func findAppInfoKubernetes(image, name string, env []string) error { for _, deployment := range deployments.Items { if deployment.Name == name { - log.Printf("[INFO] Found deployment %s - no need to deploy another", name) + if debug { + log.Printf("[DEBUG] Found deployment %s - no need to deploy another", name) + } + return nil } } From 9676317758a4e88eef94bbcbad4ee086e6d8ad52 Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 4 Sep 2025 13:21:51 +0200 Subject: [PATCH 04/13] Fixed the problem with nodes going haywire if they crash --- functions/onprem/worker/worker.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 2ecffa73..265173a5 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -3238,23 +3238,24 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl } // Max scale as well + nodeCount := uint64(1) if inputReplicas > 0 && inputReplicas < 100 { if replicas != uint64(inputReplicas) { log.Printf("[DEBUG] Overwriting replicas to %d/node as inputReplicas is set to %d", inputReplicas, inputReplicas) } replicas = uint64(inputReplicas) + } else { + cnt, err := findActiveSwarmNodes(dockercli) + if err != nil { + log.Printf("[ERROR] Unable to find active swarm nodes: %s", err) + } + + if cnt > 0 { + nodeCount = uint64(cnt) + } } - cnt, err := findActiveSwarmNodes(dockercli) - if err != nil { - log.Printf("[ERROR] Unable to find active swarm nodes: %s", err) - } - - nodeCount := uint64(1) - if cnt > 0 { - nodeCount = uint64(cnt) - } replicatedJobs := uint64(replicas * nodeCount) log.Printf("[DEBUG] Deploying app with name %s with image %s", name, image) From 61d30d9fb289715739ad62aec7fd2b4b91da3a49 Mon Sep 17 00:00:00 2001 From: Frikky Date: Tue, 9 Sep 2025 16:00:53 +0200 Subject: [PATCH 05/13] Minor print fixes to orborus/worker --- docker-compose.yml | 8 ++++++-- functions/onprem/orborus/orborus.go | 4 ++-- functions/onprem/worker/worker.go | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index ce3de3f6..c1c27efb 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -95,12 +95,16 @@ services: # image: memcached:latest # container_name: shuffle-cache # hostname: shuffle-cache - # mem_limit: 1024m + # command: memcached -m 2048 -c 2500 # environment: - # - MEMCACHED_MEMORY=1024 + # - MEMCACHED_MEMORY=2048 # - MEMCACHED_MAX_CONNECTIONS=2500 # ports: # - 11211:11211 + # deploy: + # resources: + # limits: + # memory: 2048M #docker-socket-proxy: # image: tecnativa/docker-socket-proxy diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index a6435b65..d15ae33a 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -195,10 +195,10 @@ func getThisContainerId() { if containerId == "" { if containerName != "" { containerId = containerName - log.Printf("[INFO] Falling back to CONTAINER_NAME as container ID") + log.Printf("[INFO] Falling back to ORBORUS_CONTAINER_NAME as container ID") } else { containerId = "shuffle-orborus" - log.Printf(`[WARNING] CONTAINER_NAME is not set. Falling back to default name "%s" as container ID`, containerId) + log.Printf(`[WARNING] ORBORUS_CONTAINER_NAME env is not set. Falling back to default name "%s" as container ID. This may cause issues on the same server`, containerId) } } diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 265173a5..b598c89f 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -4028,7 +4028,7 @@ func getStreamResultsWrapper(client *http.Client, req *http.Request, workflowExe if newresp.StatusCode != 200 { log.Printf("[ERROR] StatusCode (1): %d - %s", newresp.StatusCode, string(body)) time.Sleep(time.Duration(sleepTime) * time.Second) - return environments, errors.New(fmt.Sprintf("Bad status code: %d", newresp.StatusCode)) + return environments, errors.New(fmt.Sprintf("Bad status code from backend: %d", newresp.StatusCode)) } err = json.Unmarshal(body, &workflowExecution) From 85fd0e1abc20ea750226824d6c1779bd0b74b101 Mon Sep 17 00:00:00 2001 From: Frikky Date: Wed, 10 Sep 2025 01:41:33 +0200 Subject: [PATCH 06/13] Made swarm init more robust --- functions/onprem/orborus/orborus.go | 42 +++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index d15ae33a..beae4a3a 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -12,6 +12,7 @@ import ( "fmt" "io" "io/ioutil" + "regexp" "log" "math" "net" @@ -342,7 +343,10 @@ func deployServiceWorkers(image string) { if len(dockerSwarmBridgeMTU) == 0 { mtu, err = strconv.Atoi(dockerSwarmBridgeMTU) // by default if err != nil { - log.Printf("[DEBUG] Failed to convert the default MTU to int: %s. Using 1500 instead. Input: %s", err, dockerSwarmBridgeMTU) + if debug { + log.Printf("[DEBUG] Failed to convert the default MTU to int: %s. Using 1500 instead. Input: %s", err, dockerSwarmBridgeMTU) + } + mtu = 1500 } } @@ -1700,17 +1704,45 @@ func checkSwarmService(ctx context.Context) { return } + listenAddr := "0.0.0.0" req := swarm.InitRequest{ - ListenAddr: "0.0.0.0:2377", + ListenAddr: fmt.Sprintf("%s:2377", listenAddr), AdvertiseAddr: fmt.Sprintf("%s:2377", ip), } - ret, err := dockercli.SwarmInit(ctx, req) + id, err := dockercli.SwarmInit(ctx, req) if err != nil { - log.Printf("[WARNING] Swarm init: %s", err) + log.Printf("[ERROR] Swarm init issue: %s. Retrying with a failover IP address from interface.", err) + + // Dummy message used for testing + //err = errors.New("Error response from daemon: could not choose an IP address to advertise since this system has multiple addresses on different interfaces (10.52.208.221 on eno1 and 192.168.122.1 on virbr0) - specify one with --advertise-addr") + + msg := err.Error() + + // Extract all IPv4 addresses from the error message + var ipv4Re = regexp.MustCompile(`\b(?:\d{1,3}\.){3}\d{1,3}\b`) + candidates := ipv4Re.FindAllString(msg, -1) + if len(candidates) > 0 { + // Pick the first valid candidate (or implement your own heuristic) + + for cnt, candidate := range candidates { + if cnt > 5 { + break + } + + req.AdvertiseAddr = fmt.Sprintf("%s:2377", candidate) + _, err = dockercli.SwarmInit(context.Background(), req) + if err != nil { + continue + } + + break + } + + } } - log.Printf("[DEBUG] Swarm info: %s\n\n", ret) + log.Printf("[INFO] Swarm init ID: '%s'. If this is empty, there is most likely an error.", id) } func getContainerResourceUsage(ctx context.Context, cli *dockerclient.Client, containerID string) (float64, float64, error) { From 26d10c788c6dc609853edf26b98e2d38fd6422ef Mon Sep 17 00:00:00 2001 From: Frikky Date: Wed, 10 Sep 2025 01:46:19 +0200 Subject: [PATCH 07/13] Disabled the Node Scale mechanism, as it was just confusing. SHUFFLE_APP_REPLICAS and SHUFFLE_SCALE_REPLICAS are self-controlled. --- functions/onprem/orborus/orborus.go | 8 ++++++-- functions/onprem/worker/worker.go | 3 +++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index beae4a3a..51ab7a69 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -220,8 +220,8 @@ func skipCheckInCleanup(name string) bool { } func cleanupExistingNodes(ctx context.Context) error { - if cleanupEnv == "false" { - log.Printf("[INFO] Skipping cleanup of existing workers as CLEANUP is set to false. This should be auto-discovered during executions then instead.") + if cleanupEnv != "true" { + log.Printf("[INFO] Skipping cleanup of existing workers as CLEANUP is NOT set to true. Swarm actions are being auto-discovered during executions then instead.") return nil } @@ -530,6 +530,7 @@ func deployServiceWorkers(image string) { nodeCount = uint64(cnt) } + appReplicas := os.Getenv("SHUFFLE_APP_REPLICAS") appReplicaCnt := 2 if len(appReplicas) > 0 { @@ -542,6 +543,9 @@ func deployServiceWorkers(image string) { } log.Printf("[DEBUG] Found %d node(s) to replicate over. Defaulting to 1 IF we can't auto-discover them.", cnt) + + // FIXME: From September 2025 - This is set back to 1, as this doesn't really reflect how scale works at all. It is just confusing, and makes number larger/smaller "arbitrarily" instead of using default docker scale + nodeCount = 1 replicatedJobs := uint64(replicas * nodeCount) log.Printf("[DEBUG] Deploying %d container(s) for worker with swarm to each node. Service name: %s. Image: %s", replicas, innerContainerName, image) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index b598c89f..5bd0e533 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -3254,6 +3254,9 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl if cnt > 0 { nodeCount = uint64(cnt) } + + // FIXME: From September 2025 - This is set back to 1, as this doesn't really reflect how scale works at all. It is just confusing, and makes number larger/smaller "arbitrarily" instead of using default docker scale + nodeCount = 1 } From 954a858933e5128aac40aff29ea79321853752bb Mon Sep 17 00:00:00 2001 From: Frikky Date: Wed, 10 Sep 2025 01:47:43 +0200 Subject: [PATCH 08/13] Made default app replicas 3 instead of 1. --- functions/onprem/worker/worker.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 5bd0e533..fc54c0a1 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -3218,10 +3218,8 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl } // Apps used a lot should have 2 replicas (default) - replicas := uint64(1) - //if (strings.Contains(strings.ToLower(name), "shuffle") && strings.Contains(strings.ToLower(name), "tools")) || strings.Contains(strings.ToLower(name), "http") { - // replicas = 2 - //} + // New default to 3 (as the chance of queues piling up is lower) + replicas := uint64(3) // Sent from Orborus // Should be equal to From a046c3a96bfbc9d3001734c16af2cc8aad4dc066 Mon Sep 17 00:00:00 2001 From: Frikky Date: Wed, 10 Sep 2025 01:50:10 +0200 Subject: [PATCH 09/13] Made service cleanup REQUIRE CLEANUP=true. This is opposite, now that we have workers that can self-discover services. --- functions/onprem/orborus/orborus.go | 1 + 1 file changed, 1 insertion(+) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 51ab7a69..a7395107 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -225,6 +225,7 @@ func cleanupExistingNodes(ctx context.Context) error { return nil } + if isKubernetes == "true" { // Cleanup all workers created by orborus and all apps created by workers. From fc9c01abbb5d688f30da4457bbabb5ef9cceafae Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 12 Sep 2025 08:16:18 +0200 Subject: [PATCH 10/13] Fixed opensearch indexing issue in bulk --- backend/go-app/go.mod | 2 +- backend/go-app/go.sum | 4 ++-- functions/onprem/orborus/orborus.go | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 966ebb91..f235486a 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -24,7 +24,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.17 + github.com/shuffle/shuffle-shared v0.9.21 github.com/shuffle/singul v0.0.16 golang.org/x/crypto v0.40.0 google.golang.org/api v0.236.0 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 0573dfdd..dc8366b0 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -363,8 +363,8 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.9.16 h1:B3Dp3h2f62JdWmqQ1kTu7DwJY6HCOMrxxSojAeuhI9k= -github.com/shuffle/shuffle-shared v0.9.16/go.mod h1:PhDEizuz4SmJaSmy0+yrFWwD1mXVUsy8/knKlrqF1qw= +github.com/shuffle/shuffle-shared v0.9.21 h1:ZxA/g0Lk93miHJpVYAhtTQmtWLJWcCtm6kFNF772Ynk= +github.com/shuffle/shuffle-shared v0.9.21/go.mod h1:PhDEizuz4SmJaSmy0+yrFWwD1mXVUsy8/knKlrqF1qw= github.com/shuffle/singul v0.0.16 h1:dW+0Mln9R1aUJ0fjikpWcxbjoQWqJHxe4kSxh2tQN5E= github.com/shuffle/singul v0.0.16/go.mod h1:LYkp320A6gsoPlYbXUM+WvEPUVAuutlSsqnVKyRy4gs= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index a7395107..51ab7a69 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -225,7 +225,6 @@ func cleanupExistingNodes(ctx context.Context) error { return nil } - if isKubernetes == "true" { // Cleanup all workers created by orborus and all apps created by workers. From 5da98c1d0cace5367d6ca02b4f5744038c686b9e Mon Sep 17 00:00:00 2001 From: Frikky Date: Tue, 16 Sep 2025 16:00:47 +0200 Subject: [PATCH 11/13] Re-added SHUFFLE_BASE_IMAGE_NAME usage, and made REGISTRY+BASE IMAGE setting ensure that PullAlways is used to always use latest local image --- functions/onprem/orborus/orborus.go | 23 +++++++++++++++++------ functions/onprem/worker/worker.go | 8 ++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 51ab7a69..ab9b2230 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -1058,6 +1058,13 @@ func deployK8sWorker(image string, identifier string, env []string) error { env = append(env, fmt.Sprintf("SHUFFLE_BASE_IMAGE_REGISTRY=%s", os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY"))) } + if len(os.Getenv("SHUFFLE_BASE_IMAGE_NAME")) > 0 { + env = append(env, fmt.Sprintf("SHUFFLE_BASE_IMAGE_NAME=%s", os.Getenv("SHUFFLE_BASE_IMAGE_NAME"))) + } else { + log.Printf("[INFO] SHUFFLE_BASE_IMAGE_NAME is not set. Defaulting to %s", baseimagename) + env = append(env, fmt.Sprintf("SHUFFLE_BASE_IMAGE_NAME=%s", baseimagename)) + } + if len(os.Getenv("REGISTRY_URL")) > 0 { env = append(env, fmt.Sprintf("REGISTRY_URL=%s", os.Getenv("REGISTRY_URL"))) } @@ -1204,10 +1211,13 @@ func deployK8sWorker(image string, identifier string, env []string) error { ImagePullPolicy: corev1.PullIfNotPresent, } + if len(os.Getenv("REGISTRY_URL")) > 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_NAME")) > 0 { + log.Printf("[INFO] Setting image pull policy to Always as private registry is used.") + containerAttachment.ImagePullPolicy = corev1.PullAlways + } + podname := shuffle.GetPodName() - ctx := context.Background() - if len(podname) > 0 { _, err := shuffle.GetCurrentPodNetworkConfig(ctx, clientset, kubernetesNamespace, podname) if err != nil { @@ -3974,10 +3984,11 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest, image string, identifier := "shuffle-workers" if isKubernetes == "true" { - if shuffle.IsRunningInCluster() { - log.Printf("[INFO] Running in Kubernetes cluster") - // try getting the k8s worker server url - } + // FIXME: Do we need this to map the cluster? + //if shuffle.IsRunningInCluster() { + //log.Printf("[INFO] Running in Kubernetes cluster") + // try getting the k8s worker server url + //} } if strings.Contains(streamUrl, "shuffler.io") || strings.Contains(streamUrl, "localhost") || strings.Contains(streamUrl, "127.0.0.1") || strings.Contains(streamUrl, "shuffle-backend") { diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index fc54c0a1..6dd41de4 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -662,6 +662,14 @@ func deployk8sApp(image string, identifier string, env []string) error { }, } + if len(os.Getenv("REGISTRY_URL")) > 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_NAME")) > 0 { + log.Printf("[INFO] Setting image pull policy to Always as private registry is used.") + //containerAttachment.ImagePullPolicy = corev1.PullAlways + deployment.Spec.Template.Spec.Containers[0].ImagePullPolicy = corev1.PullAlways + } else { + deployment.Spec.Template.Spec.Containers[0].ImagePullPolicy = corev1.PullIfNotPresent + } + _, err = clientset.AppsV1().Deployments(kubernetesNamespace).Create(context.Background(), deployment, metav1.CreateOptions{}) if err != nil { log.Printf("[ERROR] Failed creating deployment: %v", err) From 0e4ee87d76791aa135414f7ccfee822d7c4c2f09 Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 19 Sep 2025 15:33:30 +0200 Subject: [PATCH 12/13] Changed all the code to point to latest instead of nightly for k8s --- functions/kubernetes/all-in-one.yaml | 14 +++++++------- functions/kubernetes/charts/shuffle/Chart.yaml | 2 +- functions/kubernetes/orborus.yaml | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/functions/kubernetes/all-in-one.yaml b/functions/kubernetes/all-in-one.yaml index 190a1a29..d9bec68d 100644 --- a/functions/kubernetes/all-in-one.yaml +++ b/functions/kubernetes/all-in-one.yaml @@ -90,7 +90,7 @@ data: IS_KUBERNETES: "true" REGISTRY_URL: "docker-registry:5000" REGISTRY_AUTH: "false" - SHUFFLE_KUBERNETES_WORKER: "ghcr.io/shuffle/shuffle-worker:nightly" + SHUFFLE_KUBERNETES_WORKER: "ghcr.io/shuffle/shuffle-worker:latest" kind: ConfigMap --- @@ -702,7 +702,7 @@ spec: configMapKeyRef: key: REGISTRY_AUTH name: env - image: ghcr.io/shuffle/shuffle-backend:nightly + image: ghcr.io/shuffle/shuffle-backend:latest imagePullPolicy: Always name: shuffle-backend ports: @@ -769,7 +769,7 @@ spec: spec: containers: - name: shuffle-frontend - image: ghcr.io/shuffle/shuffle-frontend:nightly + image: ghcr.io/shuffle/shuffle-frontend:latest env: - name: BACKEND_HOSTNAME valueFrom: @@ -853,13 +853,13 @@ spec: - name: ORG_ID value: Shuffle - name: SHUFFLE_APP_SDK_VERSION - value: nightly + value: latest - name: SHUFFLE_SCALE_REPLICAS value: "5" - name: SHUFFLE_SWARM_CONFIG value: run - name: SHUFFLE_WORKER_VERSION - value: nightly + value: latest - name: IS_KUBERNETES valueFrom: configMapKeyRef: @@ -885,10 +885,10 @@ spec: configMapKeyRef: key: SHUFFLE_MEMCACHED name: env - image: ghcr.io/shuffle/shuffle-orborus:nightly + image: ghcr.io/shuffle/shuffle-orborus:latest #imagePullPolicy: Never name: shuffle-orborus resources: {} hostname: shuffle-orborus restartPolicy: Always -status: {} \ No newline at end of file +status: {} diff --git a/functions/kubernetes/charts/shuffle/Chart.yaml b/functions/kubernetes/charts/shuffle/Chart.yaml index 0a4e4fa3..b01df7da 100644 --- a/functions/kubernetes/charts/shuffle/Chart.yaml +++ b/functions/kubernetes/charts/shuffle/Chart.yaml @@ -3,7 +3,7 @@ name: shuffle description: A Helm chart for deploying Shuffle on Kubernetes type: application version: 0.0.0 # Set during publishing in GitHub actions -appVersion: nightly # Overwritten during publishing in GitHub actions +appVersion: latest # Overwritten during publishing in GitHub actions dependencies: - name: common version: ^2.23.0 diff --git a/functions/kubernetes/orborus.yaml b/functions/kubernetes/orborus.yaml index a94622ba..000d9fbd 100644 --- a/functions/kubernetes/orborus.yaml +++ b/functions/kubernetes/orborus.yaml @@ -72,7 +72,7 @@ spec: - name: AUTH value: "3663a270-bb3a-4678-a365-d879601a1a0c" - image: ghcr.io/shuffle/shuffle-orborus:nightly + image: ghcr.io/shuffle/shuffle-orborus:latest #imagePullPolicy: Never name: shuffle-orborus resources: {} From 15b1a8290215459adec8204d42eb05902d56d868 Mon Sep 17 00:00:00 2001 From: Yash Singh Date: Wed, 24 Sep 2025 14:13:07 +0530 Subject: [PATCH 13/13] chore: updated shuffle-shared --- backend/go-app/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index f235486a..4896209c 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -24,7 +24,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.21 + github.com/shuffle/shuffle-shared v0.9.25 github.com/shuffle/singul v0.0.16 golang.org/x/crypto v0.40.0 google.golang.org/api v0.236.0