marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1820 from Shuffle/main

Browse Source 
Syncing new fixes in main back to nightly
This commit is contained in:
Frikky
2025-09-28 18:09:05 +02:00
committed by GitHub
parent 962345af93 15b1a82902
commit b039222117
13 changed files with 133 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env
+1 -2
View File
@@ -86,7 +86,7 @@ SHUFFLE_CONTAINER_AUTO_CLEANUP=true
SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY=5
SHUFFLE_HEALTHCHECK_DISABLED=false
SHUFFLE_ELASTIC=true
SHUFFLE_LOGS_DISABLED=false
SHUFFLE_LOGS_DISABLED=true
SHUFFLE_CHAT_DISABLED=false
SHUFFLE_DISABLE_RERUN_AND_ABORT=false
SHUFFLE_RERUN_SCHEDULE=300
@@ -99,7 +99,6 @@ SHUFFLE_MAX_EXECUTION_DEPTH=
# DATABASE CONFIGURATIONS
DATASTORE_EMULATOR_HOST=shuffle-database:8000
#SHUFFLE_OPENSEARCH_URL=http://shuffle-opensearch:9200
SHUFFLE_OPENSEARCH_URL=https://shuffle-opensearch:9200
SHUFFLE_OPENSEARCH_CERTIFICATE_FILE=
SHUFFLE_OPENSEARCH_APIKEY=
.github/workflows/dockerbuild.yaml
+4 -4
View File
@@ -20,19 +20,19 @@ jobs:
include:
- app: frontend
path: frontend
version: 2.1.0-rc2
version: 2.1.0
experimental: true
- app: backend
path: backend
version: 2.1.0-rc2
version: 2.1.0
experimental: true
- app: orborus
path: functions/onprem/orborus
version: 2.1.0-rc2
version: 2.1.0
experimental: true
- app: worker
path: functions/onprem/worker
version: 2.1.0-rc2
version: 2.1.0
experimental: true
steps:
- name: Checkout
backend/Dockerfile
+2 -2
View File
@@ -20,7 +20,7 @@ RUN go mod tidy
# From November 2022, CGO is enabled due to packages
# that we use requiring it. This is a temporary fix
# and makes us HAVE to install libc compatibility packages farther down.
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o webapp .
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o shufflebackend .
# Certificate build - gets required certs
FROM alpine:latest as certs
@@ -40,4 +40,4 @@ COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certifica
WORKDIR /app
EXPOSE 5001
CMD ["./webapp"]
CMD ["./shufflebackend"]
backend/app_gen/python-lib/baseline/Dockerfile
Executable → Regular
+6 -6
View File
@@ -1,26 +1,26 @@
# Base our app image off of the WALKOFF App SDK image
FROM frikky/shuffle:app_sdk as base
# We're going to stage away all of the bloat from the build tools so lets create a builder stage
FROM base as builder
# Install all alpine build tools needed for our pip installs
RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev
RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev git
# Install all of our pip packages in a single directory that we can copy to our base image later
RUN mkdir /install
WORKDIR /install
COPY requirements.txt /requirements.txt
RUN pip install --prefix="/install" -r /requirements.txt
RUN pip install --no-cache-dir --upgrade --prefix="/install" -r /requirements.txt
# Switch back to our base image and copy in all of our built packages and source code
FROM base
COPY --from=builder /install /usr/local
COPY src /app
# Install any binary dependencies needed in our final image - this can be a lot of different stuff
RUN apk --no-cache add --update libmagic
# Install any binary dependencies needed in our final image
# RUN apk --no-cache add --update my_binary_dependency
RUN apk --no-cache add jq git curl
# Finally, lets run our app!
WORKDIR /app
CMD python app.py --log-level DEBUG
CMD ["python", "app.py", "--log-level", "DEBUG"]
backend/go-app/go.mod
+1 -1
View File
@@ -24,7 +24,7 @@ require (
github.com/gorilla/mux v1.8.1
github.com/h2non/filetype v1.1.3
github.com/satori/go.uuid v1.2.0
github.com/shuffle/shuffle-shared v0.9.16
github.com/shuffle/shuffle-shared v0.9.25
github.com/shuffle/singul v0.0.16
golang.org/x/crypto v0.40.0
google.golang.org/api v0.236.0
backend/go-app/go.sum
+2 -2
View File
@@ -363,8 +363,8 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G
github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8=
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8=
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
github.com/shuffle/shuffle-shared v0.9.16 h1:B3Dp3h2f62JdWmqQ1kTu7DwJY6HCOMrxxSojAeuhI9k=
github.com/shuffle/shuffle-shared v0.9.16/go.mod h1:PhDEizuz4SmJaSmy0+yrFWwD1mXVUsy8/knKlrqF1qw=
github.com/shuffle/shuffle-shared v0.9.21 h1:ZxA/g0Lk93miHJpVYAhtTQmtWLJWcCtm6kFNF772Ynk=
github.com/shuffle/shuffle-shared v0.9.21/go.mod h1:PhDEizuz4SmJaSmy0+yrFWwD1mXVUsy8/knKlrqF1qw=
github.com/shuffle/singul v0.0.16 h1:dW+0Mln9R1aUJ0fjikpWcxbjoQWqJHxe4kSxh2tQN5E=
github.com/shuffle/singul v0.0.16/go.mod h1:LYkp320A6gsoPlYbXUM+WvEPUVAuutlSsqnVKyRy4gs=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
backend/go-app/main.go
+9 -4
View File
@@ -3335,10 +3335,15 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User, s
// Read and copy the baseline Dockerfile
dockerfileContent, err := ioutil.ReadFile(dockerfileSource)
if err != nil {
log.Printf("[ERROR] Failed to read baseline Dockerfile: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed to read baseline Dockerfile"}`))
return
foundDockerfile := shuffle.GetBaseDockerfile()
if len(foundDockerfile) > 0 {
dockerfileContent = foundDockerfile
} else {
log.Printf("[ERROR] Failed to read baseline Dockerfile: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed to read baseline Dockerfile"}`))
return
}
}
err = ioutil.WriteFile(dockerfileDestination, dockerfileContent, 0644)
docker-compose.yml
+8 -3
View File
@@ -55,6 +55,7 @@ services:
- SHUFFLE_STATS_DISABLED=true
- SHUFFLE_LOGS_DISABLED=true
- SHUFFLE_SWARM_CONFIG=run
- CLEANUP=false
- SHUFFLE_WORKER_IMAGE=ghcr.io/shuffle/shuffle-worker:latest
env_file: .env
restart: unless-stopped
@@ -65,7 +66,7 @@ services:
hostname: shuffle-opensearch
container_name: shuffle-opensearch
environment:
- "OPENSEARCH_JAVA_OPTS=-Xms4096m -Xmx4096m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
- "OPENSEARCH_JAVA_OPTS=-Xms3072m -Xmx3072m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
- bootstrap.memory_lock=true
- DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI=true
- cluster.initial_master_nodes=shuffle-opensearch
@@ -94,12 +95,16 @@ services:
# image: memcached:latest
# container_name: shuffle-cache
# hostname: shuffle-cache
# mem_limit: 1024m
# command: memcached -m 2048 -c 2500
# environment:
# - MEMCACHED_MEMORY=1024
# - MEMCACHED_MEMORY=2048
# - MEMCACHED_MAX_CONNECTIONS=2500
# ports:
# - 11211:11211
# deploy:
# resources:
# limits:
# memory: 2048M
#docker-socket-proxy:
# image: tecnativa/docker-socket-proxy
functions/kubernetes/all-in-one.yaml
+7 -7
View File
@@ -90,7 +90,7 @@ data:
IS_KUBERNETES: "true"
REGISTRY_URL: "docker-registry:5000"
REGISTRY_AUTH: "false"
SHUFFLE_KUBERNETES_WORKER: "ghcr.io/shuffle/shuffle-worker:nightly"
SHUFFLE_KUBERNETES_WORKER: "ghcr.io/shuffle/shuffle-worker:latest"
kind: ConfigMap
---
@@ -702,7 +702,7 @@ spec:
configMapKeyRef:
key: REGISTRY_AUTH
name: env
image: ghcr.io/shuffle/shuffle-backend:nightly
image: ghcr.io/shuffle/shuffle-backend:latest
imagePullPolicy: Always
name: shuffle-backend
ports:
@@ -769,7 +769,7 @@ spec:
spec:
containers:
- name: shuffle-frontend
image: ghcr.io/shuffle/shuffle-frontend:nightly
image: ghcr.io/shuffle/shuffle-frontend:latest
env:
- name: BACKEND_HOSTNAME
valueFrom:
@@ -853,13 +853,13 @@ spec:
- name: ORG_ID
value: Shuffle
- name: SHUFFLE_APP_SDK_VERSION
value: nightly
value: latest
- name: SHUFFLE_SCALE_REPLICAS
value: "5"
- name: SHUFFLE_SWARM_CONFIG
value: run
- name: SHUFFLE_WORKER_VERSION
value: nightly
value: latest
- name: IS_KUBERNETES
valueFrom:
configMapKeyRef:
@@ -885,10 +885,10 @@ spec:
configMapKeyRef:
key: SHUFFLE_MEMCACHED
name: env
image: ghcr.io/shuffle/shuffle-orborus:nightly
image: ghcr.io/shuffle/shuffle-orborus:latest
#imagePullPolicy: Never
name: shuffle-orborus
resources: {}
hostname: shuffle-orborus
restartPolicy: Always
status: {}
status: {}
functions/kubernetes/charts/shuffle/Chart.yaml
+1 -1
View File
@@ -3,7 +3,7 @@ name: shuffle
description: A Helm chart for deploying Shuffle on Kubernetes
type: application
version: 0.0.0 # Set during publishing in GitHub actions
appVersion: nightly # Overwritten during publishing in GitHub actions
appVersion: latest # Overwritten during publishing in GitHub actions
dependencies:
- name: common
version: ^2.23.0
functions/kubernetes/orborus.yaml
+1 -1
View File
@@ -72,7 +72,7 @@ spec:
- name: AUTH
value: "3663a270-bb3a-4678-a365-d879601a1a0c"
image: ghcr.io/shuffle/shuffle-orborus:nightly
image: ghcr.io/shuffle/shuffle-orborus:latest
#imagePullPolicy: Never
name: shuffle-orborus
resources: {}
functions/onprem/orborus/orborus.go
+62 -15
View File
@@ -12,6 +12,7 @@ import (
"fmt"
"io"
"io/ioutil"
"regexp"
"log"
"math"
"net"
@@ -195,10 +196,10 @@ func getThisContainerId() {
if containerId == "" {
if containerName != "" {
containerId = containerName
log.Printf("[INFO] Falling back to CONTAINER_NAME as container ID")
log.Printf("[INFO] Falling back to ORBORUS_CONTAINER_NAME as container ID")
} else {
containerId = "shuffle-orborus"
log.Printf(`[WARNING] CONTAINER_NAME is not set. Falling back to default name "%s" as container ID`, containerId)
log.Printf(`[WARNING] ORBORUS_CONTAINER_NAME env is not set. Falling back to default name "%s" as container ID. This may cause issues on the same server`, containerId)
}
}
@@ -219,8 +220,8 @@ func skipCheckInCleanup(name string) bool {
}
func cleanupExistingNodes(ctx context.Context) error {
if cleanupEnv == "false" {
log.Printf("[INFO] Skipping cleanup of existing workers as CLEANUP is set to false. This should be auto-discovered during executions then instead.")
if cleanupEnv != "true" {
log.Printf("[INFO] Skipping cleanup of existing workers as CLEANUP is NOT set to true. Swarm actions are being auto-discovered during executions then instead.")
return nil
}
@@ -342,7 +343,10 @@ func deployServiceWorkers(image string) {
if len(dockerSwarmBridgeMTU) == 0 {
mtu, err = strconv.Atoi(dockerSwarmBridgeMTU) // by default
if err != nil {
log.Printf("[DEBUG] Failed to convert the default MTU to int: %s. Using 1500 instead. Input: %s", err, dockerSwarmBridgeMTU)
if debug {
log.Printf("[DEBUG] Failed to convert the default MTU to int: %s. Using 1500 instead. Input: %s", err, dockerSwarmBridgeMTU)
}
mtu = 1500
}
}
@@ -526,6 +530,7 @@ func deployServiceWorkers(image string) {
nodeCount = uint64(cnt)
}
appReplicas := os.Getenv("SHUFFLE_APP_REPLICAS")
appReplicaCnt := 2
if len(appReplicas) > 0 {
@@ -538,6 +543,9 @@ func deployServiceWorkers(image string) {
}
log.Printf("[DEBUG] Found %d node(s) to replicate over. Defaulting to 1 IF we can't auto-discover them.", cnt)
// FIXME: From September 2025 - This is set back to 1, as this doesn't really reflect how scale works at all. It is just confusing, and makes number larger/smaller "arbitrarily" instead of using default docker scale
nodeCount = 1
replicatedJobs := uint64(replicas * nodeCount)
log.Printf("[DEBUG] Deploying %d container(s) for worker with swarm to each node. Service name: %s. Image: %s", replicas, innerContainerName, image)
@@ -1050,6 +1058,13 @@ func deployK8sWorker(image string, identifier string, env []string) error {
env = append(env, fmt.Sprintf("SHUFFLE_BASE_IMAGE_REGISTRY=%s", os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")))
}
if len(os.Getenv("SHUFFLE_BASE_IMAGE_NAME")) > 0 {
env = append(env, fmt.Sprintf("SHUFFLE_BASE_IMAGE_NAME=%s", os.Getenv("SHUFFLE_BASE_IMAGE_NAME")))
} else {
log.Printf("[INFO] SHUFFLE_BASE_IMAGE_NAME is not set. Defaulting to %s", baseimagename)
env = append(env, fmt.Sprintf("SHUFFLE_BASE_IMAGE_NAME=%s", baseimagename))
}
if len(os.Getenv("REGISTRY_URL")) > 0 {
env = append(env, fmt.Sprintf("REGISTRY_URL=%s", os.Getenv("REGISTRY_URL")))
}
@@ -1196,10 +1211,13 @@ func deployK8sWorker(image string, identifier string, env []string) error {
ImagePullPolicy: corev1.PullIfNotPresent,
}
if len(os.Getenv("REGISTRY_URL")) > 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_NAME")) > 0 {
log.Printf("[INFO] Setting image pull policy to Always as private registry is used.")
containerAttachment.ImagePullPolicy = corev1.PullAlways
}
podname := shuffle.GetPodName()
ctx := context.Background()
if len(podname) > 0 {
_, err := shuffle.GetCurrentPodNetworkConfig(ctx, clientset, kubernetesNamespace, podname)
if err != nil {
@@ -1700,17 +1718,45 @@ func checkSwarmService(ctx context.Context) {
return
}
listenAddr := "0.0.0.0"
req := swarm.InitRequest{
ListenAddr: "0.0.0.0:2377",
ListenAddr: fmt.Sprintf("%s:2377", listenAddr),
AdvertiseAddr: fmt.Sprintf("%s:2377", ip),
}
ret, err := dockercli.SwarmInit(ctx, req)
id, err := dockercli.SwarmInit(ctx, req)
if err != nil {
log.Printf("[WARNING] Swarm init: %s", err)
log.Printf("[ERROR] Swarm init issue: %s. Retrying with a failover IP address from interface.", err)
// Dummy message used for testing
//err = errors.New("Error response from daemon: could not choose an IP address to advertise since this system has multiple addresses on different interfaces (10.52.208.221 on eno1 and 192.168.122.1 on virbr0) - specify one with --advertise-addr")
msg := err.Error()
// Extract all IPv4 addresses from the error message
var ipv4Re = regexp.MustCompile(`\b(?:\d{1,3}\.){3}\d{1,3}\b`)
candidates := ipv4Re.FindAllString(msg, -1)
if len(candidates) > 0 {
// Pick the first valid candidate (or implement your own heuristic)
for cnt, candidate := range candidates {
if cnt > 5 {
break
}
req.AdvertiseAddr = fmt.Sprintf("%s:2377", candidate)
_, err = dockercli.SwarmInit(context.Background(), req)
if err != nil {
continue
}
break
}
}
}
log.Printf("[DEBUG] Swarm info: %s\n\n", ret)
log.Printf("[INFO] Swarm init ID: '%s'. If this is empty, there is most likely an error.", id)
}
func getContainerResourceUsage(ctx context.Context, cli *dockerclient.Client, containerID string) (float64, float64, error) {
@@ -3939,10 +3985,11 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest, image string,
identifier := "shuffle-workers"
if isKubernetes == "true" {
if shuffle.IsRunningInCluster() {
log.Printf("[INFO] Running in Kubernetes cluster")
// try getting the k8s worker server url
}
// FIXME: Do we need this to map the cluster?
//if shuffle.IsRunningInCluster() {
//log.Printf("[INFO] Running in Kubernetes cluster")
// try getting the k8s worker server url
//}
}
if strings.Contains(streamUrl, "shuffler.io") || strings.Contains(streamUrl, "localhost") || strings.Contains(streamUrl, "127.0.0.1") || strings.Contains(streamUrl, "shuffle-backend") {
functions/onprem/worker/worker.go
+29 -17
View File
@@ -662,6 +662,14 @@ func deployk8sApp(image string, identifier string, env []string) error {
},
}
if len(os.Getenv("REGISTRY_URL")) > 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_NAME")) > 0 {
log.Printf("[INFO] Setting image pull policy to Always as private registry is used.")
//containerAttachment.ImagePullPolicy = corev1.PullAlways
deployment.Spec.Template.Spec.Containers[0].ImagePullPolicy = corev1.PullAlways
} else {
deployment.Spec.Template.Spec.Containers[0].ImagePullPolicy = corev1.PullIfNotPresent
}
_, err = clientset.AppsV1().Deployments(kubernetesNamespace).Create(context.Background(), deployment, metav1.CreateOptions{})
if err != nil {
log.Printf("[ERROR] Failed creating deployment: %v", err)
@@ -1302,8 +1310,7 @@ func getWorkerURLs() ([]string, error) {
// }
// }
// }
log.Printf("[DEBUG] Worker URLs for k8s: %#v", workerUrls)
//log.Printf("[DEBUG] Worker URLs for k8s: %#v", workerUrls)
return workerUrls, nil
}
@@ -3219,10 +3226,8 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl
}
// Apps used a lot should have 2 replicas (default)
replicas := uint64(1)
//if (strings.Contains(strings.ToLower(name), "shuffle") && strings.Contains(strings.ToLower(name), "tools")) || strings.Contains(strings.ToLower(name), "http") {
// replicas = 2
//}
// New default to 3 (as the chance of queues piling up is lower)
replicas := uint64(3)
// Sent from Orborus
// Should be equal to
@@ -3239,23 +3244,27 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl
}
// Max scale as well
nodeCount := uint64(1)
if inputReplicas > 0 && inputReplicas < 100 {
if replicas != uint64(inputReplicas) {
log.Printf("[DEBUG] Overwriting replicas to %d/node as inputReplicas is set to %d", inputReplicas, inputReplicas)
}
replicas = uint64(inputReplicas)
} else {
cnt, err := findActiveSwarmNodes(dockercli)
if err != nil {
log.Printf("[ERROR] Unable to find active swarm nodes: %s", err)
}
if cnt > 0 {
nodeCount = uint64(cnt)
}
// FIXME: From September 2025 - This is set back to 1, as this doesn't really reflect how scale works at all. It is just confusing, and makes number larger/smaller "arbitrarily" instead of using default docker scale
nodeCount = 1
}
cnt, err := findActiveSwarmNodes(dockercli)
if err != nil {
log.Printf("[ERROR] Unable to find active swarm nodes: %s", err)
}
nodeCount := uint64(1)
if cnt > 0 {
nodeCount = uint64(cnt)
}
replicatedJobs := uint64(replicas * nodeCount)
log.Printf("[DEBUG] Deploying app with name %s with image %s", name, image)
@@ -3615,7 +3624,10 @@ func findAppInfoKubernetes(image, name string, env []string) error {
for _, deployment := range deployments.Items {
if deployment.Name == name {
log.Printf("[INFO] Found deployment %s - no need to deploy another", name)
if debug {
log.Printf("[DEBUG] Found deployment %s - no need to deploy another", name)
}
return nil
}
}
@@ -4025,7 +4037,7 @@ func getStreamResultsWrapper(client *http.Client, req *http.Request, workflowExe
if newresp.StatusCode != 200 {
log.Printf("[ERROR] StatusCode (1): %d - %s", newresp.StatusCode, string(body))
time.Sleep(time.Duration(sleepTime) * time.Second)
return environments, errors.New(fmt.Sprintf("Bad status code: %d", newresp.StatusCode))
return environments, errors.New(fmt.Sprintf("Bad status code from backend: %d", newresp.StatusCode))
}
err = json.Unmarshal(body, &workflowExecution)