marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1820 from Shuffle/main

Browse Source 
Syncing new fixes in main back to nightly
This commit is contained in:
Frikky
2025-09-28 18:09:05 +02:00
committed by GitHub
parent 962345af93 15b1a82902
commit b039222117
13 changed files with 133 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
functions/kubernetes/all-in-one.yaml
+7 -7
View File
@@ -90,7 +90,7 @@ data:
IS_KUBERNETES: "true"
REGISTRY_URL: "docker-registry:5000"
REGISTRY_AUTH: "false"
SHUFFLE_KUBERNETES_WORKER: "ghcr.io/shuffle/shuffle-worker:nightly"
SHUFFLE_KUBERNETES_WORKER: "ghcr.io/shuffle/shuffle-worker:latest"
kind: ConfigMap
---
@@ -702,7 +702,7 @@ spec:
configMapKeyRef:
key: REGISTRY_AUTH
name: env
image: ghcr.io/shuffle/shuffle-backend:nightly
image: ghcr.io/shuffle/shuffle-backend:latest
imagePullPolicy: Always
name: shuffle-backend
ports:
@@ -769,7 +769,7 @@ spec:
spec:
containers:
- name: shuffle-frontend
image: ghcr.io/shuffle/shuffle-frontend:nightly
image: ghcr.io/shuffle/shuffle-frontend:latest
env:
- name: BACKEND_HOSTNAME
valueFrom:
@@ -853,13 +853,13 @@ spec:
- name: ORG_ID
value: Shuffle
- name: SHUFFLE_APP_SDK_VERSION
value: nightly
value: latest
- name: SHUFFLE_SCALE_REPLICAS
value: "5"
- name: SHUFFLE_SWARM_CONFIG
value: run
- name: SHUFFLE_WORKER_VERSION
value: nightly
value: latest
- name: IS_KUBERNETES
valueFrom:
configMapKeyRef:
@@ -885,10 +885,10 @@ spec:
configMapKeyRef:
key: SHUFFLE_MEMCACHED
name: env
image: ghcr.io/shuffle/shuffle-orborus:nightly
image: ghcr.io/shuffle/shuffle-orborus:latest
#imagePullPolicy: Never
name: shuffle-orborus
resources: {}
hostname: shuffle-orborus
restartPolicy: Always
status: {}
status: {}
functions/kubernetes/charts/shuffle/Chart.yaml
+1 -1
View File
@@ -3,7 +3,7 @@ name: shuffle
description: A Helm chart for deploying Shuffle on Kubernetes
type: application
version: 0.0.0 # Set during publishing in GitHub actions
appVersion: nightly # Overwritten during publishing in GitHub actions
appVersion: latest # Overwritten during publishing in GitHub actions
dependencies:
- name: common
version: ^2.23.0
functions/kubernetes/orborus.yaml
+1 -1
View File
@@ -72,7 +72,7 @@ spec:
- name: AUTH
value: "3663a270-bb3a-4678-a365-d879601a1a0c"
image: ghcr.io/shuffle/shuffle-orborus:nightly
image: ghcr.io/shuffle/shuffle-orborus:latest
#imagePullPolicy: Never
name: shuffle-orborus
resources: {}
functions/onprem/orborus/orborus.go
+62 -15
View File
@@ -12,6 +12,7 @@ import (
"fmt"
"io"
"io/ioutil"
"regexp"
"log"
"math"
"net"
@@ -195,10 +196,10 @@ func getThisContainerId() {
if containerId == "" {
if containerName != "" {
containerId = containerName
log.Printf("[INFO] Falling back to CONTAINER_NAME as container ID")
log.Printf("[INFO] Falling back to ORBORUS_CONTAINER_NAME as container ID")
} else {
containerId = "shuffle-orborus"
log.Printf(`[WARNING] CONTAINER_NAME is not set. Falling back to default name "%s" as container ID`, containerId)
log.Printf(`[WARNING] ORBORUS_CONTAINER_NAME env is not set. Falling back to default name "%s" as container ID. This may cause issues on the same server`, containerId)
}
}
@@ -219,8 +220,8 @@ func skipCheckInCleanup(name string) bool {
}
func cleanupExistingNodes(ctx context.Context) error {
if cleanupEnv == "false" {
log.Printf("[INFO] Skipping cleanup of existing workers as CLEANUP is set to false. This should be auto-discovered during executions then instead.")
if cleanupEnv != "true" {
log.Printf("[INFO] Skipping cleanup of existing workers as CLEANUP is NOT set to true. Swarm actions are being auto-discovered during executions then instead.")
return nil
}
@@ -342,7 +343,10 @@ func deployServiceWorkers(image string) {
if len(dockerSwarmBridgeMTU) == 0 {
mtu, err = strconv.Atoi(dockerSwarmBridgeMTU) // by default
if err != nil {
log.Printf("[DEBUG] Failed to convert the default MTU to int: %s. Using 1500 instead. Input: %s", err, dockerSwarmBridgeMTU)
if debug {
log.Printf("[DEBUG] Failed to convert the default MTU to int: %s. Using 1500 instead. Input: %s", err, dockerSwarmBridgeMTU)
}
mtu = 1500
}
}
@@ -526,6 +530,7 @@ func deployServiceWorkers(image string) {
nodeCount = uint64(cnt)
}
appReplicas := os.Getenv("SHUFFLE_APP_REPLICAS")
appReplicaCnt := 2
if len(appReplicas) > 0 {
@@ -538,6 +543,9 @@ func deployServiceWorkers(image string) {
}
log.Printf("[DEBUG] Found %d node(s) to replicate over. Defaulting to 1 IF we can't auto-discover them.", cnt)
// FIXME: From September 2025 - This is set back to 1, as this doesn't really reflect how scale works at all. It is just confusing, and makes number larger/smaller "arbitrarily" instead of using default docker scale
nodeCount = 1
replicatedJobs := uint64(replicas * nodeCount)
log.Printf("[DEBUG] Deploying %d container(s) for worker with swarm to each node. Service name: %s. Image: %s", replicas, innerContainerName, image)
@@ -1050,6 +1058,13 @@ func deployK8sWorker(image string, identifier string, env []string) error {
env = append(env, fmt.Sprintf("SHUFFLE_BASE_IMAGE_REGISTRY=%s", os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")))
}
if len(os.Getenv("SHUFFLE_BASE_IMAGE_NAME")) > 0 {
env = append(env, fmt.Sprintf("SHUFFLE_BASE_IMAGE_NAME=%s", os.Getenv("SHUFFLE_BASE_IMAGE_NAME")))
} else {
log.Printf("[INFO] SHUFFLE_BASE_IMAGE_NAME is not set. Defaulting to %s", baseimagename)
env = append(env, fmt.Sprintf("SHUFFLE_BASE_IMAGE_NAME=%s", baseimagename))
}
if len(os.Getenv("REGISTRY_URL")) > 0 {
env = append(env, fmt.Sprintf("REGISTRY_URL=%s", os.Getenv("REGISTRY_URL")))
}
@@ -1196,10 +1211,13 @@ func deployK8sWorker(image string, identifier string, env []string) error {
ImagePullPolicy: corev1.PullIfNotPresent,
}
if len(os.Getenv("REGISTRY_URL")) > 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_NAME")) > 0 {
log.Printf("[INFO] Setting image pull policy to Always as private registry is used.")
containerAttachment.ImagePullPolicy = corev1.PullAlways
}
podname := shuffle.GetPodName()
ctx := context.Background()
if len(podname) > 0 {
_, err := shuffle.GetCurrentPodNetworkConfig(ctx, clientset, kubernetesNamespace, podname)
if err != nil {
@@ -1700,17 +1718,45 @@ func checkSwarmService(ctx context.Context) {
return
}
listenAddr := "0.0.0.0"
req := swarm.InitRequest{
ListenAddr: "0.0.0.0:2377",
ListenAddr: fmt.Sprintf("%s:2377", listenAddr),
AdvertiseAddr: fmt.Sprintf("%s:2377", ip),
}
ret, err := dockercli.SwarmInit(ctx, req)
id, err := dockercli.SwarmInit(ctx, req)
if err != nil {
log.Printf("[WARNING] Swarm init: %s", err)
log.Printf("[ERROR] Swarm init issue: %s. Retrying with a failover IP address from interface.", err)
// Dummy message used for testing
//err = errors.New("Error response from daemon: could not choose an IP address to advertise since this system has multiple addresses on different interfaces (10.52.208.221 on eno1 and 192.168.122.1 on virbr0) - specify one with --advertise-addr")
msg := err.Error()
// Extract all IPv4 addresses from the error message
var ipv4Re = regexp.MustCompile(`\b(?:\d{1,3}\.){3}\d{1,3}\b`)
candidates := ipv4Re.FindAllString(msg, -1)
if len(candidates) > 0 {
// Pick the first valid candidate (or implement your own heuristic)
for cnt, candidate := range candidates {
if cnt > 5 {
break
}
req.AdvertiseAddr = fmt.Sprintf("%s:2377", candidate)
_, err = dockercli.SwarmInit(context.Background(), req)
if err != nil {
continue
}
break
}
}
}
log.Printf("[DEBUG] Swarm info: %s\n\n", ret)
log.Printf("[INFO] Swarm init ID: '%s'. If this is empty, there is most likely an error.", id)
}
func getContainerResourceUsage(ctx context.Context, cli *dockerclient.Client, containerID string) (float64, float64, error) {
@@ -3939,10 +3985,11 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest, image string,
identifier := "shuffle-workers"
if isKubernetes == "true" {
if shuffle.IsRunningInCluster() {
log.Printf("[INFO] Running in Kubernetes cluster")
// try getting the k8s worker server url
}
// FIXME: Do we need this to map the cluster?
//if shuffle.IsRunningInCluster() {
//log.Printf("[INFO] Running in Kubernetes cluster")
// try getting the k8s worker server url
//}
}
if strings.Contains(streamUrl, "shuffler.io") || strings.Contains(streamUrl, "localhost") || strings.Contains(streamUrl, "127.0.0.1") || strings.Contains(streamUrl, "shuffle-backend") {
functions/onprem/worker/worker.go
+29 -17
View File
@@ -662,6 +662,14 @@ func deployk8sApp(image string, identifier string, env []string) error {
},
}
if len(os.Getenv("REGISTRY_URL")) > 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_NAME")) > 0 {
log.Printf("[INFO] Setting image pull policy to Always as private registry is used.")
//containerAttachment.ImagePullPolicy = corev1.PullAlways
deployment.Spec.Template.Spec.Containers[0].ImagePullPolicy = corev1.PullAlways
} else {
deployment.Spec.Template.Spec.Containers[0].ImagePullPolicy = corev1.PullIfNotPresent
}
_, err = clientset.AppsV1().Deployments(kubernetesNamespace).Create(context.Background(), deployment, metav1.CreateOptions{})
if err != nil {
log.Printf("[ERROR] Failed creating deployment: %v", err)
@@ -1302,8 +1310,7 @@ func getWorkerURLs() ([]string, error) {
// }
// }
// }
log.Printf("[DEBUG] Worker URLs for k8s: %#v", workerUrls)
//log.Printf("[DEBUG] Worker URLs for k8s: %#v", workerUrls)
return workerUrls, nil
}
@@ -3219,10 +3226,8 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl
}
// Apps used a lot should have 2 replicas (default)
replicas := uint64(1)
//if (strings.Contains(strings.ToLower(name), "shuffle") && strings.Contains(strings.ToLower(name), "tools")) || strings.Contains(strings.ToLower(name), "http") {
// replicas = 2
//}
// New default to 3 (as the chance of queues piling up is lower)
replicas := uint64(3)
// Sent from Orborus
// Should be equal to
@@ -3239,23 +3244,27 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl
}
// Max scale as well
nodeCount := uint64(1)
if inputReplicas > 0 && inputReplicas < 100 {
if replicas != uint64(inputReplicas) {
log.Printf("[DEBUG] Overwriting replicas to %d/node as inputReplicas is set to %d", inputReplicas, inputReplicas)
}
replicas = uint64(inputReplicas)
} else {
cnt, err := findActiveSwarmNodes(dockercli)
if err != nil {
log.Printf("[ERROR] Unable to find active swarm nodes: %s", err)
}
if cnt > 0 {
nodeCount = uint64(cnt)
}
// FIXME: From September 2025 - This is set back to 1, as this doesn't really reflect how scale works at all. It is just confusing, and makes number larger/smaller "arbitrarily" instead of using default docker scale
nodeCount = 1
}
cnt, err := findActiveSwarmNodes(dockercli)
if err != nil {
log.Printf("[ERROR] Unable to find active swarm nodes: %s", err)
}
nodeCount := uint64(1)
if cnt > 0 {
nodeCount = uint64(cnt)
}
replicatedJobs := uint64(replicas * nodeCount)
log.Printf("[DEBUG] Deploying app with name %s with image %s", name, image)
@@ -3615,7 +3624,10 @@ func findAppInfoKubernetes(image, name string, env []string) error {
for _, deployment := range deployments.Items {
if deployment.Name == name {
log.Printf("[INFO] Found deployment %s - no need to deploy another", name)
if debug {
log.Printf("[DEBUG] Found deployment %s - no need to deploy another", name)
}
return nil
}
}
@@ -4025,7 +4037,7 @@ func getStreamResultsWrapper(client *http.Client, req *http.Request, workflowExe
if newresp.StatusCode != 200 {
log.Printf("[ERROR] StatusCode (1): %d - %s", newresp.StatusCode, string(body))
time.Sleep(time.Duration(sleepTime) * time.Second)
return environments, errors.New(fmt.Sprintf("Bad status code: %d", newresp.StatusCode))
return environments, errors.New(fmt.Sprintf("Bad status code from backend: %d", newresp.StatusCode))
}
err = json.Unmarshal(body, &workflowExecution)