diff --git a/.env b/.env index 0b1e8cee..16a4f419 100755 --- a/.env +++ b/.env @@ -86,7 +86,7 @@ SHUFFLE_CONTAINER_AUTO_CLEANUP=true SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY=5 SHUFFLE_HEALTHCHECK_DISABLED=false SHUFFLE_ELASTIC=true -SHUFFLE_LOGS_DISABLED=false +SHUFFLE_LOGS_DISABLED=true SHUFFLE_CHAT_DISABLED=false SHUFFLE_DISABLE_RERUN_AND_ABORT=false SHUFFLE_RERUN_SCHEDULE=300 @@ -99,7 +99,6 @@ SHUFFLE_MAX_EXECUTION_DEPTH= # DATABASE CONFIGURATIONS DATASTORE_EMULATOR_HOST=shuffle-database:8000 -#SHUFFLE_OPENSEARCH_URL=http://shuffle-opensearch:9200 SHUFFLE_OPENSEARCH_URL=https://shuffle-opensearch:9200 SHUFFLE_OPENSEARCH_CERTIFICATE_FILE= SHUFFLE_OPENSEARCH_APIKEY= diff --git a/.github/workflows/dockerbuild.yaml b/.github/workflows/dockerbuild.yaml index 9eb946cf..3aa07ae6 100644 --- a/.github/workflows/dockerbuild.yaml +++ b/.github/workflows/dockerbuild.yaml @@ -20,19 +20,19 @@ jobs: include: - app: frontend path: frontend - version: 2.1.0-rc2 + version: 2.1.0 experimental: true - app: backend path: backend - version: 2.1.0-rc2 + version: 2.1.0 experimental: true - app: orborus path: functions/onprem/orborus - version: 2.1.0-rc2 + version: 2.1.0 experimental: true - app: worker path: functions/onprem/worker - version: 2.1.0-rc2 + version: 2.1.0 experimental: true steps: - name: Checkout diff --git a/backend/Dockerfile b/backend/Dockerfile index 16ffeff0..647f74cf 100755 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -20,7 +20,7 @@ RUN go mod tidy # From November 2022, CGO is enabled due to packages # that we use requiring it. This is a temporary fix # and makes us HAVE to install libc compatibility packages farther down. -RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o webapp . +RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o shufflebackend . # Certificate build - gets required certs FROM alpine:latest as certs @@ -40,4 +40,4 @@ COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certifica WORKDIR /app EXPOSE 5001 -CMD ["./webapp"] +CMD ["./shufflebackend"] diff --git a/backend/app_gen/python-lib/baseline/Dockerfile b/backend/app_gen/python-lib/baseline/Dockerfile old mode 100755 new mode 100644 index 740fee62..d5d7e443 --- a/backend/app_gen/python-lib/baseline/Dockerfile +++ b/backend/app_gen/python-lib/baseline/Dockerfile @@ -1,26 +1,26 @@ -# Base our app image off of the WALKOFF App SDK image FROM frikky/shuffle:app_sdk as base # We're going to stage away all of the bloat from the build tools so lets create a builder stage FROM base as builder # Install all alpine build tools needed for our pip installs -RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev +RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev git # Install all of our pip packages in a single directory that we can copy to our base image later RUN mkdir /install WORKDIR /install COPY requirements.txt /requirements.txt -RUN pip install --prefix="/install" -r /requirements.txt +RUN pip install --no-cache-dir --upgrade --prefix="/install" -r /requirements.txt # Switch back to our base image and copy in all of our built packages and source code FROM base COPY --from=builder /install /usr/local COPY src /app -# Install any binary dependencies needed in our final image - this can be a lot of different stuff -RUN apk --no-cache add --update libmagic +# Install any binary dependencies needed in our final image +# RUN apk --no-cache add --update my_binary_dependency +RUN apk --no-cache add jq git curl # Finally, lets run our app! WORKDIR /app -CMD python app.py --log-level DEBUG +CMD ["python", "app.py", "--log-level", "DEBUG"] \ No newline at end of file diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index f6d34f38..4896209c 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -24,7 +24,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.16 + github.com/shuffle/shuffle-shared v0.9.25 github.com/shuffle/singul v0.0.16 golang.org/x/crypto v0.40.0 google.golang.org/api v0.236.0 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 0573dfdd..dc8366b0 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -363,8 +363,8 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shuffle/shuffle-shared v0.9.16 h1:B3Dp3h2f62JdWmqQ1kTu7DwJY6HCOMrxxSojAeuhI9k= -github.com/shuffle/shuffle-shared v0.9.16/go.mod h1:PhDEizuz4SmJaSmy0+yrFWwD1mXVUsy8/knKlrqF1qw= +github.com/shuffle/shuffle-shared v0.9.21 h1:ZxA/g0Lk93miHJpVYAhtTQmtWLJWcCtm6kFNF772Ynk= +github.com/shuffle/shuffle-shared v0.9.21/go.mod h1:PhDEizuz4SmJaSmy0+yrFWwD1mXVUsy8/knKlrqF1qw= github.com/shuffle/singul v0.0.16 h1:dW+0Mln9R1aUJ0fjikpWcxbjoQWqJHxe4kSxh2tQN5E= github.com/shuffle/singul v0.0.16/go.mod h1:LYkp320A6gsoPlYbXUM+WvEPUVAuutlSsqnVKyRy4gs= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index d8616564..b10d441d 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3335,10 +3335,15 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User, s // Read and copy the baseline Dockerfile dockerfileContent, err := ioutil.ReadFile(dockerfileSource) if err != nil { - log.Printf("[ERROR] Failed to read baseline Dockerfile: %s", err) - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false, "reason": "Failed to read baseline Dockerfile"}`)) - return + foundDockerfile := shuffle.GetBaseDockerfile() + if len(foundDockerfile) > 0 { + dockerfileContent = foundDockerfile + } else { + log.Printf("[ERROR] Failed to read baseline Dockerfile: %s", err) + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false, "reason": "Failed to read baseline Dockerfile"}`)) + return + } } err = ioutil.WriteFile(dockerfileDestination, dockerfileContent, 0644) diff --git a/docker-compose.yml b/docker-compose.yml index 980e0764..c1c27efb 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -55,6 +55,7 @@ services: - SHUFFLE_STATS_DISABLED=true - SHUFFLE_LOGS_DISABLED=true - SHUFFLE_SWARM_CONFIG=run + - CLEANUP=false - SHUFFLE_WORKER_IMAGE=ghcr.io/shuffle/shuffle-worker:latest env_file: .env restart: unless-stopped @@ -65,7 +66,7 @@ services: hostname: shuffle-opensearch container_name: shuffle-opensearch environment: - - "OPENSEARCH_JAVA_OPTS=-Xms4096m -Xmx4096m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM + - "OPENSEARCH_JAVA_OPTS=-Xms3072m -Xmx3072m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM - bootstrap.memory_lock=true - DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI=true - cluster.initial_master_nodes=shuffle-opensearch @@ -94,12 +95,16 @@ services: # image: memcached:latest # container_name: shuffle-cache # hostname: shuffle-cache - # mem_limit: 1024m + # command: memcached -m 2048 -c 2500 # environment: - # - MEMCACHED_MEMORY=1024 + # - MEMCACHED_MEMORY=2048 # - MEMCACHED_MAX_CONNECTIONS=2500 # ports: # - 11211:11211 + # deploy: + # resources: + # limits: + # memory: 2048M #docker-socket-proxy: # image: tecnativa/docker-socket-proxy diff --git a/functions/kubernetes/all-in-one.yaml b/functions/kubernetes/all-in-one.yaml index 190a1a29..d9bec68d 100644 --- a/functions/kubernetes/all-in-one.yaml +++ b/functions/kubernetes/all-in-one.yaml @@ -90,7 +90,7 @@ data: IS_KUBERNETES: "true" REGISTRY_URL: "docker-registry:5000" REGISTRY_AUTH: "false" - SHUFFLE_KUBERNETES_WORKER: "ghcr.io/shuffle/shuffle-worker:nightly" + SHUFFLE_KUBERNETES_WORKER: "ghcr.io/shuffle/shuffle-worker:latest" kind: ConfigMap --- @@ -702,7 +702,7 @@ spec: configMapKeyRef: key: REGISTRY_AUTH name: env - image: ghcr.io/shuffle/shuffle-backend:nightly + image: ghcr.io/shuffle/shuffle-backend:latest imagePullPolicy: Always name: shuffle-backend ports: @@ -769,7 +769,7 @@ spec: spec: containers: - name: shuffle-frontend - image: ghcr.io/shuffle/shuffle-frontend:nightly + image: ghcr.io/shuffle/shuffle-frontend:latest env: - name: BACKEND_HOSTNAME valueFrom: @@ -853,13 +853,13 @@ spec: - name: ORG_ID value: Shuffle - name: SHUFFLE_APP_SDK_VERSION - value: nightly + value: latest - name: SHUFFLE_SCALE_REPLICAS value: "5" - name: SHUFFLE_SWARM_CONFIG value: run - name: SHUFFLE_WORKER_VERSION - value: nightly + value: latest - name: IS_KUBERNETES valueFrom: configMapKeyRef: @@ -885,10 +885,10 @@ spec: configMapKeyRef: key: SHUFFLE_MEMCACHED name: env - image: ghcr.io/shuffle/shuffle-orborus:nightly + image: ghcr.io/shuffle/shuffle-orborus:latest #imagePullPolicy: Never name: shuffle-orborus resources: {} hostname: shuffle-orborus restartPolicy: Always -status: {} \ No newline at end of file +status: {} diff --git a/functions/kubernetes/charts/shuffle/Chart.yaml b/functions/kubernetes/charts/shuffle/Chart.yaml index 0a4e4fa3..b01df7da 100644 --- a/functions/kubernetes/charts/shuffle/Chart.yaml +++ b/functions/kubernetes/charts/shuffle/Chart.yaml @@ -3,7 +3,7 @@ name: shuffle description: A Helm chart for deploying Shuffle on Kubernetes type: application version: 0.0.0 # Set during publishing in GitHub actions -appVersion: nightly # Overwritten during publishing in GitHub actions +appVersion: latest # Overwritten during publishing in GitHub actions dependencies: - name: common version: ^2.23.0 diff --git a/functions/kubernetes/orborus.yaml b/functions/kubernetes/orborus.yaml index a94622ba..000d9fbd 100644 --- a/functions/kubernetes/orborus.yaml +++ b/functions/kubernetes/orborus.yaml @@ -72,7 +72,7 @@ spec: - name: AUTH value: "3663a270-bb3a-4678-a365-d879601a1a0c" - image: ghcr.io/shuffle/shuffle-orborus:nightly + image: ghcr.io/shuffle/shuffle-orborus:latest #imagePullPolicy: Never name: shuffle-orborus resources: {} diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 312ee7f1..fb93ad89 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -12,6 +12,7 @@ import ( "fmt" "io" "io/ioutil" + "regexp" "log" "math" "net" @@ -195,10 +196,10 @@ func getThisContainerId() { if containerId == "" { if containerName != "" { containerId = containerName - log.Printf("[INFO] Falling back to CONTAINER_NAME as container ID") + log.Printf("[INFO] Falling back to ORBORUS_CONTAINER_NAME as container ID") } else { containerId = "shuffle-orborus" - log.Printf(`[WARNING] CONTAINER_NAME is not set. Falling back to default name "%s" as container ID`, containerId) + log.Printf(`[WARNING] ORBORUS_CONTAINER_NAME env is not set. Falling back to default name "%s" as container ID. This may cause issues on the same server`, containerId) } } @@ -219,8 +220,8 @@ func skipCheckInCleanup(name string) bool { } func cleanupExistingNodes(ctx context.Context) error { - if cleanupEnv == "false" { - log.Printf("[INFO] Skipping cleanup of existing workers as CLEANUP is set to false. This should be auto-discovered during executions then instead.") + if cleanupEnv != "true" { + log.Printf("[INFO] Skipping cleanup of existing workers as CLEANUP is NOT set to true. Swarm actions are being auto-discovered during executions then instead.") return nil } @@ -342,7 +343,10 @@ func deployServiceWorkers(image string) { if len(dockerSwarmBridgeMTU) == 0 { mtu, err = strconv.Atoi(dockerSwarmBridgeMTU) // by default if err != nil { - log.Printf("[DEBUG] Failed to convert the default MTU to int: %s. Using 1500 instead. Input: %s", err, dockerSwarmBridgeMTU) + if debug { + log.Printf("[DEBUG] Failed to convert the default MTU to int: %s. Using 1500 instead. Input: %s", err, dockerSwarmBridgeMTU) + } + mtu = 1500 } } @@ -526,6 +530,7 @@ func deployServiceWorkers(image string) { nodeCount = uint64(cnt) } + appReplicas := os.Getenv("SHUFFLE_APP_REPLICAS") appReplicaCnt := 2 if len(appReplicas) > 0 { @@ -538,6 +543,9 @@ func deployServiceWorkers(image string) { } log.Printf("[DEBUG] Found %d node(s) to replicate over. Defaulting to 1 IF we can't auto-discover them.", cnt) + + // FIXME: From September 2025 - This is set back to 1, as this doesn't really reflect how scale works at all. It is just confusing, and makes number larger/smaller "arbitrarily" instead of using default docker scale + nodeCount = 1 replicatedJobs := uint64(replicas * nodeCount) log.Printf("[DEBUG] Deploying %d container(s) for worker with swarm to each node. Service name: %s. Image: %s", replicas, innerContainerName, image) @@ -1050,6 +1058,13 @@ func deployK8sWorker(image string, identifier string, env []string) error { env = append(env, fmt.Sprintf("SHUFFLE_BASE_IMAGE_REGISTRY=%s", os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY"))) } + if len(os.Getenv("SHUFFLE_BASE_IMAGE_NAME")) > 0 { + env = append(env, fmt.Sprintf("SHUFFLE_BASE_IMAGE_NAME=%s", os.Getenv("SHUFFLE_BASE_IMAGE_NAME"))) + } else { + log.Printf("[INFO] SHUFFLE_BASE_IMAGE_NAME is not set. Defaulting to %s", baseimagename) + env = append(env, fmt.Sprintf("SHUFFLE_BASE_IMAGE_NAME=%s", baseimagename)) + } + if len(os.Getenv("REGISTRY_URL")) > 0 { env = append(env, fmt.Sprintf("REGISTRY_URL=%s", os.Getenv("REGISTRY_URL"))) } @@ -1196,10 +1211,13 @@ func deployK8sWorker(image string, identifier string, env []string) error { ImagePullPolicy: corev1.PullIfNotPresent, } + if len(os.Getenv("REGISTRY_URL")) > 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_NAME")) > 0 { + log.Printf("[INFO] Setting image pull policy to Always as private registry is used.") + containerAttachment.ImagePullPolicy = corev1.PullAlways + } + podname := shuffle.GetPodName() - ctx := context.Background() - if len(podname) > 0 { _, err := shuffle.GetCurrentPodNetworkConfig(ctx, clientset, kubernetesNamespace, podname) if err != nil { @@ -1700,17 +1718,45 @@ func checkSwarmService(ctx context.Context) { return } + listenAddr := "0.0.0.0" req := swarm.InitRequest{ - ListenAddr: "0.0.0.0:2377", + ListenAddr: fmt.Sprintf("%s:2377", listenAddr), AdvertiseAddr: fmt.Sprintf("%s:2377", ip), } - ret, err := dockercli.SwarmInit(ctx, req) + id, err := dockercli.SwarmInit(ctx, req) if err != nil { - log.Printf("[WARNING] Swarm init: %s", err) + log.Printf("[ERROR] Swarm init issue: %s. Retrying with a failover IP address from interface.", err) + + // Dummy message used for testing + //err = errors.New("Error response from daemon: could not choose an IP address to advertise since this system has multiple addresses on different interfaces (10.52.208.221 on eno1 and 192.168.122.1 on virbr0) - specify one with --advertise-addr") + + msg := err.Error() + + // Extract all IPv4 addresses from the error message + var ipv4Re = regexp.MustCompile(`\b(?:\d{1,3}\.){3}\d{1,3}\b`) + candidates := ipv4Re.FindAllString(msg, -1) + if len(candidates) > 0 { + // Pick the first valid candidate (or implement your own heuristic) + + for cnt, candidate := range candidates { + if cnt > 5 { + break + } + + req.AdvertiseAddr = fmt.Sprintf("%s:2377", candidate) + _, err = dockercli.SwarmInit(context.Background(), req) + if err != nil { + continue + } + + break + } + + } } - log.Printf("[DEBUG] Swarm info: %s\n\n", ret) + log.Printf("[INFO] Swarm init ID: '%s'. If this is empty, there is most likely an error.", id) } func getContainerResourceUsage(ctx context.Context, cli *dockerclient.Client, containerID string) (float64, float64, error) { @@ -3939,10 +3985,11 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest, image string, identifier := "shuffle-workers" if isKubernetes == "true" { - if shuffle.IsRunningInCluster() { - log.Printf("[INFO] Running in Kubernetes cluster") - // try getting the k8s worker server url - } + // FIXME: Do we need this to map the cluster? + //if shuffle.IsRunningInCluster() { + //log.Printf("[INFO] Running in Kubernetes cluster") + // try getting the k8s worker server url + //} } if strings.Contains(streamUrl, "shuffler.io") || strings.Contains(streamUrl, "localhost") || strings.Contains(streamUrl, "127.0.0.1") || strings.Contains(streamUrl, "shuffle-backend") { diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 567b52b8..81153607 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -662,6 +662,14 @@ func deployk8sApp(image string, identifier string, env []string) error { }, } + if len(os.Getenv("REGISTRY_URL")) > 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_NAME")) > 0 { + log.Printf("[INFO] Setting image pull policy to Always as private registry is used.") + //containerAttachment.ImagePullPolicy = corev1.PullAlways + deployment.Spec.Template.Spec.Containers[0].ImagePullPolicy = corev1.PullAlways + } else { + deployment.Spec.Template.Spec.Containers[0].ImagePullPolicy = corev1.PullIfNotPresent + } + _, err = clientset.AppsV1().Deployments(kubernetesNamespace).Create(context.Background(), deployment, metav1.CreateOptions{}) if err != nil { log.Printf("[ERROR] Failed creating deployment: %v", err) @@ -1302,8 +1310,7 @@ func getWorkerURLs() ([]string, error) { // } // } // } - - log.Printf("[DEBUG] Worker URLs for k8s: %#v", workerUrls) + //log.Printf("[DEBUG] Worker URLs for k8s: %#v", workerUrls) return workerUrls, nil } @@ -3219,10 +3226,8 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl } // Apps used a lot should have 2 replicas (default) - replicas := uint64(1) - //if (strings.Contains(strings.ToLower(name), "shuffle") && strings.Contains(strings.ToLower(name), "tools")) || strings.Contains(strings.ToLower(name), "http") { - // replicas = 2 - //} + // New default to 3 (as the chance of queues piling up is lower) + replicas := uint64(3) // Sent from Orborus // Should be equal to @@ -3239,23 +3244,27 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl } // Max scale as well + nodeCount := uint64(1) if inputReplicas > 0 && inputReplicas < 100 { if replicas != uint64(inputReplicas) { log.Printf("[DEBUG] Overwriting replicas to %d/node as inputReplicas is set to %d", inputReplicas, inputReplicas) } replicas = uint64(inputReplicas) + } else { + cnt, err := findActiveSwarmNodes(dockercli) + if err != nil { + log.Printf("[ERROR] Unable to find active swarm nodes: %s", err) + } + + if cnt > 0 { + nodeCount = uint64(cnt) + } + + // FIXME: From September 2025 - This is set back to 1, as this doesn't really reflect how scale works at all. It is just confusing, and makes number larger/smaller "arbitrarily" instead of using default docker scale + nodeCount = 1 } - cnt, err := findActiveSwarmNodes(dockercli) - if err != nil { - log.Printf("[ERROR] Unable to find active swarm nodes: %s", err) - } - - nodeCount := uint64(1) - if cnt > 0 { - nodeCount = uint64(cnt) - } replicatedJobs := uint64(replicas * nodeCount) log.Printf("[DEBUG] Deploying app with name %s with image %s", name, image) @@ -3615,7 +3624,10 @@ func findAppInfoKubernetes(image, name string, env []string) error { for _, deployment := range deployments.Items { if deployment.Name == name { - log.Printf("[INFO] Found deployment %s - no need to deploy another", name) + if debug { + log.Printf("[DEBUG] Found deployment %s - no need to deploy another", name) + } + return nil } } @@ -4025,7 +4037,7 @@ func getStreamResultsWrapper(client *http.Client, req *http.Request, workflowExe if newresp.StatusCode != 200 { log.Printf("[ERROR] StatusCode (1): %d - %s", newresp.StatusCode, string(body)) time.Sleep(time.Duration(sleepTime) * time.Second) - return environments, errors.New(fmt.Sprintf("Bad status code: %d", newresp.StatusCode)) + return environments, errors.New(fmt.Sprintf("Bad status code from backend: %d", newresp.StatusCode)) } err = json.Unmarshal(body, &workflowExecution)