From 3491ba6471630152acde240f679fdd2237713bc7 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Wed, 16 Apr 2025 11:34:41 +0200 Subject: [PATCH 1/2] feat(k8s): allow to change exposed app port Signed-off-by: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> --- functions/onprem/orborus/orborus.go | 7 +++-- functions/onprem/worker/worker.go | 48 ++++++++++++++++------------- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 1f37da85..36e7979e 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -747,7 +747,7 @@ func handleBackendImageDownload(ctx context.Context, images string) error { //log.Printf("[DEBUG] Removing existing image (s): %s", images) newImages := []string{} - successful := []string{} + successful := []string{} for _, curimage := range strings.Split(images, ",") { curimage = strings.TrimSpace(curimage) if shuffle.ArrayContains(handled, curimage) { @@ -996,6 +996,10 @@ func deployK8sWorker(image string, identifier string, env []string) error { env = append(env, fmt.Sprintf("SHUFFLE_USE_GHCR_OVERRIDE_FOR_AUTODEPLOY=%s", os.Getenv("SHUFFLE_USE_GHCR_OVERRIDE_FOR_AUTODEPLOY"))) } + if len(os.Getenv("SHUFFLE_APP_EXPOSED_PORT")) > 0 { + env = append(env, fmt.Sprintf("SHUFFLE_APP_EXPOSED_PORT=%s", os.Getenv("SHUFFLE_APP_EXPOSED_PORT"))) + } + if len(appServiceAccountName) > 0 { env = append(env, fmt.Sprintf("SHUFFLE_APP_SERVICE_ACCOUNT_NAME=%s", appServiceAccountName)) } @@ -1271,7 +1275,6 @@ func deployWorker(image string, identifier string, env []string, executionReques Resources: container.Resources{}, } - // This is just to test the mounting locally so // I can control from what source I'm mounting // the certs to. Default behaviour is: diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index a8f1c204..c411ae52 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -399,6 +399,11 @@ func deployk8sApp(image string, identifier string, env []string) error { kubernetesNamespace = "default" } + deployport, err := strconv.Atoi(os.Getenv("SHUFFLE_APP_EXPOSED_PORT")) + if err != nil { + deployport = 80 + } + envMap := make(map[string]string) for _, envStr := range env { parts := strings.SplitN(envStr, "=", 2) @@ -408,9 +413,7 @@ func deployk8sApp(image string, identifier string, env []string) error { } // add to env - // fmt.Sprintf("SHUFFLE_APP_EXPOSED_PORT=%d", deployport), - // fmt.Sprintf("SHUFFLE_SWARM_CONFIG=%s", os.Getenv("SHUFFLE_SWARM_CONFIG")), - envMap["SHUFFLE_APP_EXPOSED_PORT"] = "80" + envMap["SHUFFLE_APP_EXPOSED_PORT"] = strconv.Itoa(deployport) envMap["SHUFFLE_SWARM_CONFIG"] = os.Getenv("SHUFFLE_SWARM_CONFIG") envMap["BASE_URL"] = "http://shuffle-workers:33333" @@ -599,6 +602,12 @@ func deployk8sApp(image string, identifier string, env []string) error { Name: value, Image: image, Env: buildEnvVars(envMap), + Ports: []corev1.ContainerPort{ + { + Protocol: "TCP", + ContainerPort: int32(deployport), + }, + }, }, }, DNSPolicy: corev1.DNSClusterFirst, @@ -614,7 +623,6 @@ func deployk8sApp(image string, identifier string, env []string) error { return err } - // kubectl expose deployment {podName} --type=NodePort --port=80 --target-port=80 service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -626,7 +634,7 @@ func deployk8sApp(image string, identifier string, env []string) error { { Protocol: "TCP", Port: 80, - TargetPort: intstr.FromInt(80), + TargetPort: intstr.FromInt(deployport), }, }, Type: corev1.ServiceTypeNodePort, @@ -917,7 +925,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] // Add more volume binds if possible if len(volumeBinds) > 0 { - // Only use mounts, not direct binds + // Only use mounts, not direct binds hostConfig.Binds = []string{} hostConfig.Mounts = []mount.Mount{} for _, bind := range volumeBinds { @@ -931,7 +939,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] sourceFolder := bindSplit[0] destinationFolder := bindSplit[1] - readOnly := false + readOnly := false if len(bindSplit) > 2 { mode := bindSplit[2] if mode == "ro" { @@ -940,9 +948,9 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] } builtMount := mount.Mount{ - Type: mount.TypeBind, - Source: sourceFolder, - Target: destinationFolder, + Type: mount.TypeBind, + Source: sourceFolder, + Target: destinationFolder, ReadOnly: readOnly, } @@ -1853,18 +1861,18 @@ func executionInit(workflowExecution shuffle.WorkflowExecution) error { } } - // Validates RERUN of single actions - // Identified by: + // Validates RERUN of single actions + // Identified by: // 1. Predefined result from previous exec // 2. Only ONE action // 3. Every predefined result having result.Action.Category == "rerun" /* - if len(workflowExecution.Workflow.Actions) == 1 && len(workflowExecution.Results) > 0 { - finished := shuffle.ValidateFinished(ctx, extra, workflowExecution) - if finished { - return nil + if len(workflowExecution.Workflow.Actions) == 1 && len(workflowExecution.Results) > 0 { + finished := shuffle.ValidateFinished(ctx, extra, workflowExecution) + if finished { + return nil + } } - } */ nextActions = append(nextActions, startAction) @@ -1954,7 +1962,6 @@ func executionInit(workflowExecution shuffle.WorkflowExecution) error { //log.Printf("Successfully downloaded and built %s", image) } - visited := []string{} executed := []string{} environments := []string{} @@ -3777,7 +3784,7 @@ func checkStandaloneRun() { if !strings.Contains(backendUrl, "http") { log.Printf("[ERROR] Backend URL should start with http:// or https://") return - + } // Format: @@ -3851,7 +3858,7 @@ func checkStandaloneRun() { continue } - // This is to handle reruns of SINGLE actions + // This is to handle reruns of SINGLE actions if result.Action.Category == "rerun" { newResults = append(newResults, result) continue @@ -3905,7 +3912,6 @@ func checkStandaloneRun() { log.Printf("\n\n\n[DEBUG] Finished resetting execution %s. Body: %s. Starting execution.\n\n\n", newresp.Status, string(body)) - } // Initial loop etc From e4a42128e7722abb9728a8681ef2b4fb8aa803b0 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Thu, 17 Apr 2025 12:06:59 +0200 Subject: [PATCH 2/2] feat(helm): allow to configure exposed app container port Signed-off-by: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> --- functions/kubernetes/charts/shuffle/README.md | 2 ++ .../shuffle/templates/orborus/orborus-dpl.yaml | 2 ++ .../shuffle-app/shuffle-app-network-policy.yaml | 13 +++++++------ .../kubernetes/charts/shuffle/values.schema.json | 5 +++++ functions/kubernetes/charts/shuffle/values.yaml | 4 ++++ 5 files changed, 20 insertions(+), 6 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/README.md b/functions/kubernetes/charts/shuffle/README.md index ff3885d1..1faef45c 100644 --- a/functions/kubernetes/charts/shuffle/README.md +++ b/functions/kubernetes/charts/shuffle/README.md @@ -510,6 +510,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `app.networkPolicy.allowExternalEgress` | Allow the pod to access any range of port and all destinations. | `true` | | `app.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` | | `app.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | +| `app.exposedContainerPort` | The port that shuffle app containers will listen on for new requests. | `80` | ### Traffic Exposure Parameters @@ -607,3 +608,4 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia ### Other Parameters + diff --git a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml index a2d9c278..3e6616ef 100644 --- a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml +++ b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml @@ -88,6 +88,8 @@ spec: value: "true" - name: SHUFFLE_WORKER_SERVICE_ACCOUNT_NAME value: {{ include "shuffle.worker.serviceAccount.name" . }} + - name: SHUFFLE_APP_EXPOSED_PORT + value: {{ .Values.app.exposedContainerPort | quote }} - name: SHUFFLE_APP_SERVICE_ACCOUNT_NAME value: {{ include "shuffle.app.serviceAccount.name" . }} {{- if .Values.orborus.extraEnvVars }} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-network-policy.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-network-policy.yaml index d4a24fe6..74600475 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-network-policy.yaml +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-network-policy.yaml @@ -44,17 +44,18 @@ spec: {{- end }} {{- end }} ingress: - {{- if .Values.app.networkPolicy.allowExternal }} - - {} - {{- else }} - # Allow access from workers. Apps will typicaly use port 80/TCP, but this is not enforced. - - from: + - ports: + - port: {{ .Values.app.exposedContainerPort }} + protocol: TCP + {{- if not .Values.app.networkPolicy.allowExternal }} + # Allow traffic from workers + from: - namespaceSelector: matchLabels: kubernetes.io/metadata.name: {{ .Release.Namespace }} podSelector: matchLabels: {{ include "shuffle.worker.matchLabels" . | nindent 14 }} - {{- end }} + {{- end }} {{- if .Values.app.networkPolicy.extraIngress }} {{- include "common.tplvalues.render" ( dict "value" .Values.app.networkPolicy.extraIngress "context" $ ) | nindent 4 }} {{- end }} diff --git a/functions/kubernetes/charts/shuffle/values.schema.json b/functions/kubernetes/charts/shuffle/values.schema.json index b785b76e..c1b89282 100644 --- a/functions/kubernetes/charts/shuffle/values.schema.json +++ b/functions/kubernetes/charts/shuffle/values.schema.json @@ -2224,6 +2224,11 @@ "items": {} } } + }, + "exposedContainerPort": { + "type": "number", + "description": "The port that shuffle app containers will listen on for new requests. ", + "default": 80 } } }, diff --git a/functions/kubernetes/charts/shuffle/values.yaml b/functions/kubernetes/charts/shuffle/values.yaml index 507c2468..c63bfd22 100644 --- a/functions/kubernetes/charts/shuffle/values.yaml +++ b/functions/kubernetes/charts/shuffle/values.yaml @@ -1449,6 +1449,10 @@ app: ## extraEgress: [] + ## @param app.exposedContainerPort The port that shuffle app containers will listen on for new requests. + ## + exposedContainerPort: 80 + ## @section Traffic Exposure Parameters ##