From 5923c6865758354118baba947ec79937394f6e27 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Mon, 1 Dec 2025 10:52:18 +0100 Subject: [PATCH 01/41] start working on helm-deployed workers and apps Signed-off-by: Pascal Sthamer --- .../charts/shuffle/templates/_helpers.tpl | 38 +++-- .../templates/orborus/orborus-cm-env.yaml | 7 +- .../templates/orborus/orborus-role.yaml | 16 ++ .../shuffle-app/shuffle-app-deployments.yaml | 17 ++ .../shuffle-worker/shuffle-worker-dpl.yaml | 161 ++++++++++++++++++ .../shuffle-worker/shuffle-worker-role.yaml | 8 + .../kubernetes/charts/shuffle/values.yaml | 47 ++++- functions/onprem/orborus/orborus.go | 1 + 8 files changed, 280 insertions(+), 15 deletions(-) create mode 100644 functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-deployments.yaml create mode 100644 functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml diff --git a/functions/kubernetes/charts/shuffle/templates/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/_helpers.tpl index dc93dfc2..67ca45cf 100644 --- a/functions/kubernetes/charts/shuffle/templates/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/_helpers.tpl @@ -60,19 +60,31 @@ app.kubernetes.io/component: orborus {{- end -}} {{/* -Return the common labels for worker components +Return the common labels for worker components deployed via helm */}} {{- define "shuffle.worker.labels" -}} -{{- include "common.labels.standard" . }} -app.kubernetes.io/component: worker +app.kubernetes.io/name: shuffle-worker +helm.sh/chart: {{ include "common.names.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/part-of: shuffle +{{- with .Chart.AppVersion }} +app.kubernetes.io/version: {{ . | replace "+" "_" | quote }} +{{- end -}} {{- end -}} {{/* -Return the common labels for app components +Return the common labels for app components deployed via helm */}} {{- define "shuffle.app.labels" -}} -{{- include "common.labels.standard" . }} -app.kubernetes.io/component: app +app.kubernetes.io/name: shuffle-app +helm.sh/chart: {{ include "common.names.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/part-of: shuffle +{{- with .Chart.AppVersion }} +app.kubernetes.io/version: {{ . | replace "+" "_" | quote }} +{{- end -}} {{- end -}} {{/* @@ -100,16 +112,18 @@ app.kubernetes.io/component: orborus {{- end -}} {{/* -Return the match labels for worker components -NOTE: This does not match the labels from shuffle.worker.labels, but the labels set by the orborus GoLang app. +Return the match labels for worker components. +These must match the labels of helm-deployed worker components (shuffle.worker.labels), +as well as orborus-deployed worker components (deployk8sworker). */}} {{- define "shuffle.worker.matchLabels" -}} app.kubernetes.io/name: shuffle-worker {{- end -}} {{/* -Return the match labels for app components -NOTE: This does not match the labels from shuffle.worker.labels, but the labels set by the orborus GoLang app. +Return the match labels for app components. +These must match the labels of helm-deployed app components (shuffle.app.labels), +as well as worker-deployed app deployments (deployK8sApp). */}} {{- define "shuffle.app.matchLabels" -}} app.kubernetes.io/name: shuffle-app @@ -375,3 +389,7 @@ imagePullSecrets: {{- end }} {{- end }} {{- end -}} + +{{- define "shuffle.backend.baseUrl -}} +http://{{ include "shuffle.backend.name" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.backend.containerPorts.http }} +{{- end - }} \ No newline at end of file diff --git a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-cm-env.yaml b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-cm-env.yaml index 70209c21..28f2aa7e 100644 --- a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-cm-env.yaml +++ b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-cm-env.yaml @@ -11,10 +11,10 @@ data: ENVIRONMENT_NAME: "{{ .Values.shuffle.org }}" ORG_ID: "{{ .Values.shuffle.org }}" TZ: "{{ .Values.shuffle.timezone }}" - BASE_URL: "http://{{ include "shuffle.backend.name" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.backend.containerPorts.http }}" - KUBERNETES_NAMESPACE: "{{ .Release.Namespace }}" + BASE_URL: {{ import "shuffle.backend.baseUrl" . | quote }} REGISTRY_URL: "{{ .Values.shuffle.appRegistry }}" + {{- if .Values.orborus.manageWorkerDeployments }} # Shuffle worker configuration SHUFFLE_WORKER_IMAGE: {{ include "shuffle.worker.image" . | quote }} SHUFFLE_WORKER_SERVICE_ACCOUNT_NAME: {{ include "shuffle.worker.serviceAccount.name" . | quote }} @@ -46,6 +46,7 @@ data: SHUFFLE_WORKER_EPHEMERAL_STORAGE_LIMIT: {{ (index $workerResources.limits "ephemeral-storage") | quote }} {{- end }} + {{- if .Values.worker.manageAppDeployments }} # Shuffle app configuration SHUFFLE_APP_EXPOSED_PORT: {{ .Values.app.exposedContainerPort | quote }} SHUFFLE_APP_SERVICE_ACCOUNT_NAME: {{ include "shuffle.app.serviceAccount.name" . | quote }} @@ -76,3 +77,5 @@ data: {{- if and $appResources.limits (index $appResources.limits "ephemeral-storage") }} SHUFFLE_APP_EPHEMERAL_STORAGE_LIMIT: {{ (index $appResources.limits "ephemeral-storage") | quote }} {{- end }} + {{- end }} + {{- end }} diff --git a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-role.yaml b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-role.yaml index 90041d89..b3f0ca13 100644 --- a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-role.yaml +++ b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-role.yaml @@ -9,6 +9,7 @@ metadata: annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} {{- end }} rules: + {{- if .Values.orborus.manageWorkerDeployments }} - verbs: - list - create @@ -26,4 +27,19 @@ rules: - apps resources: - deployments + {{- else }} + - verbs: + - list + apiGroups: + - '' + resources: + - pods + - services + - verbs: + - list + apiGroups: + - apps + resources: + - deployments + {{- end }} {{- end }} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-deployments.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-deployments.yaml new file mode 100644 index 00000000..a1679e40 --- /dev/null +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-deployments.yaml @@ -0,0 +1,17 @@ +TODO: + - Deploy each app (configurable in values) here. + - Have defaults for all apps + - Allow overriding per app (serviceAccount, securityContext, volumes , env, resources, replicas, ...) + +{{- range $index, $app := .Values.app.deployViaHelm }} +--- +apiVersion: v1 +kind: Service +metadata: +spec: +--- +apiVersion: apps/v1 +kind: Deployment +metadata: +spec: +{{- end }} \ No newline at end of file diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml new file mode 100644 index 00000000..18641999 --- /dev/null +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml @@ -0,0 +1,161 @@ +{{- if .Values.worker.enableHelmDeployment }} +apiVersion: {{ include "common.capabilities.deployment.apiVersion" . }} +kind: Deployment +metadata: + name: {{ template "shuffle.worker.name" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "shuffle.worker.labels" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if or .Values.worker.deploymentAnnotations .Values.commonAnnotations }} + {{- $annotations := include "common.tplvalues.merge" (dict "values" (list .Values.worker.deploymentAnnotations .Values.commonAnnotations) "context" .) }} + annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + {{- if not .Values.worker.autoscaling.hpa.enabled }} + replicas: {{ .Values.worker.replicaCount }} + {{- end }} + {{- if .Values.worker.updateStrategy }} + strategy: {{- toYaml .Values.worker.updateStrategy | nindent 4 }} + {{- end }} + {{- $podLabels := include "common.tplvalues.merge" (dict "values" (list .Values.worker.podLabels .Values.commonLabels) "context" .) }} + selector: + matchLabels: {{- include "shuffle.worker.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 6 }} + template: + metadata: + {{- if .Values.worker.podAnnotations }} + annotations: {{- include "common.tplvalues.render" (dict "value" .Values.worker.podAnnotations "context" $) | nindent 8 }} + {{- end }} + labels: {{- include "shuffle.worker.labels" ( dict "customLabels" $podLabels "context" $ ) | nindent 8 }} + spec: + {{- include "shuffle.worker.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ template "shuffle.worker.serviceAccount.name" . }} + automountServiceAccountToken: {{ .Values.worker.automountServiceAccountToken }} + {{- if .Values.worker.hostAliases }} + hostAliases: {{- include "common.tplvalues.render" (dict "value" .Values.worker.hostAliases "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.worker.affinity }} + affinity: {{- include "common.tplvalues.render" ( dict "value" .Values.worker.affinity "context" $) | nindent 8 }} + {{- else }} + affinity: + podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.worker.podAffinityPreset "component" "orborus" "customLabels" $podLabels "context" $) | nindent 10 }} + podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.worker.podAntiAffinityPreset "component" "orborus" "customLabels" $podLabels "context" $) | nindent 10 }} + nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.worker.nodeAffinityPreset.type "key" .Values.worker.nodeAffinityPreset.key "values" .Values.worker.nodeAffinityPreset.values) | nindent 10 }} + {{- end }} + {{- if .Values.worker.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" ( dict "value" .Values.worker.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.worker.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.worker.tolerations "context" .) | nindent 8 }} + {{- end }} + {{- if .Values.worker.priorityClassName }} + priorityClassName: {{ .Values.worker.priorityClassName | quote }} + {{- end }} + {{- if .Values.worker.schedulerName }} + schedulerName: {{ .Values.worker.schedulerName | quote }} + {{- end }} + {{- if .Values.worker.topologySpreadConstraints }} + topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.worker.topologySpreadConstraints "context" .) | nindent 8 }} + {{- end }} + {{- if .Values.worker.podSecurityContext.enabled }} + securityContext: {{- omit .Values.worker.podSecurityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + {{- if .Values.worker.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ .Values.worker.terminationGracePeriodSeconds }} + {{- end }} + initContainers: + {{- if .Values.worker.initContainers }} + {{- include "common.tplvalues.render" (dict "value" .Values.worker.initContainers "context" $) | nindent 8 }} + {{- end }} + containers: + - name: orborus + image: {{ template "shuffle.worker.image" . }} + imagePullPolicy: {{ .Values.worker.image.pullPolicy }} + {{- if .Values.worker.containerSecurityContext.enabled }} + securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.worker.containerSecurityContext "context" $) | nindent 12 }} + {{- end }} + {{- if .Values.diagnosticMode.enabled }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }} + {{- else if .Values.worker.command }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.worker.command "context" $) | nindent 12 }} + {{- end }} + {{- if .Values.diagnosticMode.enabled }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 12 }} + {{- else if .Values.worker.args }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.worker.args "context" $) | nindent 12 }} + {{- end }} + env: + - name: IS_KUBERNETES + value: "true" + - name: KUBERNETES_NAMESPACE + value: "{{ .Release.Namespace }}" + - name: SHUFFLE_LOGS_DISABLED + value: "false" # TODO: Allow to configure. Share with orborus / apps? + - name: BASE_URL + value: {{ import "shuffle.backend.baseUrl" . | quote }} + - name: REGISTRY_URL # TODO: REGISTRY_URL vs. SHUFFLE_BASE_IMAGE_REGISTRY + value: {{ .Values.shuffle.appRegistry }} + - name: SHUFFLE_BASE_IMAGE_REGISTRY + value: {{ .Values.shuffle.appRegistry }} + - name: SHUFFLE_BASE_IMAGE_NAME + value: shuffle # TODO: Allow to configure? Use same as for orborus (shared value) + - name: SHUFFLE_APP_EXPOSED_PORT + value: {{ .Values.app.exposedContainerPort | quote }} + - name: WORKER_HOSTNAME + value: {{ include "shuffle.worker.name" }}.{{ .Release.Namespace }}.svc.cluster.local + {{- if .Values.worker.extraEnvVars }} + {{- include "common.tplvalues.render" (dict "value" .Values.worker.extraEnvVars "context" $) | nindent 12 }} + {{- end }} + envFrom: + {{- if .Values.worker.extraEnvVarsCM }} + - configMapRef: + name: {{ include "common.tplvalues.render" (dict "value" .Values.worker.extraEnvVarsCM "context" $) }} + {{- end }} + {{- if .Values.worker.extraEnvVarsSecret }} + - secretRef: + name: {{ include "common.tplvalues.render" (dict "value" .Values.worker.extraEnvVarsSecret "context" $) }} + {{- end }} + {{- if .Values.worker.resources }} + resources: {{- toYaml .Values.worker.resources | nindent 12 }} + {{- else if ne .Values.worker.resourcesPreset "none" }} + resources: {{- include "common.resources.preset" (dict "type" .Values.worker.resourcesPreset) | nindent 12 }} + {{- end }} + ports: + {{- if .Values.worker.extraContainerPorts }} + {{- include "common.tplvalues.render" (dict "value" .Values.worker.extraContainerPorts "context" $) | nindent 12 }} + {{- end }} + {{- if not .Values.diagnosticMode.enabled }} + {{- if .Values.worker.customLivenessProbe }} + livenessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.worker.customLivenessProbe "context" $) | nindent 12 }} + {{- else if .Values.worker.livenessProbe.enabled }} + livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.worker.livenessProbe "enabled") "context" $) | nindent 12 }} + {{- end }} + {{- if .Values.worker.customReadinessProbe }} + readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.worker.customReadinessProbe "context" $) | nindent 12 }} + {{- else if .Values.worker.readinessProbe.enabled }} + readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.worker.readinessProbe "enabled") "context" $) | nindent 12 }} + {{- end }} + {{- if .Values.worker.customStartupProbe }} + startupProbe: {{- include "common.tplvalues.render" (dict "value" .Values.worker.customStartupProbe "context" $) | nindent 12 }} + {{- else if .Values.worker.startupProbe.enabled }} + startupProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.worker.startupProbe "enabled") "context" $) | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.worker.lifecycleHooks }} + lifecycle: {{- include "common.tplvalues.render" (dict "value" .Values.worker.lifecycleHooks "context" $) | nindent 12 }} + {{- end }} + volumeMounts: + - name: empty-dir + mountPath: /tmp + subPath: tmp-dir + {{- if .Values.worker.extraVolumeMounts }} + {{- include "common.tplvalues.render" (dict "value" .Values.worker.extraVolumeMounts "context" $) | nindent 12 }} + {{- end }} + {{- if .Values.worker.sidecars }} + {{- include "common.tplvalues.render" ( dict "value" .Values.worker.sidecars "context" $) | nindent 8 }} + {{- end }} + volumes: + - name: empty-dir + emptyDir: {} + {{- if .Values.worker.extraVolumes }} + {{- include "common.tplvalues.render" (dict "value" .Values.worker.extraVolumes "context" $) | nindent 8 }} + {{- end }} +{{- end }} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-role.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-role.yaml index fdef5855..fffa4d4b 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-role.yaml +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-role.yaml @@ -9,6 +9,7 @@ metadata: annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} {{- end }} rules: + {{- if .Values.worker.manageAppDeployments }} - apiGroups: [""] resources: ["pods"] verbs: ["list", "delete"] @@ -18,4 +19,11 @@ rules: - apiGroups: ["apps"] resources: ["deployments"] verbs: ["list", "create"] + {{- else }} + - apiGroups: [""] + resources: ["pods"] + verbs: ["list"] + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["list"] {{- end }} diff --git a/functions/kubernetes/charts/shuffle/values.yaml b/functions/kubernetes/charts/shuffle/values.yaml index 6e1b6b8f..8ee8ee06 100644 --- a/functions/kubernetes/charts/shuffle/values.yaml +++ b/functions/kubernetes/charts/shuffle/values.yaml @@ -84,7 +84,7 @@ shuffle: ## @param shuffle.appRegistry The registry from / to which shuffle apps are pulled / pushed ## - appRegistry: "" + appRegistry: "" # TODO: Default? ## @param shuffle.timezone The timezone used by Shuffle ## @@ -1209,7 +1209,7 @@ orborus: - name: KUBERNETES_NAMESPACE value: "shuffle" - name: SHUFFLE_BASE_IMAGE_NAME - value: "frikky/shuffle" + value: "frikky/shuffle" # TODO: Allow to configure, share with worker ## @param orborus.extraEnvVarsCM Name of existing ConfigMap containing extra env vars for orborus containers ## extraEnvVarsCM: "" @@ -1346,10 +1346,20 @@ orborus: ## istio: pilot ## extraEgress: [] + + ## @param orborus.manageWorkerDeployments Whether workers are deployed and managed by orborus. When disabled, every worker is expected to be already deployed (see worker.enableHelmDeployment). + ## This effectively removes required RBAC permissions from the shuffle-orborus service account to create deployments and services. + ## Orborus might still attempt to create kubernetes objects, resulting in an error. There is currently no way to tell orborus, that it should not manage k8s resources. + ## You likely want to enable worker.enableHelmDeployment when enabling this. + manageWorkerDeployments: true ## @section worker Parameters ## -worker: +worker: # TODO: Add values that other helm deployed components have, mark helm-only params + ## @param worker.enableHelmDeployment Deploy worker via helm. By default, workers are deployed by Orborus. + ## You might want to disable orborus.manageWorkerDeployments when enabling this. + enableHelmDeployment: false + ## worker image ## @param worker.image.registry worker image registry ## @param worker.image.repository worker image repository @@ -1480,6 +1490,11 @@ worker: ## istio: pilot ## extraEgress: [] + + ## @param worker.manageAppDeployments Whether apps are deployed and managed by worker. When disabled, every used app is expected to to be already deployed (see app.deployViaHelm). + ## This effectively removes required RBAC permissions from the shuffle-worker service account to create deployments and services. + ## The worker might still attempt to create kubernetes objects, resulting in an error. There is currently no way to tell the worker, that it should not manage k8s resources. + manageAppDeployments: true ## @section app Parameters ## @@ -1598,6 +1613,32 @@ app: ## exposedContainerPort: 80 + ## @param app.deployViaHelm A list of apps that should be deployed using helm. + ## By default, workers create deployments and services for apps when they are first needed (or during startup for some selected apps). + ## Deploying apps via workers has some drawbacks, such as: + ## - A workflow fails when the app is not deployed when the workflow gets executed (see https://github.com/Shuffle/Shuffle/issues/1739) + ## - There is no way to set different service accounts, security contexts, resources, env variables, volume mounts, or replicas for different apps + ## - Worker needs elevated permissions in Kubernetes + ## Note that you can deploy some apps via helm, while keeping the flexibility of letting workers deploy apps if they are not already deployed. + ## If you deploy all needed apps via helm and dont want workers to create additional deployments, set worker.manageAppDeployments to false. + ## + ## Sample value: + ## - app: my-app # required + ## version: 1.0.0 # required + ## replicas: 1 # optional, default to 1 + ## image: "" # optional, default to appregistry/baseimage/appname:appversion + ## imagePullSecrets: null # optional, defaults to global.imagePullSecrets + ## resources: null # optional, defaults to app.resources (or app.resourcePreset) + ## extraVolumes: [] # optional + ## extraVolumeMounts: [] # optional + ## extraEnv: [] # optional + ## podSecurityContext: null # optional, defaults to app.podSecurityContext + ## containerSecurityContext: null # optional, defaults to app.containerSecurityContext + ## + ## TODO: allow to override resources, serviceAccount, mounts, env, security Contexts. Fall back to app defaults as defined above. + ## + deployViaHelm: [] + ## @section Traffic Exposure Parameters ## diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index a1cda9a2..b9e7607f 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -1160,6 +1160,7 @@ func fixk8sRoles() { } } +// TODO: Check if deployment or service already exist by labels and only create if not already exists func deployK8sWorker(image string, identifier string, env []string) error { env = append(env, fmt.Sprintf("IS_KUBERNETES=true")) env = append(env, fmt.Sprintf("KUBERNETES_NAMESPACE=%s", os.Getenv("KUBERNETES_NAMESPACE"))) From 9cbbf533cca547266920b1a61d68abd9d3aa8bb9 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Mon, 1 Dec 2025 11:04:51 +0100 Subject: [PATCH 02/41] readd missing env variable Signed-off-by: Pascal Sthamer --- .../charts/shuffle/templates/orborus/orborus-cm-env.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-cm-env.yaml b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-cm-env.yaml index 28f2aa7e..4ce584bd 100644 --- a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-cm-env.yaml +++ b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-cm-env.yaml @@ -12,6 +12,7 @@ data: ORG_ID: "{{ .Values.shuffle.org }}" TZ: "{{ .Values.shuffle.timezone }}" BASE_URL: {{ import "shuffle.backend.baseUrl" . | quote }} + KUBERNETES_NAMESPACE: "{{ .Release.Namespace }}" REGISTRY_URL: "{{ .Values.shuffle.appRegistry }}" {{- if .Values.orborus.manageWorkerDeployments }} From 42c28926cb75342774ecfb57350a2e1f127a387b Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Mon, 1 Dec 2025 14:11:27 +0100 Subject: [PATCH 03/41] add documentation Signed-off-by: Pascal Sthamer --- functions/kubernetes/charts/shuffle/README.md | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/functions/kubernetes/charts/shuffle/README.md b/functions/kubernetes/charts/shuffle/README.md index c9810210..0fea5581 100644 --- a/functions/kubernetes/charts/shuffle/README.md +++ b/functions/kubernetes/charts/shuffle/README.md @@ -105,6 +105,46 @@ SHUFFLE_DEFAULT_APIKEY: "72E41083-A6F6-4A1B-8538-B06B577F47F0" # Shuffle uses uu SHUFFLE_ENCRYPTION_MODIFIER: "MyShuffleEncryptionModifier" ``` +## Shuffle Worker + +By default, Orborus creates a Kubernetes Deployment and Service for Shuffle Worker. +The deployment can be customized to some degree using some of the `worker.*` helm variables. They are converted to Orborus env variables. + +If you want full control, you can also deploy Shuffle Worker using helm by enabling `worker.enableHelmDeployment`. +This approach respects all of the `worker.*` helm variables. + +You can then set `orborus.manageWorkerDeployments=false` to reduce the permissions assigned to the Shuffle Orborus Kubernetes service account. + +## Shuffle Apps + +By default, Shuffle Worker is responsible for creating Kubernetes Deployments and Services for each app. +Each app and version has their own Deployment and Service. Shuffle automatically deploys a set of apps. +Other apps are deployed on demand, when they are first used. + +You can use some of the `app.*` helm variables to control some aspects of the deployment, e.g. resources and security context. +Helm variables are converted to env variables set on Orborus. Orborus in turn passes the env variables to Worker when creating the Deployment. +When `worker.enableHelmDeployment` is set, env variables for app configuration are set on the worker directly. +Configuration using env variables applies to ALL deployed apps. There is no way to assign different options (e.g. resources) to different apps, or scale apps individually. + +If you want full control, you can deploy apps using helm. This has the following advantages: +- full control over the deployment using helm values +- granular control per app and version (e.g. have more replicas and resources for frequently used apps) +- avoid problems with on-demand started apps (see https://github.com/Shuffle/Shuffle/issues/1739) + +To deploy apps using helm, use the `app.deployViaHelm` value: +```yaml +app: + deployViaHelm: + - app: shuffle-tools + version: 1.2.0 + TODO +``` + +It is possible to use a hybrid approach - deploy some apps using helm, while still allowing Worker to create additional apps on-demand. + +If you do not want Worker to manage app deployments, set `worker.manageAppDeployments=true`. This effectively removes the required permissions from the Shuffle Worker Kubernetes Service Account. +You are required to deploy all apps that are in use by your Shuffle instance manually using Helm. + ## OpenSearch Shuffle uses OpenSearch as its database. This helm chart installs a single-node OpenSearch cluster using [the Bitnami Helm Chart](https://github.com/bitnami/charts/blob/main/bitnami/opensearch/values.yaml). From 894c6318ad391a72990f44fdd610862c6a88792b Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Mon, 1 Dec 2025 14:12:35 +0100 Subject: [PATCH 04/41] customizable worker, fix worker and app labels Signed-off-by: Pascal Sthamer --- .../charts/shuffle/templates/_helpers.tpl | 91 ++++-- .../backend/backend-network-policy.yaml | 4 +- .../orborus/orborus-network-policy.yaml | 2 +- .../shuffle-app-network-policy.yaml | 6 +- .../shuffle-worker/shuffle-worker-dpl.yaml | 7 +- .../shuffle-worker-network-policy.yaml | 6 +- .../shuffle-worker/shuffle-worker-svc.yaml | 21 ++ .../kubernetes/charts/shuffle/values.yaml | 276 +++++++++++++++++- 8 files changed, 370 insertions(+), 43 deletions(-) create mode 100644 functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-svc.yaml diff --git a/functions/kubernetes/charts/shuffle/templates/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/_helpers.tpl index 67ca45cf..061ea4f4 100644 --- a/functions/kubernetes/charts/shuffle/templates/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/_helpers.tpl @@ -27,7 +27,7 @@ Return the common name for worker components {{- end -}} {{/* -Return the common name for app components +Return the common name for app components. */}} {{- define "shuffle.app.name" -}} {{- printf "%s-app" (include "common.names.fullname" .) | trunc 63 -}} @@ -60,31 +60,78 @@ app.kubernetes.io/component: orborus {{- end -}} {{/* -Return the common labels for worker components deployed via helm +Return the common labels for worker components deployed via helm. +NOTE: Worker deployments and services use shuffle.workerInstance.labels instead. */}} {{- define "shuffle.worker.labels" -}} +{{- include "common.labels.standard" . }} +app.kubernetes.io/component: worker +{{- end -}} + + +{{/* +Return the labels for a specific worker instance deployed via helm. +Usage: +{{ include "shuffle.workerInstance.labels" (dict "customLabels" .Values.commonLabels "context" $) -}} +*/}} +{{- define "shuffle.workerInstance.labels" -}} app.kubernetes.io/name: shuffle-worker -helm.sh/chart: {{ include "common.names.chart" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -app.kubernetes.io/managed-by: {{ .Release.Service }} +helm.sh/chart: {{ include "common.names.chart" .context }} +app.kubernetes.io/instance: {{ .context.Release.Name }} +app.kubernetes.io/managed-by: {{ .context.Release.Service }} app.kubernetes.io/part-of: shuffle -{{- with .Chart.AppVersion }} +{{- with .context.Chart.AppVersion }} app.kubernetes.io/version: {{ . | replace "+" "_" | quote }} {{- end -}} +{{- range $key, $value := .customLabels }} +{{ $key }}: {{ $value }} +{{- end }} {{- end -}} {{/* -Return the common labels for app components deployed via helm +Return the common labels for app components deployed via helm. +NOTE: App deployments and services use shuffle.appInstance.labels instead. */}} {{- define "shuffle.app.labels" -}} -app.kubernetes.io/name: shuffle-app -helm.sh/chart: {{ include "common.names.chart" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -app.kubernetes.io/part-of: shuffle -{{- with .Chart.AppVersion }} -app.kubernetes.io/version: {{ . | replace "+" "_" | quote }} +{{- include "common.labels.standard" . }} +app.kubernetes.io/component: app {{- end -}} + +{{/* +Return the sanitized name of a shuffle app. +Usage: +{{ include "shuffle.appInstance.name" $app }} +*/}} +{{- define "shuffle.appInstance.name -}} +{{ .name | replace "_" | "-" | lower }} +{{- end -}} + +{{/* +Return the sanitized name of a shuffle app, including the version of the app. +Usage: +{{ include "shuffle.appInstance.fullname" $app }} +*/}} +{{- define "shuffle.appInstance.fullname -}} +{{ printf "%s-%s" .name .version | replace "_" | "-" | lower }} +{{- end -}} + +{{/* +Return the labels for a shuffle app deployed by helm. +Usage: +{{ include "shuffle.appInstance.labels" (dict "app" $app "customLabels" .Values.commonLabels "context" $) }} +*/}} +{{- define "shuffle.appInstance.labels" -}} +app.kubernetes.io/name: shuffle-app +app.kubernetes.io/instance: {{ include "shuffle.appInstance.fullname" .app }} +helm.sh/chart: {{ include "common.names.chart" .context }} +app.kubernetes.io/instance: {{ .context.Release.Name }} +app.kubernetes.io/managed-by: {{ .context.Release.Service }} +app.kubernetes.io/part-of: shuffle +app.shuffler.io/name: {{ include "shuffle.appInstance.name" .app }} +app.shuffler.io/version: {{ .app.version | quote }} +{{- range $key, $value := .customLabels }} +{{ $key }}: {{ $value }} +{{- end }} {{- end -}} {{/* @@ -112,20 +159,20 @@ app.kubernetes.io/component: orborus {{- end -}} {{/* -Return the match labels for worker components. -These must match the labels of helm-deployed worker components (shuffle.worker.labels), -as well as orborus-deployed worker components (deployk8sworker). +Return the match labels for workers. +These must match the labels of helm-deployed workers (shuffle.workerInstance.labels), +as well as orborus-deployed workers (deployk8sworker). */}} -{{- define "shuffle.worker.matchLabels" -}} +{{- define "shuffle.workerInstance.matchLabels" -}} app.kubernetes.io/name: shuffle-worker {{- end -}} {{/* -Return the match labels for app components. -These must match the labels of helm-deployed app components (shuffle.app.labels), -as well as worker-deployed app deployments (deployK8sApp). +Return the match labels for apps. +These must match the labels of helm-deployed apps (shuffle.appInstance.labels), +as well as worker-deployed apps (deployK8sApp). */}} -{{- define "shuffle.app.matchLabels" -}} +{{- define "shuffle.appInstance.matchLabels" -}} app.kubernetes.io/name: shuffle-app {{- end -}} diff --git a/functions/kubernetes/charts/shuffle/templates/backend/backend-network-policy.yaml b/functions/kubernetes/charts/shuffle/templates/backend/backend-network-policy.yaml index 48010706..516c9f33 100644 --- a/functions/kubernetes/charts/shuffle/templates/backend/backend-network-policy.yaml +++ b/functions/kubernetes/charts/shuffle/templates/backend/backend-network-policy.yaml @@ -51,14 +51,14 @@ spec: matchLabels: kubernetes.io/metadata.name: {{ .Release.Namespace }} podSelector: - matchLabels: {{ include "shuffle.worker.matchLabels" . | nindent 14 }} + matchLabels: {{ include "shuffle.workerInstance.matchLabels" . | nindent 14 }} # Allow traffic from apps - namespaceSelector: matchLabels: kubernetes.io/metadata.name: {{ .Release.Namespace }} podSelector: - matchLabels: {{ include "shuffle.app.matchLabels" . | nindent 14 }} + matchLabels: {{ include "shuffle.appInstance.matchLabels" . | nindent 14 }} {{- end }} {{- if .Values.backend.networkPolicy.extraIngress }} {{- include "common.tplvalues.render" ( dict "value" .Values.backend.networkPolicy.extraIngress "context" $ ) | nindent 4 }} diff --git a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-network-policy.yaml b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-network-policy.yaml index f55bf957..68ea9aa5 100644 --- a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-network-policy.yaml +++ b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-network-policy.yaml @@ -48,7 +48,7 @@ spec: matchLabels: kubernetes.io/metadata.name: {{ .Release.Namespace }} podSelector: - matchLabels: {{ include "shuffle.worker.matchLabels" . | nindent 14 }} + matchLabels: {{ include "shuffle.workerInstance.matchLabels" . | nindent 14 }} {{- if .Values.orborus.networkPolicy.extraEgress }} {{- include "common.tplvalues.render" ( dict "value" .Values.orborus.networkPolicy.extraEgress "context" $ ) | nindent 4 }} {{- end }} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-network-policy.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-network-policy.yaml index a8493002..73c0572e 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-network-policy.yaml +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-network-policy.yaml @@ -11,7 +11,7 @@ metadata: spec: {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.app.podLabels .Values.commonLabels ) "context" . ) }} podSelector: - matchLabels: {{- include "shuffle.app.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 6 }} + matchLabels: {{- include "shuffle.appInstance.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 6 }} policyTypes: - Ingress - Egress @@ -48,7 +48,7 @@ spec: matchLabels: kubernetes.io/metadata.name: {{ .Release.Namespace }} podSelector: - matchLabels: {{ include "shuffle.worker.matchLabels" . | nindent 14 }} + matchLabels: {{ include "shuffle.workerInstance.matchLabels" . | nindent 14 }} {{- if .Values.app.networkPolicy.extraEgress }} {{- include "common.tplvalues.render" ( dict "value" .Values.app.networkPolicy.extraEgress "context" $ ) | nindent 4 }} {{- end }} @@ -64,7 +64,7 @@ spec: matchLabels: kubernetes.io/metadata.name: {{ .Release.Namespace }} podSelector: - matchLabels: {{ include "shuffle.worker.matchLabels" . | nindent 14 }} + matchLabels: {{ include "shuffle.workerInstance.matchLabels" . | nindent 14 }} {{- end }} {{- if .Values.app.networkPolicy.extraIngress }} {{- include "common.tplvalues.render" ( dict "value" .Values.app.networkPolicy.extraIngress "context" $ ) | nindent 4 }} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml index 18641999..1832991c 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml @@ -4,7 +4,7 @@ kind: Deployment metadata: name: {{ template "shuffle.worker.name" . }} namespace: {{ include "common.names.namespace" . | quote }} - labels: {{- include "shuffle.worker.labels" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + labels: {{- include "shuffle.workerInstance.labels" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} {{- if or .Values.worker.deploymentAnnotations .Values.commonAnnotations }} {{- $annotations := include "common.tplvalues.merge" (dict "values" (list .Values.worker.deploymentAnnotations .Values.commonAnnotations) "context" .) }} annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $ ) | nindent 4 }} @@ -18,13 +18,13 @@ spec: {{- end }} {{- $podLabels := include "common.tplvalues.merge" (dict "values" (list .Values.worker.podLabels .Values.commonLabels) "context" .) }} selector: - matchLabels: {{- include "shuffle.worker.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 6 }} + matchLabels: {{- include "shuffle.workerInstance.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 6 }} template: metadata: {{- if .Values.worker.podAnnotations }} annotations: {{- include "common.tplvalues.render" (dict "value" .Values.worker.podAnnotations "context" $) | nindent 8 }} {{- end }} - labels: {{- include "shuffle.worker.labels" ( dict "customLabels" $podLabels "context" $ ) | nindent 8 }} + labels: {{- include "shuffle.workerInstance.labels" ( dict "customLabels" $podLabels "context" $ ) | nindent 8 }} spec: {{- include "shuffle.worker.imagePullSecrets" . | nindent 6 }} serviceAccountName: {{ template "shuffle.worker.serviceAccount.name" . }} @@ -101,6 +101,7 @@ spec: value: {{ .Values.app.exposedContainerPort | quote }} - name: WORKER_HOSTNAME value: {{ include "shuffle.worker.name" }}.{{ .Release.Namespace }}.svc.cluster.local + # TODO: If manageAppDeployments is set, we need to add some variables! {{- if .Values.worker.extraEnvVars }} {{- include "common.tplvalues.render" (dict "value" .Values.worker.extraEnvVars "context" $) | nindent 12 }} {{- end }} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-network-policy.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-network-policy.yaml index 3637d975..259300e1 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-network-policy.yaml +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-network-policy.yaml @@ -11,7 +11,7 @@ metadata: spec: {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.worker.podLabels .Values.commonLabels ) "context" . ) }} podSelector: - matchLabels: {{- include "shuffle.worker.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 6 }} + matchLabels: {{- include "shuffle.workerInstance.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 6 }} policyTypes: - Ingress - Egress @@ -35,7 +35,7 @@ spec: matchLabels: kubernetes.io/metadata.name: {{ .Release.Namespace }} podSelector: - matchLabels: {{ include "shuffle.app.matchLabels" . | nindent 14 }} + matchLabels: {{ include "shuffle.appInstance.matchLabels" . | nindent 14 }} {{- if .Values.worker.networkPolicy.extraEgress }} {{- include "common.tplvalues.render" ( dict "value" .Values.worker.networkPolicy.extraEgress "context" $ ) | nindent 4 }} {{- end }} @@ -57,7 +57,7 @@ spec: matchLabels: kubernetes.io/metadata.name: {{ .Release.Namespace }} podSelector: - matchLabels: {{ include "shuffle.app.matchLabels" . | nindent 14 }} + matchLabels: {{ include "shuffle.appInstance.matchLabels" . | nindent 14 }} {{- end }} {{- if .Values.worker.networkPolicy.extraIngress }} {{- include "common.tplvalues.render" ( dict "value" .Values.worker.networkPolicy.extraIngress "context" $ ) | nindent 4 }} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-svc.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-svc.yaml new file mode 100644 index 00000000..79f5be00 --- /dev/null +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-svc.yaml @@ -0,0 +1,21 @@ +{{- if .Values.worker.enableHelmDeployment }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "shuffle.worker.name" . }} + namespace: {{ include "common.names.namespace" . | quote }} + {{- $serviceLabels := include "common.tplvalues.merge" (dict "values" (list .Values.worker.service.labels .Values.commonLabels) "context" .) }} + labels: {{- include "shuffle.workerInstance.labels" (dict "customLabels" $serviceLabels "context" $) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + ports: + - name: http + port: 33333 + targetPort: 33333 + protocol: TCP + appProtocol: http + selector: {{- include "shuffle.workerInstance.matchLabels" $ | nindent 4 }} +{{- end }} \ No newline at end of file diff --git a/functions/kubernetes/charts/shuffle/values.yaml b/functions/kubernetes/charts/shuffle/values.yaml index 8ee8ee06..4b745749 100644 --- a/functions/kubernetes/charts/shuffle/values.yaml +++ b/functions/kubernetes/charts/shuffle/values.yaml @@ -1111,7 +1111,7 @@ orborus: ## args: [] ## @param orborus.automountServiceAccountToken Mount Service Account token in orborus pods - ## NOTE: orborus requires the service account credentials to be mounted + ## NOTE: orborus requires the service account credentials to be mounted if manageWorkerDeployments is enabled. ## automountServiceAccountToken: true ## @param orborus.hostAliases orborus pods host aliases @@ -1199,7 +1199,7 @@ orborus: ## - name: FOO ## value: "bar" ## - extraEnvVars: + extraEnvVars: # TODO: Make this list empty and move to other values. Also check if these need to be set on worker or apps. - name: SHUFFLE_APP_SDK_TIMEOUT value: "300" - name: SHUFFLE_ORBORUS_EXCUTION_CONCURRENCY @@ -1347,15 +1347,15 @@ orborus: ## extraEgress: [] - ## @param orborus.manageWorkerDeployments Whether workers are deployed and managed by orborus. When disabled, every worker is expected to be already deployed (see worker.enableHelmDeployment). - ## This effectively removes required RBAC permissions from the shuffle-orborus service account to create deployments and services. - ## Orborus might still attempt to create kubernetes objects, resulting in an error. There is currently no way to tell orborus, that it should not manage k8s resources. - ## You likely want to enable worker.enableHelmDeployment when enabling this. - manageWorkerDeployments: true + ## @param orborus.manageWorkerDeployments Whether workers are deployed and managed by orborus. When disabled, every worker is expected to be already deployed (see worker.enableHelmDeployment). + ## This effectively removes required RBAC permissions from the shuffle-orborus service account to create deployments and services. + ## Orborus might still attempt to create kubernetes objects, resulting in an error. There is currently no way to tell orborus, that it should not manage k8s resources. + ## You likely want to enable worker.enableHelmDeployment when enabling this. + manageWorkerDeployments: true ## @section worker Parameters ## -worker: # TODO: Add values that other helm deployed components have, mark helm-only params +worker: ## @param worker.enableHelmDeployment Deploy worker via helm. By default, workers are deployed by Orborus. ## You might want to disable orborus.manageWorkerDeployments when enabling this. enableHelmDeployment: false @@ -1365,13 +1365,90 @@ worker: # TODO: Add values that other helm deployed components have, mark helm-o ## @param worker.image.repository worker image repository ## @param worker.image.tag worker image tag (immutable tags are recommended, defaults to appVersion) ## @param worker.image.digest worker image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended) + ## @param worker.image.pullPolicy worker image pull policy + ## @param worker.image.pullSecrets worker image pull secrets ## image: registry: ghcr.io repository: shuffle/shuffle-worker tag: "" digest: "" - + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/concepts/containers/images/#pre-pulled-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## e.g: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## @param worker.replicaCount Number of worker replicas to deploy + ## + replicaCount: 1 + ## @param worker.extraContainerPorts Optionally specify extra list of additional ports for worker containers + ## e.g: + ## extraContainerPorts: + ## - name: myservice + ## containerPort: 9090 + ## + extraContainerPorts: [] + ## Configure extra options for worker containers' liveness and readiness probes + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes + ## @param worker.livenessProbe.enabled Enable livenessProbe on worker containers + ## @param worker.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe + ## @param worker.livenessProbe.periodSeconds Period seconds for livenessProbe + ## @param worker.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe + ## @param worker.livenessProbe.failureThreshold Failure threshold for livenessProbe + ## @param worker.livenessProbe.successThreshold Success threshold for livenessProbe + ## + livenessProbe: + enabled: false + initialDelaySeconds: 0 + periodSeconds: 15 + timeoutSeconds: 1 + failureThreshold: 4 + successThreshold: 1 + ## @param worker.readinessProbe.enabled Enable readinessProbe on worker containers + ## @param worker.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe + ## @param worker.readinessProbe.periodSeconds Period seconds for readinessProbe + ## @param worker.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe + ## @param worker.readinessProbe.failureThreshold Failure threshold for readinessProbe + ## @param worker.readinessProbe.successThreshold Success threshold for readinessProbe + ## + readinessProbe: + enabled: false + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + ## @param worker.startupProbe.enabled Enable startupProbe on worker containers + ## @param worker.startupProbe.initialDelaySeconds Initial delay seconds for startupProbe + ## @param worker.startupProbe.periodSeconds Period seconds for startupProbe + ## @param worker.startupProbe.timeoutSeconds Timeout seconds for startupProbe + ## @param worker.startupProbe.failureThreshold Failure threshold for startupProbe + ## @param worker.startupProbe.successThreshold Success threshold for startupProbe + ## + startupProbe: + enabled: false + initialDelaySeconds: 0 + periodSeconds: 1 + timeoutSeconds: 1 + failureThreshold: 60 + successThreshold: 1 + ## @param worker.customLivenessProbe Custom livenessProbe that overrides the default one + ## + customLivenessProbe: {} + ## @param worker.customReadinessProbe Custom readinessProbe that overrides the default one + ## + customReadinessProbe: {} + ## @param worker.customStartupProbe Custom startupProbe that overrides the default one + ## + customStartupProbe: {} ## worker resource requests and limits ## ref: http://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ ## @param worker.resourcesPreset Set worker container resources according to one common preset (allowed values: none, nano, small, medium, large, xlarge, 2xlarge). This is ignored if worker.resources is set (worker.resources is recommended for production). @@ -1431,6 +1508,187 @@ worker: # TODO: Add values that other helm deployed components have, mark helm-o drop: ["ALL"] seccompProfile: type: "RuntimeDefault" + ## @param worker.command Override default worker container command (useful when using custom images) + ## + command: [] + ## @param worker.args Override default worker container args (useful when using custom images) + ## + args: [] + ## @param worker.automountServiceAccountToken Mount Service Account token in worker pods + ## NOTE: worker requires the service account credentials to be mounted if manageAppDeployments is enabled. + ## + automountServiceAccountToken: true + ## @param worker.hostAliases worker pods host aliases + ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ + ## + hostAliases: [] + ## @param worker.deploymentAnnotations Annotations for worker deployment + ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ + ## + deploymentAnnotations: {} + ## @param worker.podLabels Extra labels for worker pods + ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + ## + podLabels: {} + ## @param worker.podAnnotations Annotations for worker pods + ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ + ## + podAnnotations: {} + ## @param worker.podAffinityPreset Pod affinity preset. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard` + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity + ## + podAffinityPreset: "" + ## @param worker.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard` + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity + ## + podAntiAffinityPreset: soft + ## Node worker.affinity preset + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity + ## + nodeAffinityPreset: + ## @param worker.nodeAffinityPreset.type Node affinity preset type. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard` + ## + type: "" + ## @param worker.nodeAffinityPreset.key Node label key to match. Ignored if `worker.affinity` is set + ## + key: "" + ## @param worker.nodeAffinityPreset.values Node label values to match. Ignored if `worker.affinity` is set + ## E.g. + ## values: + ## - e2e-az1 + ## - e2e-az2 + ## + values: [] + ## @param worker.affinity Affinity for worker pods assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## NOTE: `worker.podAffinityPreset`, `worker.podAntiAffinityPreset`, and `worker.nodeAffinityPreset` will be ignored when it's set + ## + affinity: {} + ## @param worker.nodeSelector Node labels for worker pods assignment + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/ + ## + nodeSelector: {} + ## @param worker.tolerations Tolerations for worker pods assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + ## @param worker.updateStrategy.type worker deployment strategy type + ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy + ## + updateStrategy: + ## Can be set to RollingUpdate or Recreate + ## + type: RollingUpdate + ## @param worker.priorityClassName worker pods' priorityClassName + ## + priorityClassName: "" + ## @param worker.topologySpreadConstraints Topology Spread Constraints for worker pod assignment spread across your cluster among failure-domains + ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods + ## + topologySpreadConstraints: [] + ## @param worker.schedulerName Name of the k8s scheduler (other than default) for worker pods + ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ + ## + schedulerName: "" + ## @param worker.terminationGracePeriodSeconds Seconds worker pods need to terminate gracefully + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods + ## + terminationGracePeriodSeconds: "" + ## @param worker.lifecycleHooks for worker containers to automate configuration before or after startup + ## + lifecycleHooks: {} + ## @param worker.extraEnvVars Array with extra environment variables to add to worker containers + ## e.g: + ## extraEnvVars: + ## - name: FOO + ## value: "bar" + ## + extraEnvVars: [] + ## @param worker.extraEnvVarsCM Name of existing ConfigMap containing extra env vars for worker containers + ## + extraEnvVarsCM: "" + ## @param worker.extraEnvVarsSecret Name of existing Secret containing extra env vars for worker containers + ## + extraEnvVarsSecret: "" + ## @param worker.extraVolumes Optionally specify extra list of additional volumes for the worker pods + ## + extraVolumes: [] + ## @param worker.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the worker containers + ## + extraVolumeMounts: [] + ## @param worker.sidecars Add additional sidecar containers to the worker pods + ## e.g: + ## sidecars: + ## - name: your-image-name + ## image: your-image + ## imagePullPolicy: Always + ## ports: + ## - name: portname + ## containerPort: 1234 + ## + sidecars: [] + ## @param worker.initContainers Add additional init containers to the worker pods + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ + ## e.g: + ## initContainers: + ## - name: your-image-name + ## image: your-image + ## imagePullPolicy: Always + ## command: ['sh', '-c', 'echo "hello world"'] + ## + initContainers: [] + ## Pod Disruption Budget configuration + ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb + ## @param worker.pdb.create Enable/disable a Pod Disruption Budget creation + ## @param worker.pdb.minAvailable Minimum number/percentage of pods that should remain scheduled + ## @param worker.pdb.maxUnavailable Maximum number/percentage of pods that may be made unavailable. Defaults to `1` if both `worker.pdb.minAvailable` and `worker.pdb.maxUnavailable` are empty. + ## + pdb: + create: true + minAvailable: "" + maxUnavailable: "" + ## Autoscaling configuration + ## ref: https://kubernetes.io/docs/concepts/workloads/autoscaling/ + ## + autoscaling: + ## @param worker.autoscaling.vpa.enabled Enable VPA for worker pods + ## @param worker.autoscaling.vpa.annotations Annotations for VPA resource + ## @param worker.autoscaling.vpa.controlledResources VPA List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory + ## @param worker.autoscaling.vpa.maxAllowed VPA Max allowed resources for the pod + ## @param worker.autoscaling.vpa.minAllowed VPA Min allowed resources for the pod + ## + vpa: + enabled: false + annotations: {} + controlledResources: [] + maxAllowed: {} + minAllowed: {} + ## @param worker.autoscaling.vpa.updatePolicy.updateMode Autoscaling update policy + ## Specifies whether recommended updates are applied when a Pod is started and whether recommended updates are applied during the life of a Pod + ## Possible values are "Off", "Initial", "Recreate", and "Auto". + ## + updatePolicy: + updateMode: Auto + ## @param worker.autoscaling.hpa.enabled Enable HPA for worker pods + ## @param worker.autoscaling.hpa.minReplicas Minimum number of replicas + ## @param worker.autoscaling.hpa.maxReplicas Maximum number of replicas + ## @param worker.autoscaling.hpa.targetCPU Target CPU utilization percentage + ## @param worker.autoscaling.hpa.targetMemory Target Memory utilization percentage + ## + hpa: + enabled: false + minReplicas: "" + maxReplicas: "" + targetCPU: "" + targetMemory: "" + + ## Service configuration + ## + service: + ## @param worker.service.labels Extra labels for worker service + ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + ## + labels: {} ## ServiceAccount configuration ## From 3d3885376f77aa1d9dd712527ba1dacc35de1f2e Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Mon, 1 Dec 2025 16:30:57 +0100 Subject: [PATCH 05/41] remove shuffle-backend configmap Signed-off-by: Pascal Sthamer --- .../shuffle/templates/backend/_helpers.tpl | 32 +++++++++++++++++++ .../templates/backend/backend-cm-env.yaml | 32 ------------------- .../templates/backend/backend-dpl.yaml | 14 +++----- 3 files changed, 36 insertions(+), 42 deletions(-) create mode 100644 functions/kubernetes/charts/shuffle/templates/backend/_helpers.tpl delete mode 100644 functions/kubernetes/charts/shuffle/templates/backend/backend-cm-env.yaml diff --git a/functions/kubernetes/charts/shuffle/templates/backend/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/backend/_helpers.tpl new file mode 100644 index 00000000..94836dbe --- /dev/null +++ b/functions/kubernetes/charts/shuffle/templates/backend/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* +Return the environment variables of shuffle-backend in the format +KEY: VALUE +*/}} +{{- define "shuffle.backend.env" -}} +RUNNING_MODE: kubernetes +IS_KUBERNETES: "true" +SHUFFLE_APP_HOTLOAD_FOLDER: /shuffle-apps +SHUFFLE_FILE_LOCATION: /shuffle-files +BACKEND_PORT: "{{ .Values.backend.containerPorts.http }}" +{{- if .Values.shuffle.baseUrl }} +BASE_URL: "{{ .Values.shuffle.baseUrl }}" +SSO_REDIRECT_URL: "{{ .Values.shuffle.baseUrl }}" +{{- else }} +BASE_URL: "http://{{ include "shuffle.backend.name" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.backend.containerPorts.http }}" +{{- end }} +ORG_ID: "{{ .Values.shuffle.org }}" +SHUFFLE_APP_DOWNLOAD_LOCATION: "{{ .Values.backend.apps.downloadLocation }}" +SHUFFLE_DOWNLOAD_AUTH_BRANCH: "{{ .Values.backend.apps.downloadBranch }}" +SHUFFLE_APP_FORCE_UPDATE: "{{ .Values.backend.apps.forceUpdate }}" +SHUFFLE_CHAT_DISABLED: "true" +# Sets backend_url parameter for workflow execution to the cluster-internal shuffle-backend address +SHUFFLE_CLOUDRUN_URL: "http://{{ include "shuffle.backend.name" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.backend.containerPorts.http }}" +SHUFFLE_OPENSEARCH_URL: {{ include "common.tplvalues.render" (dict "value" .Values.backend.openSearch.url "context" $) }} +SHUFFLE_OPENSEARCH_USERNAME: "{{ .Values.backend.openSearch.username }}" +SHUFFLE_OPENSEARCH_CERTIFICATE_FILE: "{{ .Values.backend.openSearch.certificateFile }}" +SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY: "{{ .Values.backend.openSearch.skipSSLVerify }}" +SHUFFLE_OPENSEARCH_INDEX_PREFIX: "{{ .Values.backend.openSearch.indexPrefix }}" +SHUFFLE_RERUN_SCHEDULE: "{{ .Values.backend.cleanupSchedule }}" +TZ: "{{ .Values.shuffle.timezone }}" +REGISTRY_URL: "{{ .Values.shuffle.appRegistry }}" # Used by app builder +{{- end -}} \ No newline at end of file diff --git a/functions/kubernetes/charts/shuffle/templates/backend/backend-cm-env.yaml b/functions/kubernetes/charts/shuffle/templates/backend/backend-cm-env.yaml deleted file mode 100644 index dbbdff94..00000000 --- a/functions/kubernetes/charts/shuffle/templates/backend/backend-cm-env.yaml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "shuffle.backend.name" . }}-env - namespace: {{ include "common.names.namespace" . | quote }} - labels: {{- include "shuffle.backend.labels" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} - {{- if .Values.commonAnnotations }} - annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} - {{- end }} -data: - BACKEND_PORT: "{{ .Values.backend.containerPorts.http }}" - {{- if .Values.shuffle.baseUrl }} - BASE_URL: "{{ .Values.shuffle.baseUrl }}" - SSO_REDIRECT_URL: "{{ .Values.shuffle.baseUrl }}" - {{- else }} - BASE_URL: "http://{{ include "shuffle.backend.name" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.backend.containerPorts.http }}" - {{- end }} - ORG_ID: "{{ .Values.shuffle.org }}" - SHUFFLE_APP_DOWNLOAD_LOCATION: "{{ .Values.backend.apps.downloadLocation }}" - SHUFFLE_DOWNLOAD_AUTH_BRANCH: "{{ .Values.backend.apps.downloadBranch }}" - SHUFFLE_APP_FORCE_UPDATE: "{{ .Values.backend.apps.forceUpdate }}" - SHUFFLE_CHAT_DISABLED: "true" - # Sets backend_url parameter for workflow execution to the cluster-internal shuffle-backend address - SHUFFLE_CLOUDRUN_URL: "http://{{ include "shuffle.backend.name" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.backend.containerPorts.http }}" - SHUFFLE_OPENSEARCH_URL: {{ include "common.tplvalues.render" (dict "value" .Values.backend.openSearch.url "context" $) }} - SHUFFLE_OPENSEARCH_USERNAME: "{{ .Values.backend.openSearch.username }}" - SHUFFLE_OPENSEARCH_CERTIFICATE_FILE: "{{ .Values.backend.openSearch.certificateFile }}" - SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY: "{{ .Values.backend.openSearch.skipSSLVerify }}" - SHUFFLE_OPENSEARCH_INDEX_PREFIX: "{{ .Values.backend.openSearch.indexPrefix }}" - SHUFFLE_RERUN_SCHEDULE: "{{ .Values.backend.cleanupSchedule }}" - TZ: "{{ .Values.shuffle.timezone }}" - REGISTRY_URL: "{{ .Values.shuffle.appRegistry }}" diff --git a/functions/kubernetes/charts/shuffle/templates/backend/backend-dpl.yaml b/functions/kubernetes/charts/shuffle/templates/backend/backend-dpl.yaml index b1cc606c..046f55c6 100644 --- a/functions/kubernetes/charts/shuffle/templates/backend/backend-dpl.yaml +++ b/functions/kubernetes/charts/shuffle/templates/backend/backend-dpl.yaml @@ -115,20 +115,14 @@ spec: args: {{- include "common.tplvalues.render" (dict "value" .Values.backend.args "context" $) | nindent 12 }} {{- end }} env: - - name: RUNNING_MODE - value: kubernetes - - name: IS_KUBERNETES - value: "true" - - name: SHUFFLE_APP_HOTLOAD_FOLDER - value: /shuffle-apps - - name: SHUFFLE_FILE_LOCATION - value: /shuffle-files + {{- $env := include "shuffle.backend.env" . | fromYaml }} + {{- range $key, $val := range $env }} + - name: {{ $key | quote }} + value: {{ $val | quote }} {{- if .Values.backend.extraEnvVars }} {{- include "common.tplvalues.render" (dict "value" .Values.backend.extraEnvVars "context" $) | nindent 12 }} {{- end }} envFrom: - - configMapRef: - name: {{ include "shuffle.backend.name" . }}-env {{- if .Values.backend.extraEnvVarsCM }} - configMapRef: name: {{ include "common.tplvalues.render" (dict "value" .Values.backend.extraEnvVarsCM "context" $) }} From 3cdc77d802fd386cf6b3843abce062dd1214c691 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Mon, 1 Dec 2025 16:41:48 +0100 Subject: [PATCH 06/41] split helpers Signed-off-by: Pascal Sthamer --- .../charts/shuffle/templates/_helpers.tpl | 400 +----------------- .../shuffle/templates/backend/_helpers.tpl | 88 +++- .../shuffle/templates/frontend/_helpers.tpl | 77 ++++ .../shuffle/templates/orborus/_helpers.tpl | 77 ++++ .../templates/shuffle-app/_helpers.tpl | 102 +++++ .../templates/shuffle-worker/_helpers.tpl | 91 ++++ 6 files changed, 435 insertions(+), 400 deletions(-) create mode 100644 functions/kubernetes/charts/shuffle/templates/frontend/_helpers.tpl create mode 100644 functions/kubernetes/charts/shuffle/templates/orborus/_helpers.tpl create mode 100644 functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl create mode 100644 functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl diff --git a/functions/kubernetes/charts/shuffle/templates/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/_helpers.tpl index 061ea4f4..52628d2b 100644 --- a/functions/kubernetes/charts/shuffle/templates/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/_helpers.tpl @@ -1,181 +1,3 @@ -{{/* -Return the common name for backend componentes -*/}} -{{- define "shuffle.backend.name" -}} - {{- printf "%s-backend" (include "common.names.fullname" .) | trunc 63 -}} -{{- end -}} - -{{/* -Return the common name for frontend components -*/}} -{{- define "shuffle.frontend.name" -}} - {{- printf "%s-frontend" (include "common.names.fullname" .) | trunc 63 -}} -{{- end -}} - -{{/* -Return the common name for orborus components -*/}} -{{- define "shuffle.orborus.name" -}} - {{- printf "%s-orborus" (include "common.names.fullname" .) | trunc 63 -}} -{{- end -}} - -{{/* -Return the common name for worker components -*/}} -{{- define "shuffle.worker.name" -}} - {{- printf "%s-worker" (include "common.names.fullname" .) | trunc 63 -}} -{{- end -}} - -{{/* -Return the common name for app components. -*/}} -{{- define "shuffle.app.name" -}} - {{- printf "%s-app" (include "common.names.fullname" .) | trunc 63 -}} -{{- end -}} - -{{/* -Return the common labels for backend components -The shuffle app builder requires the io.kompose.service=backend label to be set on the backend pod. -*/}} -{{- define "shuffle.backend.labels" -}} -{{- include "common.labels.standard" . }} -app.kubernetes.io/component: backend -io.kompose.service: backend -{{- end -}} - -{{/* -Return the common labels for frontend components -*/}} -{{- define "shuffle.frontend.labels" -}} -{{- include "common.labels.standard" . }} -app.kubernetes.io/component: frontend -{{- end -}} - -{{/* -Return the common labels for orborus components -*/}} -{{- define "shuffle.orborus.labels" -}} -{{- include "common.labels.standard" . }} -app.kubernetes.io/component: orborus -{{- end -}} - -{{/* -Return the common labels for worker components deployed via helm. -NOTE: Worker deployments and services use shuffle.workerInstance.labels instead. -*/}} -{{- define "shuffle.worker.labels" -}} -{{- include "common.labels.standard" . }} -app.kubernetes.io/component: worker -{{- end -}} - - -{{/* -Return the labels for a specific worker instance deployed via helm. -Usage: -{{ include "shuffle.workerInstance.labels" (dict "customLabels" .Values.commonLabels "context" $) -}} -*/}} -{{- define "shuffle.workerInstance.labels" -}} -app.kubernetes.io/name: shuffle-worker -helm.sh/chart: {{ include "common.names.chart" .context }} -app.kubernetes.io/instance: {{ .context.Release.Name }} -app.kubernetes.io/managed-by: {{ .context.Release.Service }} -app.kubernetes.io/part-of: shuffle -{{- with .context.Chart.AppVersion }} -app.kubernetes.io/version: {{ . | replace "+" "_" | quote }} -{{- end -}} -{{- range $key, $value := .customLabels }} -{{ $key }}: {{ $value }} -{{- end }} -{{- end -}} - -{{/* -Return the common labels for app components deployed via helm. -NOTE: App deployments and services use shuffle.appInstance.labels instead. -*/}} -{{- define "shuffle.app.labels" -}} -{{- include "common.labels.standard" . }} -app.kubernetes.io/component: app -{{- end -}} - -{{/* -Return the sanitized name of a shuffle app. -Usage: -{{ include "shuffle.appInstance.name" $app }} -*/}} -{{- define "shuffle.appInstance.name -}} -{{ .name | replace "_" | "-" | lower }} -{{- end -}} - -{{/* -Return the sanitized name of a shuffle app, including the version of the app. -Usage: -{{ include "shuffle.appInstance.fullname" $app }} -*/}} -{{- define "shuffle.appInstance.fullname -}} -{{ printf "%s-%s" .name .version | replace "_" | "-" | lower }} -{{- end -}} - -{{/* -Return the labels for a shuffle app deployed by helm. -Usage: -{{ include "shuffle.appInstance.labels" (dict "app" $app "customLabels" .Values.commonLabels "context" $) }} -*/}} -{{- define "shuffle.appInstance.labels" -}} -app.kubernetes.io/name: shuffle-app -app.kubernetes.io/instance: {{ include "shuffle.appInstance.fullname" .app }} -helm.sh/chart: {{ include "common.names.chart" .context }} -app.kubernetes.io/instance: {{ .context.Release.Name }} -app.kubernetes.io/managed-by: {{ .context.Release.Service }} -app.kubernetes.io/part-of: shuffle -app.shuffler.io/name: {{ include "shuffle.appInstance.name" .app }} -app.shuffler.io/version: {{ .app.version | quote }} -{{- range $key, $value := .customLabels }} -{{ $key }}: {{ $value }} -{{- end }} -{{- end -}} - -{{/* -Return the match labels for backend components -*/}} -{{- define "shuffle.backend.matchLabels" -}} -{{- include "common.labels.matchLabels" . }} -app.kubernetes.io/component: backend -{{- end -}} - -{{/* -Return the match labels for frontend components -*/}} -{{- define "shuffle.frontend.matchLabels" -}} -{{- include "common.labels.matchLabels" . }} -app.kubernetes.io/component: frontend -{{- end -}} - -{{/* -Return the match labels for orborus components -*/}} -{{- define "shuffle.orborus.matchLabels" -}} -{{- include "common.labels.matchLabels" . }} -app.kubernetes.io/component: orborus -{{- end -}} - -{{/* -Return the match labels for workers. -These must match the labels of helm-deployed workers (shuffle.workerInstance.labels), -as well as orborus-deployed workers (deployk8sworker). -*/}} -{{- define "shuffle.workerInstance.matchLabels" -}} -app.kubernetes.io/name: shuffle-worker -{{- end -}} - -{{/* -Return the match labels for apps. -These must match the labels of helm-deployed apps (shuffle.appInstance.labels), -as well as worker-deployed apps (deployK8sApp). -*/}} -{{- define "shuffle.appInstance.matchLabels" -}} -app.kubernetes.io/name: shuffle-app -{{- end -}} - {{/* Return the proper image name (for the init container volume-permissions image) */}} @@ -183,87 +5,17 @@ Return the proper image name (for the init container volume-permissions image) {{- include "common.images.image" ( dict "imageRoot" .Values.volumePermissions.image "global" .Values.global "chart" .Chart ) -}} {{- end -}} -{{/* -Return the proper Shuffle backend image name -*/}} -{{- define "shuffle.backend.image" -}} -{{- include "common.images.image" ( dict "imageRoot" .Values.backend.image "global" .Values.global "chart" .Chart ) -}} -{{- end -}} -{{/* -Return the proper Docker Image Registry Secret Names for the backend pod -*/}} -{{- define "shuffle.backend.imagePullSecrets" -}} -{{- include "common.images.renderPullSecrets" (dict "images" (list .Values.backend.image) "context" $) -}} -{{- end -}} -{{/* -Return the proper Shuffle frontend image name -*/}} -{{- define "shuffle.frontend.image" -}} -{{- include "common.images.image" ( dict "imageRoot" .Values.frontend.image "global" .Values.global "chart" .Chart ) -}} -{{- end -}} -{{/* -Return the proper Docker Image Registry Secret Names for the frontend pod -*/}} -{{- define "shuffle.frontend.imagePullSecrets" -}} -{{- include "common.images.renderPullSecrets" (dict "images" (list .Values.frontend.image) "context" $) -}} -{{- end -}} -{{/* -Return the proper Shuffle orborus image name -*/}} -{{- define "shuffle.orborus.image" -}} -{{- include "common.images.image" ( dict "imageRoot" .Values.orborus.image "global" .Values.global "chart" .Chart ) -}} -{{- end -}} -{{/* -Return the proper Docker Image Registry Secret Names for the orborus pod -*/}} -{{- define "shuffle.orborus.imagePullSecrets" -}} -{{- include "common.images.renderPullSecrets" (dict "images" (list .Values.orborus.image) "context" $) -}} -{{- end -}} -{{/* -Return the proper Shuffle worker image name -*/}} -{{- define "shuffle.worker.image" -}} -{{- include "common.images.image" ( dict "imageRoot" .Values.worker.image "global" .Values.global "chart" .Chart ) -}} -{{- end -}} -{{/* -Create the name of the service account to use for the Shuffle backend -*/}} -{{- define "shuffle.backend.serviceAccount.name" -}} -{{- if .Values.backend.serviceAccount.create -}} - {{ default (include "shuffle.backend.name" .) .Values.backend.serviceAccount.name | trunc 63 | trimSuffix "-" }} -{{- else -}} - {{ default "default" .Values.backend.serviceAccount.name }} -{{- end -}} -{{- end -}} -{{/* -Return the proper Docker Image Registry Secret Names for the backend service account -*/}} -{{- define "shuffle.backend.serviceAccount.imagePullSecrets" -}} -{{- $pullSecrets := list }} -{{- range .Values.global.imagePullSecrets -}} - {{- if kindIs "map" . -}} - {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" .)) -}} - {{- else -}} - {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" .)) -}} - {{- end -}} -{{- end -}} -{{- range .Values.backend.serviceAccount.imagePullSecrets -}} - {{- if kindIs "map" . -}} - {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" .)) -}} - {{- else -}} - {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" .)) -}} - {{- end -}} -{{- end -}} + {{- if (not (empty $pullSecrets)) -}} imagePullSecrets: @@ -273,22 +25,7 @@ imagePullSecrets: {{- end }} {{- end -}} -{{/* -Create the name of the service account to use for the Shuffle frontend -*/}} -{{- define "shuffle.frontend.serviceAccount.name" -}} -{{- if .Values.frontend.serviceAccount.create -}} - {{ default (include "shuffle.frontend.name" .) .Values.frontend.serviceAccount.name | trunc 63 | trimSuffix "-" }} -{{- else -}} - {{ default "default" .Values.frontend.serviceAccount.name }} -{{- end -}} -{{- end -}} -{{/* -Return the proper Docker Image Registry Secret Names for the frontend service account -*/}} -{{- define "shuffle.frontend.serviceAccount.imagePullSecrets" -}} -{{- $pullSecrets := list }} {{- range .Values.global.imagePullSecrets -}} {{- if kindIs "map" . -}} @@ -298,13 +35,7 @@ Return the proper Docker Image Registry Secret Names for the frontend service ac {{- end -}} {{- end -}} -{{- range .Values.frontend.serviceAccount.imagePullSecrets -}} - {{- if kindIs "map" . -}} - {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" .)) -}} - {{- else -}} - {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" .)) -}} - {{- end -}} -{{- end -}} + {{- if (not (empty $pullSecrets)) -}} imagePullSecrets: @@ -313,130 +44,3 @@ imagePullSecrets: {{- end }} {{- end }} {{- end -}} - -{{/* -Create the name of the service account to use for Shuffle orborus -*/}} -{{- define "shuffle.orborus.serviceAccount.name" -}} -{{- if .Values.orborus.serviceAccount.create -}} - {{ default (include "shuffle.orborus.name" .) .Values.orborus.serviceAccount.name | trunc 63 | trimSuffix "-" }} -{{- else -}} - {{ default "default" .Values.orborus.serviceAccount.name }} -{{- end -}} -{{- end -}} - -{{/* -Return the proper Docker Image Registry Secret Names for the orborus service account -*/}} -{{- define "shuffle.orborus.serviceAccount.imagePullSecrets" -}} -{{- $pullSecrets := list }} - -{{- range .Values.global.imagePullSecrets -}} - {{- if kindIs "map" . -}} - {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" .)) -}} - {{- else -}} - {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" .)) -}} - {{- end -}} -{{- end -}} - -{{- range .Values.orborus.serviceAccount.imagePullSecrets -}} - {{- if kindIs "map" . -}} - {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" .)) -}} - {{- else -}} - {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" .)) -}} - {{- end -}} -{{- end -}} - -{{- if (not (empty $pullSecrets)) -}} -imagePullSecrets: -{{- range $pullSecrets | uniq }} - - name: {{ . }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* -Create the name of the service account to use for Shuffle workers -*/}} -{{- define "shuffle.worker.serviceAccount.name" -}} -{{- if .Values.worker.serviceAccount.create -}} - {{ default (include "shuffle.worker.name" .) .Values.worker.serviceAccount.name | trunc 63 | trimSuffix "-" }} -{{- else -}} - {{ default "default" .Values.worker.serviceAccount.name }} -{{- end -}} -{{- end -}} - -{{/* -Return the proper Docker Image Registry Secret Names for the worker service account -*/}} -{{- define "shuffle.worker.serviceAccount.imagePullSecrets" -}} -{{- $pullSecrets := list }} - -{{- range .Values.global.imagePullSecrets -}} - {{- if kindIs "map" . -}} - {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" .)) -}} - {{- else -}} - {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" .)) -}} - {{- end -}} -{{- end -}} - -{{- range .Values.worker.serviceAccount.imagePullSecrets -}} - {{- if kindIs "map" . -}} - {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" .)) -}} - {{- else -}} - {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" .)) -}} - {{- end -}} -{{- end -}} - -{{- if (not (empty $pullSecrets)) -}} -imagePullSecrets: -{{- range $pullSecrets | uniq }} - - name: {{ . }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* -Create the name of the service account to use for Shuffle apps -*/}} -{{- define "shuffle.app.serviceAccount.name" -}} -{{- if .Values.app.serviceAccount.create -}} - {{ default (include "shuffle.app.name" .) .Values.app.serviceAccount.name | trunc 63 | trimSuffix "-" }} -{{- else -}} - {{ default "default" .Values.app.serviceAccount.name }} -{{- end -}} -{{- end -}} - -{{/* -Return the proper Docker Image Registry Secret Names for the app service account -*/}} -{{- define "shuffle.app.serviceAccount.imagePullSecrets" -}} -{{- $pullSecrets := list }} - -{{- range .Values.global.imagePullSecrets -}} - {{- if kindIs "map" . -}} - {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" .)) -}} - {{- else -}} - {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" .)) -}} - {{- end -}} -{{- end -}} - -{{- range .Values.app.serviceAccount.imagePullSecrets -}} - {{- if kindIs "map" . -}} - {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" .)) -}} - {{- else -}} - {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" .)) -}} - {{- end -}} -{{- end -}} - -{{- if (not (empty $pullSecrets)) -}} -imagePullSecrets: -{{- range $pullSecrets | uniq }} - - name: {{ . }} -{{- end }} -{{- end }} -{{- end -}} - -{{- define "shuffle.backend.baseUrl -}} -http://{{ include "shuffle.backend.name" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.backend.containerPorts.http }} -{{- end - }} \ No newline at end of file diff --git a/functions/kubernetes/charts/shuffle/templates/backend/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/backend/_helpers.tpl index 94836dbe..07a8516f 100644 --- a/functions/kubernetes/charts/shuffle/templates/backend/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/backend/_helpers.tpl @@ -1,3 +1,87 @@ +{{/* +Return the common name for backend componentes +*/}} +{{- define "shuffle.backend.name" -}} + {{- printf "%s-backend" (include "common.names.fullname" .) | trunc 63 -}} +{{- end -}} + +{{/* +Return the common labels for backend components +The shuffle app builder requires the io.kompose.service=backend label to be set on the backend pod. +*/}} +{{- define "shuffle.backend.labels" -}} +{{- include "common.labels.standard" . }} +app.kubernetes.io/component: backend +io.kompose.service: backend +{{- end -}} + +{{/* +Return the match labels for backend components +*/}} +{{- define "shuffle.backend.matchLabels" -}} +{{- include "common.labels.matchLabels" . }} +app.kubernetes.io/component: backend +{{- end -}} + +{{/* +Return the proper Shuffle backend image name +*/}} +{{- define "shuffle.backend.image" -}} +{{- include "common.images.image" ( dict "imageRoot" .Values.backend.image "global" .Values.global "chart" .Chart ) -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names for the backend pod +*/}} +{{- define "shuffle.backend.imagePullSecrets" -}} +{{- include "common.images.renderPullSecrets" (dict "images" (list .Values.backend.image) "context" $) -}} +{{- end -}} + +{{/* +Create the name of the service account to use for the Shuffle backend +*/}} +{{- define "shuffle.backend.serviceAccount.name" -}} +{{- if .Values.backend.serviceAccount.create -}} + {{ default (include "shuffle.backend.name" .) .Values.backend.serviceAccount.name | trunc 63 | trimSuffix "-" }} +{{- else -}} + {{ default "default" .Values.backend.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names for the backend service account +*/}} +{{- define "shuffle.backend.serviceAccount.imagePullSecrets" -}} +{{- $pullSecrets := list }} + +{{- range .Values.global.imagePullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" .)) -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" .)) -}} + {{- end -}} +{{- end -}} + +{{- range .Values.backend.serviceAccount.imagePullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" .)) -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" .)) -}} + {{- end -}} +{{- end -}} + +{{- if (not (empty $pullSecrets)) -}} +imagePullSecrets: +{{- range $pullSecrets | uniq }} + - name: {{ . }} +{{- end }} +{{- end }} +{{- end -}} + +{{- define "shuffle.backend.baseUrl -}} +http://{{ include "shuffle.backend.name" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.backend.containerPorts.http }} +{{- end - }} + {{/* Return the environment variables of shuffle-backend in the format KEY: VALUE @@ -12,7 +96,7 @@ BACKEND_PORT: "{{ .Values.backend.containerPorts.http }}" BASE_URL: "{{ .Values.shuffle.baseUrl }}" SSO_REDIRECT_URL: "{{ .Values.shuffle.baseUrl }}" {{- else }} -BASE_URL: "http://{{ include "shuffle.backend.name" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.backend.containerPorts.http }}" +BASE_URL: "{{ include "shuffle.backend.baseUrl .}}" {{- end }} ORG_ID: "{{ .Values.shuffle.org }}" SHUFFLE_APP_DOWNLOAD_LOCATION: "{{ .Values.backend.apps.downloadLocation }}" @@ -20,7 +104,7 @@ SHUFFLE_DOWNLOAD_AUTH_BRANCH: "{{ .Values.backend.apps.downloadBranch }}" SHUFFLE_APP_FORCE_UPDATE: "{{ .Values.backend.apps.forceUpdate }}" SHUFFLE_CHAT_DISABLED: "true" # Sets backend_url parameter for workflow execution to the cluster-internal shuffle-backend address -SHUFFLE_CLOUDRUN_URL: "http://{{ include "shuffle.backend.name" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.backend.containerPorts.http }}" +SHUFFLE_CLOUDRUN_URL: "{{ include "shuffle.backend.baseUrl .}}" SHUFFLE_OPENSEARCH_URL: {{ include "common.tplvalues.render" (dict "value" .Values.backend.openSearch.url "context" $) }} SHUFFLE_OPENSEARCH_USERNAME: "{{ .Values.backend.openSearch.username }}" SHUFFLE_OPENSEARCH_CERTIFICATE_FILE: "{{ .Values.backend.openSearch.certificateFile }}" diff --git a/functions/kubernetes/charts/shuffle/templates/frontend/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/frontend/_helpers.tpl new file mode 100644 index 00000000..47d76bfd --- /dev/null +++ b/functions/kubernetes/charts/shuffle/templates/frontend/_helpers.tpl @@ -0,0 +1,77 @@ +{{/* +Return the common name for frontend components +*/}} +{{- define "shuffle.frontend.name" -}} + {{- printf "%s-frontend" (include "common.names.fullname" .) | trunc 63 -}} +{{- end -}} + +{{/* +Return the common labels for frontend components +*/}} +{{- define "shuffle.frontend.labels" -}} +{{- include "common.labels.standard" . }} +app.kubernetes.io/component: frontend +{{- end -}} + +{{/* +Return the match labels for frontend components +*/}} +{{- define "shuffle.frontend.matchLabels" -}} +{{- include "common.labels.matchLabels" . }} +app.kubernetes.io/component: frontend +{{- end -}} + +{{/* +Return the proper Shuffle frontend image name +*/}} +{{- define "shuffle.frontend.image" -}} +{{- include "common.images.image" ( dict "imageRoot" .Values.frontend.image "global" .Values.global "chart" .Chart ) -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names for the frontend pod +*/}} +{{- define "shuffle.frontend.imagePullSecrets" -}} +{{- include "common.images.renderPullSecrets" (dict "images" (list .Values.frontend.image) "context" $) -}} +{{- end -}} + +{{/* +Create the name of the service account to use for the Shuffle frontend +*/}} +{{- define "shuffle.frontend.serviceAccount.name" -}} +{{- if .Values.frontend.serviceAccount.create -}} + {{ default (include "shuffle.frontend.name" .) .Values.frontend.serviceAccount.name | trunc 63 | trimSuffix "-" }} +{{- else -}} + {{ default "default" .Values.frontend.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names for the frontend service account +*/}} +{{- define "shuffle.frontend.serviceAccount.imagePullSecrets" -}} +{{- $pullSecrets := list }} + +{{- range .Values.global.imagePullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" .)) -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" .)) -}} + {{- end -}} +{{- end -}} + +{{- range .Values.frontend.serviceAccount.imagePullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" .)) -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" .)) -}} + {{- end -}} +{{- end -}} + +{{- if (not (empty $pullSecrets)) -}} +imagePullSecrets: +{{- range $pullSecrets | uniq }} + - name: {{ . }} +{{- end }} +{{- end }} +{{- end -}} \ No newline at end of file diff --git a/functions/kubernetes/charts/shuffle/templates/orborus/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/orborus/_helpers.tpl new file mode 100644 index 00000000..3b3249b0 --- /dev/null +++ b/functions/kubernetes/charts/shuffle/templates/orborus/_helpers.tpl @@ -0,0 +1,77 @@ +{{/* +Return the common name for orborus components +*/}} +{{- define "shuffle.orborus.name" -}} + {{- printf "%s-orborus" (include "common.names.fullname" .) | trunc 63 -}} +{{- end -}} + +{{/* +Return the common labels for orborus components +*/}} +{{- define "shuffle.orborus.labels" -}} +{{- include "common.labels.standard" . }} +app.kubernetes.io/component: orborus +{{- end -}} + +{{/* +Return the match labels for orborus components +*/}} +{{- define "shuffle.orborus.matchLabels" -}} +{{- include "common.labels.matchLabels" . }} +app.kubernetes.io/component: orborus +{{- end -}} + +{{/* +Return the proper Shuffle orborus image name +*/}} +{{- define "shuffle.orborus.image" -}} +{{- include "common.images.image" ( dict "imageRoot" .Values.orborus.image "global" .Values.global "chart" .Chart ) -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names for the orborus pod +*/}} +{{- define "shuffle.orborus.imagePullSecrets" -}} +{{- include "common.images.renderPullSecrets" (dict "images" (list .Values.orborus.image) "context" $) -}} +{{- end -}} + +{{/* +Create the name of the service account to use for Shuffle orborus +*/}} +{{- define "shuffle.orborus.serviceAccount.name" -}} +{{- if .Values.orborus.serviceAccount.create -}} + {{ default (include "shuffle.orborus.name" .) .Values.orborus.serviceAccount.name | trunc 63 | trimSuffix "-" }} +{{- else -}} + {{ default "default" .Values.orborus.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names for the orborus service account +*/}} +{{- define "shuffle.orborus.serviceAccount.imagePullSecrets" -}} +{{- $pullSecrets := list }} + +{{- range .Values.global.imagePullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" .)) -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" .)) -}} + {{- end -}} +{{- end -}} + +{{- range .Values.orborus.serviceAccount.imagePullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" .)) -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" .)) -}} + {{- end -}} +{{- end -}} + +{{- if (not (empty $pullSecrets)) -}} +imagePullSecrets: +{{- range $pullSecrets | uniq }} + - name: {{ . }} +{{- end }} +{{- end }} +{{- end -}} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl new file mode 100644 index 00000000..533c9dde --- /dev/null +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl @@ -0,0 +1,102 @@ +{{/* +Return the common name for app components. +*/}} +{{- define "shuffle.app.name" -}} + {{- printf "%s-app" (include "common.names.fullname" .) | trunc 63 -}} +{{- end -}} + +{{/* +Return the common labels for app components deployed via helm. +NOTE: App deployments and services use shuffle.appInstance.labels instead. +*/}} +{{- define "shuffle.app.labels" -}} +{{- include "common.labels.standard" . }} +app.kubernetes.io/component: app +{{- end -}} + +{{/* +Create the name of the service account to use for Shuffle apps +*/}} +{{- define "shuffle.app.serviceAccount.name" -}} +{{- if .Values.app.serviceAccount.create -}} + {{ default (include "shuffle.app.name" .) .Values.app.serviceAccount.name | trunc 63 | trimSuffix "-" }} +{{- else -}} + {{ default "default" .Values.app.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names for the app service account +*/}} +{{- define "shuffle.app.serviceAccount.imagePullSecrets" -}} +{{- $pullSecrets := list }} + +{{- range .Values.global.imagePullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" .)) -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" .)) -}} + {{- end -}} +{{- end -}} + +{{- range .Values.app.serviceAccount.imagePullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" .)) -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" .)) -}} + {{- end -}} +{{- end -}} + +{{- if (not (empty $pullSecrets)) -}} +imagePullSecrets: +{{- range $pullSecrets | uniq }} + - name: {{ . }} +{{- end }} +{{- end }} +{{- end -}} + +{{/* +Return the sanitized name of a shuffle app. +Usage: +{{ include "shuffle.appInstance.name" $app }} +*/}} +{{- define "shuffle.appInstance.name" -}} +{{ .name | replace "_" "-" | lower }} +{{- end -}} + +{{/* +Return the sanitized name of a shuffle app, including the version of the app. +Usage: +{{ include "shuffle.appInstance.fullname" $app }} +*/}} +{{- define "shuffle.appInstance.fullname" -}} +{{ printf "%s-%s" .name .version | replace "_" "-" | lower }} +{{- end -}} + +{{/* +Return the labels for a shuffle app deployed by helm. +Usage: +{{ include "shuffle.appInstance.labels" (dict "app" $app "customLabels" .Values.commonLabels "context" $) }} +*/}} +{{- define "shuffle.appInstance.labels" -}} +app.kubernetes.io/name: shuffle-app +app.kubernetes.io/instance: {{ include "shuffle.appInstance.fullname" .app }} +helm.sh/chart: {{ include "common.names.chart" .context }} +app.kubernetes.io/instance: {{ .context.Release.Name }} +app.kubernetes.io/managed-by: {{ .context.Release.Service }} +app.kubernetes.io/part-of: shuffle +app.shuffler.io/name: {{ include "shuffle.appInstance.name" .app }} +app.shuffler.io/version: {{ .app.version | quote }} +{{- range $key, $value := .customLabels }} +{{ $key }}: {{ $value }} +{{- end }} +{{- end -}} + +{{/* +Return the match labels for apps. +These must match the labels of helm-deployed apps (shuffle.appInstance.labels), +as well as worker-deployed apps (deployK8sApp). +*/}} +{{- define "shuffle.appInstance.matchLabels" -}} +app.kubernetes.io/name: shuffle-app +{{- end -}} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl new file mode 100644 index 00000000..5a165ffa --- /dev/null +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl @@ -0,0 +1,91 @@ +{{/* +Return the common name for worker components +*/}} +{{- define "shuffle.worker.name" -}} + {{- printf "%s-worker" (include "common.names.fullname" .) | trunc 63 -}} +{{- end -}} + +{{/* +Return the common labels for worker components deployed via helm. +NOTE: Worker deployments and services use shuffle.workerInstance.labels instead. +*/}} +{{- define "shuffle.worker.labels" -}} +{{- include "common.labels.standard" . }} +app.kubernetes.io/component: worker +{{- end -}} + +{{/* +Return the proper Shuffle worker image name +*/}} +{{- define "shuffle.worker.image" -}} +{{- include "common.images.image" ( dict "imageRoot" .Values.worker.image "global" .Values.global "chart" .Chart ) -}} +{{- end -}} + +{{/* +Create the name of the service account to use for Shuffle workers +*/}} +{{- define "shuffle.worker.serviceAccount.name" -}} +{{- if .Values.worker.serviceAccount.create -}} + {{ default (include "shuffle.worker.name" .) .Values.worker.serviceAccount.name | trunc 63 | trimSuffix "-" }} +{{- else -}} + {{ default "default" .Values.worker.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names for the worker service account +*/}} +{{- define "shuffle.worker.serviceAccount.imagePullSecrets" -}} +{{- $pullSecrets := list }} + +{{- range .Values.global.imagePullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" .)) -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" .)) -}} + {{- end -}} +{{- end -}} + +{{- range .Values.worker.serviceAccount.imagePullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" .)) -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" .)) -}} + {{- end -}} +{{- end -}} + +{{- if (not (empty $pullSecrets)) -}} +imagePullSecrets: +{{- range $pullSecrets | uniq }} + - name: {{ . }} +{{- end }} +{{- end }} +{{- end -}} + +{{/* +Return the labels for a specific worker instance deployed via helm. +Usage: +{{ include "shuffle.workerInstance.labels" (dict "customLabels" .Values.commonLabels "context" $) -}} +*/}} +{{- define "shuffle.workerInstance.labels" -}} +app.kubernetes.io/name: shuffle-worker +helm.sh/chart: {{ include "common.names.chart" .context }} +app.kubernetes.io/instance: {{ .context.Release.Name }} +app.kubernetes.io/managed-by: {{ .context.Release.Service }} +app.kubernetes.io/part-of: shuffle +{{- with .context.Chart.AppVersion }} +app.kubernetes.io/version: {{ . | replace "+" "_" | quote }} +{{- end -}} +{{- range $key, $value := .customLabels }} +{{ $key }}: {{ $value }} +{{- end }} +{{- end -}} + +{{/* +Return the match labels for workers. +These must match the labels of helm-deployed workers (shuffle.workerInstance.labels), +as well as orborus-deployed workers (deployk8sworker). +*/}} +{{- define "shuffle.workerInstance.matchLabels" -}} +app.kubernetes.io/name: shuffle-worker +{{- end -}} From 1bc4ac3a1327d43aa5a64aa19be1f4aaebfa64ba Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Mon, 1 Dec 2025 16:42:26 +0100 Subject: [PATCH 07/41] add missing end Signed-off-by: Pascal Sthamer --- .../shuffle/templates/shuffle-worker/shuffle-worker-role.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-role.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-role.yaml index fffa4d4b..89dc3d4c 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-role.yaml +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-role.yaml @@ -27,3 +27,4 @@ rules: resources: ["deployments"] verbs: ["list"] {{- end }} +{{- end }} From 1aa375b28099e0ebae01ee9af84b1b55a410fd39 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Mon, 1 Dec 2025 16:44:38 +0100 Subject: [PATCH 08/41] cleanup main helpers Signed-off-by: Pascal Sthamer --- .../charts/shuffle/templates/_helpers.tpl | 40 ------------------- 1 file changed, 40 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/templates/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/_helpers.tpl index 52628d2b..b78602bb 100644 --- a/functions/kubernetes/charts/shuffle/templates/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/_helpers.tpl @@ -4,43 +4,3 @@ Return the proper image name (for the init container volume-permissions image) {{- define "shuffle.volumePermissions.image" -}} {{- include "common.images.image" ( dict "imageRoot" .Values.volumePermissions.image "global" .Values.global "chart" .Chart ) -}} {{- end -}} - - - - - - - - - - - - - -{{- if (not (empty $pullSecrets)) -}} -imagePullSecrets: -{{- range $pullSecrets | uniq }} - - name: {{ . }} -{{- end }} -{{- end }} -{{- end -}} - - - -{{- range .Values.global.imagePullSecrets -}} - {{- if kindIs "map" . -}} - {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" .)) -}} - {{- else -}} - {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" .)) -}} - {{- end -}} -{{- end -}} - - - -{{- if (not (empty $pullSecrets)) -}} -imagePullSecrets: -{{- range $pullSecrets | uniq }} - - name: {{ . }} -{{- end }} -{{- end }} -{{- end -}} From 2dcd0944af1b6b72e57c32de483511245660a8a8 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Mon, 1 Dec 2025 16:51:22 +0100 Subject: [PATCH 09/41] fix backend env Signed-off-by: Pascal Sthamer --- .../charts/shuffle/templates/backend/_helpers.tpl | 8 ++++---- .../charts/shuffle/templates/backend/backend-dpl.yaml | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/templates/backend/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/backend/_helpers.tpl index 07a8516f..48bd6543 100644 --- a/functions/kubernetes/charts/shuffle/templates/backend/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/backend/_helpers.tpl @@ -78,9 +78,9 @@ imagePullSecrets: {{- end }} {{- end -}} -{{- define "shuffle.backend.baseUrl -}} +{{- define "shuffle.backend.baseUrl" -}} http://{{ include "shuffle.backend.name" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.backend.containerPorts.http }} -{{- end - }} +{{- end -}} {{/* Return the environment variables of shuffle-backend in the format @@ -96,7 +96,7 @@ BACKEND_PORT: "{{ .Values.backend.containerPorts.http }}" BASE_URL: "{{ .Values.shuffle.baseUrl }}" SSO_REDIRECT_URL: "{{ .Values.shuffle.baseUrl }}" {{- else }} -BASE_URL: "{{ include "shuffle.backend.baseUrl .}}" +BASE_URL: "{{ include "shuffle.backend.baseUrl" . }}" {{- end }} ORG_ID: "{{ .Values.shuffle.org }}" SHUFFLE_APP_DOWNLOAD_LOCATION: "{{ .Values.backend.apps.downloadLocation }}" @@ -104,7 +104,7 @@ SHUFFLE_DOWNLOAD_AUTH_BRANCH: "{{ .Values.backend.apps.downloadBranch }}" SHUFFLE_APP_FORCE_UPDATE: "{{ .Values.backend.apps.forceUpdate }}" SHUFFLE_CHAT_DISABLED: "true" # Sets backend_url parameter for workflow execution to the cluster-internal shuffle-backend address -SHUFFLE_CLOUDRUN_URL: "{{ include "shuffle.backend.baseUrl .}}" +SHUFFLE_CLOUDRUN_URL: "{{ include "shuffle.backend.baseUrl" .}}" SHUFFLE_OPENSEARCH_URL: {{ include "common.tplvalues.render" (dict "value" .Values.backend.openSearch.url "context" $) }} SHUFFLE_OPENSEARCH_USERNAME: "{{ .Values.backend.openSearch.username }}" SHUFFLE_OPENSEARCH_CERTIFICATE_FILE: "{{ .Values.backend.openSearch.certificateFile }}" diff --git a/functions/kubernetes/charts/shuffle/templates/backend/backend-dpl.yaml b/functions/kubernetes/charts/shuffle/templates/backend/backend-dpl.yaml index 046f55c6..85650134 100644 --- a/functions/kubernetes/charts/shuffle/templates/backend/backend-dpl.yaml +++ b/functions/kubernetes/charts/shuffle/templates/backend/backend-dpl.yaml @@ -116,9 +116,10 @@ spec: {{- end }} env: {{- $env := include "shuffle.backend.env" . | fromYaml }} - {{- range $key, $val := range $env }} + {{- range $key, $val := $env }} - name: {{ $key | quote }} value: {{ $val | quote }} + {{- end }} {{- if .Values.backend.extraEnvVars }} {{- include "common.tplvalues.render" (dict "value" .Values.backend.extraEnvVars "context" $) | nindent 12 }} {{- end }} From 478c6fc947b0d925f6bc65e0c85850fbff0d5f64 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Mon, 1 Dec 2025 17:11:07 +0100 Subject: [PATCH 10/41] remove orborus-env configmap, add helpers for env variables Signed-off-by: Pascal Sthamer --- .../shuffle/templates/orborus/_helpers.tpl | 51 ++++++++++++ .../templates/orborus/orborus-cm-env.yaml | 82 ------------------- .../templates/orborus/orborus-dpl.yaml | 11 ++- .../templates/shuffle-app/_helpers.tpl | 10 +++ .../templates/shuffle-worker/_helpers.tpl | 52 ++++++++++++ .../shuffle-worker/shuffle-worker-dpl.yaml | 24 ++---- 6 files changed, 123 insertions(+), 107 deletions(-) delete mode 100644 functions/kubernetes/charts/shuffle/templates/orborus/orborus-cm-env.yaml diff --git a/functions/kubernetes/charts/shuffle/templates/orborus/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/orborus/_helpers.tpl index 3b3249b0..3ced6821 100644 --- a/functions/kubernetes/charts/shuffle/templates/orborus/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/orborus/_helpers.tpl @@ -75,3 +75,54 @@ imagePullSecrets: {{- end }} {{- end }} {{- end -}} + +{{/* +Return the environment variables of shuffle-orborus in the format +KEY: VALUE +*/}} +{{- define "shuffle.orborus.env" -}} +RUNNING_MODE: kubernetes +IS_KUBERNETES: "true" +ENVIRONMENT_NAME: "{{ .Values.shuffle.org }}" +ORG_ID: "{{ .Values.shuffle.org }}" +TZ: "{{ .Values.shuffle.timezone }}" +BASE_URL: {{ include "shuffle.backend.baseUrl" . | quote }} +KUBERNETES_NAMESPACE: "{{ .Release.Namespace }}" +SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY: {{ .Values.orborus.executionConcurrency | quote }} + +{{- if .Values.orborus.manageWorkerDeployments }} +# Shuffle worker configuration +SHUFFLE_WORKER_IMAGE: {{ include "shuffle.worker.image" . | quote }} +SHUFFLE_WORKER_SERVICE_ACCOUNT_NAME: {{ include "shuffle.worker.serviceAccount.name" . | quote }} +{{- if .Values.worker.podSecurityContext.enabled }} +SHUFFLE_WORKER_POD_SECURITY_CONTEXT: {{ omit .Values.worker.podSecurityContext "enabled" | mustToJson | quote }} +{{- end }} +{{- if .Values.worker.containerSecurityContext.enabled }} +SHUFFLE_WORKER_CONTAINER_SECURITY_CONTEXT: {{ include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.worker.containerSecurityContext "context" $) | fromYaml | mustToJson | quote }} +{{- end }} + +# Shuffle worker resources +{{- $workerResources := (.Values.worker.resources | default (include "common.resources.preset" (dict "type" .Values.worker.resourcesPreset) | fromYaml)) -}} +{{- if and $workerResources.requests $workerResources.requests.cpu }} +SHUFFLE_WORKER_CPU_REQUEST: {{ $workerResources.requests.cpu | quote }} +{{- end }} +{{- if and $workerResources.requests $workerResources.requests.memory}} +SHUFFLE_WORKER_MEMORY_REQUEST: {{ $workerResources.requests.memory | quote }} +{{- end }} +{{- if and $workerResources.requests (index $workerResources.requests "ephemeral-storage") }} +SHUFFLE_WORKER_EPHEMERAL_STORAGE_REQUEST: {{ (index $workerResources.requests "ephemeral-storage") | quote }} +{{- end }} +{{- if and $workerResources.limits $workerResources.limits.cpu }} +SHUFFLE_WORKER_CPU_LIMIT: {{ $workerResources.limits.cpu | quote }} +{{- end }} +{{- if and $workerResources.limits $workerResources.limits.memory}} +SHUFFLE_WORKER_MEMORY_LIMIT: {{ $workerResources.limits.memory | quote }} +{{- end }} +{{- if and $workerResources.limits (index $workerResources.limits "ephemeral-storage") }} +SHUFFLE_WORKER_EPHEMERAL_STORAGE_LIMIT: {{ (index $workerResources.limits "ephemeral-storage") | quote }} +{{- end }} + +# Include shuffle worker environment variables. Orborus passes them down to worker, when creating the deployment. +{{ include "shuffle.workerInstance.env" . }} +{{- end }} +{{- end -}} \ No newline at end of file diff --git a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-cm-env.yaml b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-cm-env.yaml deleted file mode 100644 index 4ce584bd..00000000 --- a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-cm-env.yaml +++ /dev/null @@ -1,82 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "shuffle.orborus.name" . }}-env - namespace: {{ include "common.names.namespace" . | quote }} - labels: {{- include "shuffle.orborus.labels" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} - {{- if .Values.commonAnnotations }} - annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} - {{- end }} -data: - ENVIRONMENT_NAME: "{{ .Values.shuffle.org }}" - ORG_ID: "{{ .Values.shuffle.org }}" - TZ: "{{ .Values.shuffle.timezone }}" - BASE_URL: {{ import "shuffle.backend.baseUrl" . | quote }} - KUBERNETES_NAMESPACE: "{{ .Release.Namespace }}" - REGISTRY_URL: "{{ .Values.shuffle.appRegistry }}" - - {{- if .Values.orborus.manageWorkerDeployments }} - # Shuffle worker configuration - SHUFFLE_WORKER_IMAGE: {{ include "shuffle.worker.image" . | quote }} - SHUFFLE_WORKER_SERVICE_ACCOUNT_NAME: {{ include "shuffle.worker.serviceAccount.name" . | quote }} - {{- if .Values.worker.podSecurityContext.enabled }} - SHUFFLE_WORKER_POD_SECURITY_CONTEXT: {{ omit .Values.worker.podSecurityContext "enabled" | mustToJson | quote }} - {{- end }} - {{- if .Values.worker.containerSecurityContext.enabled }} - SHUFFLE_WORKER_CONTAINER_SECURITY_CONTEXT: {{ include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.worker.containerSecurityContext "context" $) | fromYaml | mustToJson | quote }} - {{- end }} - - # Shuffle worker resources - {{- $workerResources := (.Values.worker.resources | default (include "common.resources.preset" (dict "type" .Values.worker.resourcesPreset) | fromYaml)) -}} - {{- if and $workerResources.requests $workerResources.requests.cpu }} - SHUFFLE_WORKER_CPU_REQUEST: {{ $workerResources.requests.cpu | quote }} - {{- end }} - {{- if and $workerResources.requests $workerResources.requests.memory}} - SHUFFLE_WORKER_MEMORY_REQUEST: {{ $workerResources.requests.memory | quote }} - {{- end }} - {{- if and $workerResources.requests (index $workerResources.requests "ephemeral-storage") }} - SHUFFLE_WORKER_EPHEMERAL_STORAGE_REQUEST: {{ (index $workerResources.requests "ephemeral-storage") | quote }} - {{- end }} - {{- if and $workerResources.limits $workerResources.limits.cpu }} - SHUFFLE_WORKER_CPU_LIMIT: {{ $workerResources.limits.cpu | quote }} - {{- end }} - {{- if and $workerResources.limits $workerResources.limits.memory}} - SHUFFLE_WORKER_MEMORY_LIMIT: {{ $workerResources.limits.memory | quote }} - {{- end }} - {{- if and $workerResources.limits (index $workerResources.limits "ephemeral-storage") }} - SHUFFLE_WORKER_EPHEMERAL_STORAGE_LIMIT: {{ (index $workerResources.limits "ephemeral-storage") | quote }} - {{- end }} - - {{- if .Values.worker.manageAppDeployments }} - # Shuffle app configuration - SHUFFLE_APP_EXPOSED_PORT: {{ .Values.app.exposedContainerPort | quote }} - SHUFFLE_APP_SERVICE_ACCOUNT_NAME: {{ include "shuffle.app.serviceAccount.name" . | quote }} - {{- if .Values.app.podSecurityContext.enabled }} - SHUFFLE_APP_POD_SECURITY_CONTEXT: {{ omit .Values.app.podSecurityContext "enabled" | mustToJson | quote }} - {{- end }} - {{- if .Values.app.containerSecurityContext.enabled }} - SHUFFLE_APP_CONTAINER_SECURITY_CONTEXT: {{ include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.app.containerSecurityContext "context" $) | fromYaml | mustToJson | quote }} - {{- end }} - - # Shuffle app resources - {{- $appResources := (.Values.app.resources | default (include "common.resources.preset" (dict "type" .Values.app.resourcesPreset) | fromYaml)) -}} - {{- if and $appResources.requests $appResources.requests.cpu }} - SHUFFLE_APP_CPU_REQUEST: {{ $appResources.requests.cpu | quote }} - {{- end }} - {{- if and $appResources.requests $appResources.requests.memory }} - SHUFFLE_APP_MEMORY_REQUEST: {{ $appResources.requests.memory | quote }} - {{- end }} - {{- if and $appResources.requests (index $appResources.requests "ephemeral-storage") }} - SHUFFLE_APP_EPHEMERAL_STORAGE_REQUEST: {{ (index $appResources.requests "ephemeral-storage") | quote }} - {{- end }} - {{- if and $appResources.limits $appResources.limits.cpu }} - SHUFFLE_APP_CPU_LIMIT: {{ $appResources.limits.cpu | quote }} - {{- end }} - {{- if and $appResources.limits $appResources.limits.memory }} - SHUFFLE_APP_MEMORY_LIMIT: {{ $appResources.limits.memory | quote }} - {{- end }} - {{- if and $appResources.limits (index $appResources.limits "ephemeral-storage") }} - SHUFFLE_APP_EPHEMERAL_STORAGE_LIMIT: {{ (index $appResources.limits "ephemeral-storage") | quote }} - {{- end }} - {{- end }} - {{- end }} diff --git a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml index a786dcd2..88f19bed 100644 --- a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml +++ b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml @@ -82,16 +82,15 @@ spec: args: {{- include "common.tplvalues.render" (dict "value" .Values.orborus.args "context" $) | nindent 12 }} {{- end }} env: - - name: RUNNING_MODE - value: kubernetes - - name: IS_KUBERNETES - value: "true" + {{- $env := include "shuffle.orborus.env" . | fromYaml }} + {{- range $key, $val := $env }} + - name: {{ $key | quote }} + value: {{ $val | quote }} + {{- end }} {{- if .Values.orborus.extraEnvVars }} {{- include "common.tplvalues.render" (dict "value" .Values.orborus.extraEnvVars "context" $) | nindent 12 }} {{- end }} envFrom: - - configMapRef: - name: {{ include "shuffle.orborus.name" . }}-env {{- if .Values.orborus.extraEnvVarsCM }} - configMapRef: name: {{ include "common.tplvalues.render" (dict "value" .Values.orborus.extraEnvVarsCM "context" $) }} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl index 533c9dde..4a482335 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl @@ -100,3 +100,13 @@ as well as worker-deployed apps (deployK8sApp). {{- define "shuffle.appInstance.matchLabels" -}} app.kubernetes.io/name: shuffle-app {{- end -}} + +{{/* +Return the environment variables of shuffle apps in the format +KEY: VALUE +*/}} +{{- define "shuffle.appInstance.env" -}} +SHUFFLE_APP_SDK_TIMEOUT: {{ .Values.app.sdkTimeout | quote }} +SHUFFLE_APP_EXPOSED_PORT: {{ .Values.app.exposedContainerPort | quote }} +SHUFFLE_LOGS_DISABLED: {{ .Values.app.disableLogs | quote }} +{{- end -}} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl index 5a165ffa..4ac59321 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl @@ -89,3 +89,55 @@ as well as orborus-deployed workers (deployk8sworker). {{- define "shuffle.workerInstance.matchLabels" -}} app.kubernetes.io/name: shuffle-worker {{- end -}} + +{{/* +Return the environment variables of shuffle-worker in the format +KEY: VALUE +*/}} +{{- define "shuffle.workerInstance.env" -}} +IS_KUBERNETES: "true" +KUBERNETES_NAMESPACE: "{{ .Release.Namespace }}" +BASE_URL: {{ include "shuffle.backend.baseUrl" . | quote }} +SHUFFLE_APP_EXPOSED_PORT: {{ .Values.app.exposedContainerPort | quote }} +WORKER_HOSTNAME: "{{ include "shuffle.worker.name" . }}.{{ .Release.Namespace }}.svc.cluster.local" + +{{- if .Values.worker.manageAppDeployments }} +# Shuffle app images +REGISTRY_URL: "{{ .Values.shuffle.appRegistry }}" +SHUFFLE_BASE_IMAGE_REGISTRY: "{{ .Values.shuffle.appRegistry }}" +SHUFFLE_BASE_IMAGE_NAME: "{{ .Values.shuffle.appBaseImageName }}" + +# Shuffle app deployment configuration +SHUFFLE_APP_SERVICE_ACCOUNT_NAME: {{ include "shuffle.app.serviceAccount.name" . | quote }} +{{- if .Values.app.podSecurityContext.enabled }} +SHUFFLE_APP_POD_SECURITY_CONTEXT: {{ omit .Values.app.podSecurityContext "enabled" | mustToJson | quote }} +{{- end }} +{{- if .Values.app.containerSecurityContext.enabled }} +SHUFFLE_APP_CONTAINER_SECURITY_CONTEXT: {{ include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.app.containerSecurityContext "context" $) | fromYaml | mustToJson | quote }} +{{- end }} + +# Shuffle app resources +{{- $appResources := (.Values.app.resources | default (include "common.resources.preset" (dict "type" .Values.app.resourcesPreset) | fromYaml)) -}} +{{- if and $appResources.requests $appResources.requests.cpu }} +SHUFFLE_APP_CPU_REQUEST: {{ $appResources.requests.cpu | quote }} +{{- end }} +{{- if and $appResources.requests $appResources.requests.memory }} +SHUFFLE_APP_MEMORY_REQUEST: {{ $appResources.requests.memory | quote }} +{{- end }} +{{- if and $appResources.requests (index $appResources.requests "ephemeral-storage") }} +SHUFFLE_APP_EPHEMERAL_STORAGE_REQUEST: {{ (index $appResources.requests "ephemeral-storage") | quote }} +{{- end }} +{{- if and $appResources.limits $appResources.limits.cpu }} +SHUFFLE_APP_CPU_LIMIT: {{ $appResources.limits.cpu | quote }} +{{- end }} +{{- if and $appResources.limits $appResources.limits.memory }} +SHUFFLE_APP_MEMORY_LIMIT: {{ $appResources.limits.memory | quote }} +{{- end }} +{{- if and $appResources.limits (index $appResources.limits "ephemeral-storage") }} +SHUFFLE_APP_EPHEMERAL_STORAGE_LIMIT: {{ (index $appResources.limits "ephemeral-storage") | quote }} +{{- end }} + +# Include shuffle app environment variables. Worker passes them down to apps, when creating their deployment. +{{ include "shuffle.appInstance.env" . }} +{{- end }} +{{- end -}} \ No newline at end of file diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml index 1832991c..09d00269 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml @@ -83,25 +83,11 @@ spec: args: {{- include "common.tplvalues.render" (dict "value" .Values.worker.args "context" $) | nindent 12 }} {{- end }} env: - - name: IS_KUBERNETES - value: "true" - - name: KUBERNETES_NAMESPACE - value: "{{ .Release.Namespace }}" - - name: SHUFFLE_LOGS_DISABLED - value: "false" # TODO: Allow to configure. Share with orborus / apps? - - name: BASE_URL - value: {{ import "shuffle.backend.baseUrl" . | quote }} - - name: REGISTRY_URL # TODO: REGISTRY_URL vs. SHUFFLE_BASE_IMAGE_REGISTRY - value: {{ .Values.shuffle.appRegistry }} - - name: SHUFFLE_BASE_IMAGE_REGISTRY - value: {{ .Values.shuffle.appRegistry }} - - name: SHUFFLE_BASE_IMAGE_NAME - value: shuffle # TODO: Allow to configure? Use same as for orborus (shared value) - - name: SHUFFLE_APP_EXPOSED_PORT - value: {{ .Values.app.exposedContainerPort | quote }} - - name: WORKER_HOSTNAME - value: {{ include "shuffle.worker.name" }}.{{ .Release.Namespace }}.svc.cluster.local - # TODO: If manageAppDeployments is set, we need to add some variables! + {{- $env := include "shuffle.workerInstance.env" . | fromYaml }} + {{- range $key, $val := $env }} + - name: {{ $key | quote }} + value: {{ $val | quote }} + {{- end }} {{- if .Values.worker.extraEnvVars }} {{- include "common.tplvalues.render" (dict "value" .Values.worker.extraEnvVars "context" $) | nindent 12 }} {{- end }} From ff25e58abb413d273a7bd608875cf7199a7dfa58 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Mon, 1 Dec 2025 17:12:12 +0100 Subject: [PATCH 11/41] cleanup values, add docs, regenerate parameters and values schema Signed-off-by: Pascal Sthamer --- functions/kubernetes/charts/shuffle/README.md | 142 +++++- .../charts/shuffle/values.schema.json | 450 +++++++++++++++++- .../kubernetes/charts/shuffle/values.yaml | 140 +++--- 3 files changed, 633 insertions(+), 99 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/README.md b/functions/kubernetes/charts/shuffle/README.md index 0fea5581..8d516904 100644 --- a/functions/kubernetes/charts/shuffle/README.md +++ b/functions/kubernetes/charts/shuffle/README.md @@ -131,14 +131,30 @@ If you want full control, you can deploy apps using helm. This has the following - granular control per app and version (e.g. have more replicas and resources for frequently used apps) - avoid problems with on-demand started apps (see https://github.com/Shuffle/Shuffle/issues/1739) -To deploy apps using helm, use the `app.deployViaHelm` value: +To deploy apps using helm, use the `app.deployViaHelm` value. Here is an example, which deploys the shuffle-tools app and 2 versions of the opensearch app: ```yaml app: + replicas: 1 # default to 1 replica per app deployViaHelm: - - app: shuffle-tools + tools: + app: shuffle-tools version: 1.2.0 - TODO + opensearchOld: + app: opensearch + version: 1.0.0 # Also deploy OpenSearch version 1.0.0, because some workflows still use it. + opensearch: + app: opensearch + version: 1.1.0 + replicas: 3 # use 3 replicas for OpenSearch app + resources: # Custom resources for OpenSearch app + limits: + memory: 512Mi ``` +The key of an app in the `deployViaHelm` map does not matter. We are not using an array here, to allow overriding values in stage-specific value files or using the command line, e.g. +`helm upgrade ... --set app.deployViaHelm.tools.replicas=3`. + +See the app Paramters section below for a complete list of helm values, that can be used to customize app deployments. +Take a look at the `app.deployViaHelm.MY_APP.*` parameters for a complete list of helm values that can be set on every app. It is possible to use a hybrid approach - deploy some apps using helm, while still allowing Worker to create additional apps on-demand. @@ -184,12 +200,13 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia ### Shared Shuffle Parameters -| Name | Description | Value | -| --------------------- | ------------------------------------------------------------- | --------------- | -| `shuffle.baseUrl` | The external base URL under which Shuffle is reachable. | `""` | -| `shuffle.org` | Default shuffle organization | `Shuffle` | -| `shuffle.appRegistry` | The registry from / to which shuffle apps are pulled / pushed | `""` | -| `shuffle.timezone` | The timezone used by Shuffle | `Europe/Berlin` | +| Name | Description | Value | +| -------------------------- | -------------------------------------------------------------------------------------------------------------------------- | --------------- | +| `shuffle.baseUrl` | The external base URL under which Shuffle is reachable. | `""` | +| `shuffle.org` | Default shuffle organization | `Shuffle` | +| `shuffle.appRegistry` | The registry from / to which shuffle apps are pulled / pushed | `docker.io` | +| `shuffle.appBaseImageName` | The base image used for shuffle apps. The final image for an app is //: | `frikky` | +| `shuffle.timezone` | The timezone used by Shuffle | `Europe/Berlin` | ### backend Parameters @@ -418,7 +435,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | ----------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------- | | `orborus.image.registry` | orborus image registry | `ghcr.io` | | `orborus.image.repository` | orborus image repository | `shuffle/shuffle-orborus` | -| `orborus.image.tag` | orborus image tag (immutable tags are recommended, defaults to appVersion) | `""` | +| `orborus.image.tag` | orborus image tag (immutable tags are recommended, defaults to appVersion) | `nightly` | | `orborus.image.digest` | orborus image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended) | `""` | | `orborus.image.pullPolicy` | orborus image pull policy | `IfNotPresent` | | `orborus.image.pullSecrets` | orborus image pull secrets | `[]` | @@ -515,15 +532,43 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `orborus.networkPolicy.allowExternalEgress` | Allow the pod to access any range of port and all destinations. | `true` | | `orborus.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` | | `orborus.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | +| `orborus.executionConcurrency` | The maximum amount of concurrent workflow executions per worker | `25` | +| `orborus.manageWorkerDeployments` | Whether workers are deployed and managed by orborus. When disabled, every worker is expected to be already deployed (see worker.enableHelmDeployment). | `true` | ### worker Parameters | Name | Description | Value | | ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------ | +| `worker.enableHelmDeployment` | Deploy worker via helm. By default, workers are deployed by Orborus. | `false` | | `worker.image.registry` | worker image registry | `ghcr.io` | | `worker.image.repository` | worker image repository | `shuffle/shuffle-worker` | | `worker.image.tag` | worker image tag (immutable tags are recommended, defaults to appVersion) | `""` | | `worker.image.digest` | worker image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended) | `""` | +| `worker.image.pullPolicy` | worker image pull policy. Only effective with worker.enableHelmDeployment. | `IfNotPresent` | +| `worker.image.pullSecrets` | worker image pull secrets. Only effective with worker.enableHelmDeployment. | `[]` | +| `worker.replicaCount` | Number of worker replicas to deploy. Only effective with worker.enableHelmDeployment. | `1` | +| `worker.extraContainerPorts` | Optionally specify extra list of additional ports for worker containers. Only effective with worker.enableHelmDeployment. | `[]` | +| `worker.livenessProbe.enabled` | Enable livenessProbe on worker containers. Only effective with worker.enableHelmDeployment. | `false` | +| `worker.livenessProbe.initialDelaySeconds` | Initial delay seconds for livenessProbe | `0` | +| `worker.livenessProbe.periodSeconds` | Period seconds for livenessProbe | `15` | +| `worker.livenessProbe.timeoutSeconds` | Timeout seconds for livenessProbe | `1` | +| `worker.livenessProbe.failureThreshold` | Failure threshold for livenessProbe | `4` | +| `worker.livenessProbe.successThreshold` | Success threshold for livenessProbe | `1` | +| `worker.readinessProbe.enabled` | Enable readinessProbe on worker containers. Only effective with worker.enableHelmDeployment. | `false` | +| `worker.readinessProbe.initialDelaySeconds` | Initial delay seconds for readinessProbe | `0` | +| `worker.readinessProbe.periodSeconds` | Period seconds for readinessProbe | `5` | +| `worker.readinessProbe.timeoutSeconds` | Timeout seconds for readinessProbe | `1` | +| `worker.readinessProbe.failureThreshold` | Failure threshold for readinessProbe | `3` | +| `worker.readinessProbe.successThreshold` | Success threshold for readinessProbe | `1` | +| `worker.startupProbe.enabled` | Enable startupProbe on worker containers. Only effective with worker.enableHelmDeployment. | `false` | +| `worker.startupProbe.initialDelaySeconds` | Initial delay seconds for startupProbe | `0` | +| `worker.startupProbe.periodSeconds` | Period seconds for startupProbe | `1` | +| `worker.startupProbe.timeoutSeconds` | Timeout seconds for startupProbe | `1` | +| `worker.startupProbe.failureThreshold` | Failure threshold for startupProbe | `60` | +| `worker.startupProbe.successThreshold` | Success threshold for startupProbe | `1` | +| `worker.customLivenessProbe` | Custom livenessProbe that overrides the default one. Only effective with worker.enableHelmDeployment. | `{}` | +| `worker.customReadinessProbe` | Custom readinessProbe that overrides the default one. Only effective with worker.enableHelmDeployment. | `{}` | +| `worker.customStartupProbe` | Custom startupProbe that overrides the default one. Only effective with worker.enableHelmDeployment. | `{}` | | `worker.resourcesPreset` | Set worker container resources according to one common preset (allowed values: none, nano, small, medium, large, xlarge, 2xlarge). This is ignored if worker.resources is set (worker.resources is recommended for production). | `nano` | | `worker.resources` | Set worker container requests and limits for different resources like CPU or memory (essential for production workloads) | `{}` | | `worker.podSecurityContext.enabled` | Enable worker pods' Security Context | `true` | @@ -541,6 +586,49 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `worker.containerSecurityContext.allowPrivilegeEscalation` | Set allowPrivilegeEscalation in worker container' Security Context | `false` | | `worker.containerSecurityContext.capabilities.drop` | List of capabilities to be dropped in worker container | `["ALL"]` | | `worker.containerSecurityContext.seccompProfile.type` | Set seccomp profile in worker container | `RuntimeDefault` | +| `worker.command` | Override default worker container command (useful when using custom images). Only effective with worker.enableHelmDeployment. | `[]` | +| `worker.args` | Override default worker container args (useful when using custom images). Only effective with worker.enableHelmDeployment. | `[]` | +| `worker.automountServiceAccountToken` | Mount Service Account token in worker pods. Only effective with worker.enableHelmDeployment. | `true` | +| `worker.hostAliases` | worker pods host aliases. Only effective with worker.enableHelmDeployment. | `[]` | +| `worker.deploymentAnnotations` | Annotations for worker deployment. Only effective with worker.enableHelmDeployment. | `{}` | +| `worker.podLabels` | Extra labels for worker pods. Only effective with worker.enableHelmDeployment. | `{}` | +| `worker.podAnnotations` | Annotations for worker pods. Only effective with worker.enableHelmDeployment. | `{}` | +| `worker.podAffinityPreset` | Pod affinity preset. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`. Only effective with worker.enableHelmDeployment. | `""` | +| `worker.podAntiAffinityPreset` | Pod anti-affinity preset. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`. Only effective with worker.enableHelmDeployment. | `soft` | +| `worker.nodeAffinityPreset.type` | Node affinity preset type. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`. Only effective with worker.enableHelmDeployment. | `""` | +| `worker.nodeAffinityPreset.key` | Node label key to match. Ignored if `worker.affinity` is set | `""` | +| `worker.nodeAffinityPreset.values` | Node label values to match. Ignored if `worker.affinity` is set | `[]` | +| `worker.affinity` | Affinity for worker pods assignment. Only effective with worker.enableHelmDeployment. | `{}` | +| `worker.nodeSelector` | Node labels for worker pods assignment. Only effective with worker.enableHelmDeployment. | `{}` | +| `worker.tolerations` | Tolerations for worker pods assignment. Only effective with worker.enableHelmDeployment. | `[]` | +| `worker.updateStrategy.type` | worker deployment strategy type. Only effective with worker.enableHelmDeployment. | `RollingUpdate` | +| `worker.priorityClassName` | worker pods' priorityClassName. Only effective with worker.enableHelmDeployment. | `""` | +| `worker.topologySpreadConstraints` | Topology Spread Constraints for worker pod assignment spread across your cluster among failure-domains. Only effective with worker.enableHelmDeployment. | `[]` | +| `worker.schedulerName` | Name of the k8s scheduler (other than default) for worker pods. Only effective with worker.enableHelmDeployment. | `""` | +| `worker.terminationGracePeriodSeconds` | Seconds worker pods need to terminate gracefully. Only effective with worker.enableHelmDeployment. | `""` | +| `worker.lifecycleHooks` | for worker containers to automate configuration before or after startup. Only effective with worker.enableHelmDeployment. | `{}` | +| `worker.extraEnvVars` | Array with extra environment variables to add to worker containers. Only effective with worker.enableHelmDeployment. | `[]` | +| `worker.extraEnvVarsCM` | Name of existing ConfigMap containing extra env vars for worker containers. Only effective with worker.enableHelmDeployment. | `""` | +| `worker.extraEnvVarsSecret` | Name of existing Secret containing extra env vars for worker containers. Only effective with worker.enableHelmDeployment. | `""` | +| `worker.extraVolumes` | Optionally specify extra list of additional volumes for the worker pods. Only effective with worker.enableHelmDeployment. | `[]` | +| `worker.extraVolumeMounts` | Optionally specify extra list of additional volumeMounts for the worker containers. Only effective with worker.enableHelmDeployment. | `[]` | +| `worker.sidecars` | Add additional sidecar containers to the worker pods. Only effective with worker.enableHelmDeployment. | `[]` | +| `worker.initContainers` | Add additional init containers to the worker pods. Only effective with worker.enableHelmDeployment. | `[]` | +| `worker.pdb.create` | Enable/disable a Pod Disruption Budget creation. Only effective with worker.enableHelmDeployment. | `true` | +| `worker.pdb.minAvailable` | Minimum number/percentage of pods that should remain scheduled | `""` | +| `worker.pdb.maxUnavailable` | Maximum number/percentage of pods that may be made unavailable. Defaults to `1` if both `worker.pdb.minAvailable` and `worker.pdb.maxUnavailable` are empty. | `""` | +| `worker.autoscaling.vpa.enabled` | Enable VPA for worker pods. Only effective with worker.enableHelmDeployment. | `false` | +| `worker.autoscaling.vpa.annotations` | Annotations for VPA resource | `{}` | +| `worker.autoscaling.vpa.controlledResources` | VPA List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory | `[]` | +| `worker.autoscaling.vpa.maxAllowed` | VPA Max allowed resources for the pod | `{}` | +| `worker.autoscaling.vpa.minAllowed` | VPA Min allowed resources for the pod | `{}` | +| `worker.autoscaling.vpa.updatePolicy.updateMode` | Autoscaling update policy | `Auto` | +| `worker.autoscaling.hpa.enabled` | Enable HPA for worker pods. Only effective with worker.enableHelmDeployment. | `false` | +| `worker.autoscaling.hpa.minReplicas` | Minimum number of replicas | `""` | +| `worker.autoscaling.hpa.maxReplicas` | Maximum number of replicas | `""` | +| `worker.autoscaling.hpa.targetCPU` | Target CPU utilization percentage | `""` | +| `worker.autoscaling.hpa.targetMemory` | Target Memory utilization percentage | `""` | +| `worker.service.labels` | Extra labels for worker service. Only effective with worker.enableHelmDeployment. | `{}` | | `worker.serviceAccount.create` | Specifies whether a ServiceAccount should be created | `true` | | `worker.serviceAccount.name` | The name of the ServiceAccount to use. | `""` | | `worker.serviceAccount.annotations` | Additional Service Account annotations (evaluated as a template) | `{}` | @@ -552,6 +640,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `worker.networkPolicy.allowExternalEgress` | Allow the pod to access any range of port and all destinations. | `true` | | `worker.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` | | `worker.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | +| `worker.manageAppDeployments` | Whether apps are deployed and managed by worker. When disabled, every used app is expected to to be already deployed (see app.deployViaHelm). | `true` | ### app Parameters @@ -586,6 +675,11 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `app.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` | | `app.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | | `app.exposedContainerPort` | The port that shuffle app containers will listen on for new requests. | `80` | +| `app.sdkTimeout` | The timeout in seconds for app actions. | `300` | +| `app.disableLogs` | Do not capture app logs. By default, app logs are captured, so that they are visible in the frontend. | `false` | +| `app.deployViaHelm` | A list of apps that should be deployed using helm. | `{}` | +| `app.deployViaHelm.MY_APP.app` | The name of the app (required, e.g. shuffle-tools) | | +| `app.deployViaHelm.MY_APP.version` | The version of the app (required, e.g. 1.2.0) | | ### Traffic Exposure Parameters @@ -654,19 +748,19 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia ### Init Container Parameters -| Name | Description | Value | -| ----------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------ | -| `volumePermissions.enabled` | Enable init container that changes the owner/group of the PV mount point to `runAsUser:fsGroup` | `false` | -| `volumePermissions.image.registry` | OS Shell + Utility image registry | `docker.io` | -| `volumePermissions.image.repository` | OS Shell + Utility image repository | `bitnami/os-shell` | -| `volumePermissions.image.tag` | OS Shell + Utility image tag (immutable tags are recommended) | `12-debian-12-r30` | -| `volumePermissions.image.pullPolicy` | OS Shell + Utility image pull policy | `IfNotPresent` | -| `volumePermissions.image.pullSecrets` | OS Shell + Utility image pull secrets | `[]` | -| `volumePermissions.resourcesPreset` | Set init container resources according to one common preset (allowed values: none, nano, small, medium, large, xlarge, 2xlarge). This is ignored if volumePermissions.resources is set (volumePermissions.resources is recommended for production). | `nano` | -| `volumePermissions.resources` | Set init container requests and limits for different resources like CPU or memory (essential for production workloads) | `{}` | -| `volumePermissions.containerSecurityContext.enabled` | Enabled init container' Security Context | `true` | -| `volumePermissions.containerSecurityContext.seLinuxOptions` | Set SELinux options in init container | `{}` | -| `volumePermissions.containerSecurityContext.runAsUser` | Set init container's Security Context runAsUser | `0` | +| Name | Description | Value | +| ----------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------ | +| `volumePermissions.enabled` | Enable init container that changes the owner/group of the PV mount point to `runAsUser:fsGroup` | `false` | +| `volumePermissions.image.registry` | OS Shell + Utility image registry | `docker.io` | +| `volumePermissions.image.repository` | OS Shell + Utility image repository | `bitnamilegacy/os-shell` | +| `volumePermissions.image.tag` | OS Shell + Utility image tag (immutable tags are recommended) | `12-debian-12-r30` | +| `volumePermissions.image.pullPolicy` | OS Shell + Utility image pull policy | `IfNotPresent` | +| `volumePermissions.image.pullSecrets` | OS Shell + Utility image pull secrets | `[]` | +| `volumePermissions.resourcesPreset` | Set init container resources according to one common preset (allowed values: none, nano, small, medium, large, xlarge, 2xlarge). This is ignored if volumePermissions.resources is set (volumePermissions.resources is recommended for production). | `nano` | +| `volumePermissions.resources` | Set init container requests and limits for different resources like CPU or memory (essential for production workloads) | `{}` | +| `volumePermissions.containerSecurityContext.enabled` | Enabled init container' Security Context | `true` | +| `volumePermissions.containerSecurityContext.seLinuxOptions` | Set SELinux options in init container | `{}` | +| `volumePermissions.containerSecurityContext.runAsUser` | Set init container's Security Context runAsUser | `0` | ### OpenSearch Parameters @@ -685,3 +779,5 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia + + diff --git a/functions/kubernetes/charts/shuffle/values.schema.json b/functions/kubernetes/charts/shuffle/values.schema.json index c2f9821c..d997e81e 100644 --- a/functions/kubernetes/charts/shuffle/values.schema.json +++ b/functions/kubernetes/charts/shuffle/values.schema.json @@ -130,7 +130,12 @@ "appRegistry": { "type": "string", "description": "The registry from / to which shuffle apps are pulled / pushed", - "default": "" + "default": "docker.io" + }, + "appBaseImageName": { + "type": "string", + "description": "The base image used for shuffle apps. The final image for an app is //:", + "default": "frikky" }, "timezone": { "type": "string", @@ -1466,7 +1471,7 @@ "tag": { "type": "string", "description": "orborus image tag (immutable tags are recommended, defaults to appVersion)", - "default": "" + "default": "nightly" }, "digest": { "type": "string", @@ -2053,12 +2058,27 @@ "items": {} } } + }, + "executionConcurrency": { + "type": "number", + "description": "The maximum amount of concurrent workflow executions per worker", + "default": 25 + }, + "manageWorkerDeployments": { + "type": "boolean", + "description": "Whether workers are deployed and managed by orborus. When disabled, every worker is expected to be already deployed (see worker.enableHelmDeployment).", + "default": true } } }, "worker": { "type": "object", "properties": { + "enableHelmDeployment": { + "type": "boolean", + "description": "Deploy worker via helm. By default, workers are deployed by Orborus.", + "default": false + }, "image": { "type": "object", "properties": { @@ -2081,9 +2101,151 @@ "type": "string", "description": "worker image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended)", "default": "" + }, + "pullPolicy": { + "type": "string", + "description": "worker image pull policy. Only effective with worker.enableHelmDeployment.", + "default": "IfNotPresent" + }, + "pullSecrets": { + "type": "array", + "description": "worker image pull secrets. Only effective with worker.enableHelmDeployment.", + "default": [], + "items": {} } } }, + "replicaCount": { + "type": "number", + "description": "Number of worker replicas to deploy. Only effective with worker.enableHelmDeployment.", + "default": 1 + }, + "extraContainerPorts": { + "type": "array", + "description": "Optionally specify extra list of additional ports for worker containers. Only effective with worker.enableHelmDeployment.", + "default": [], + "items": {} + }, + "livenessProbe": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable livenessProbe on worker containers. Only effective with worker.enableHelmDeployment.", + "default": false + }, + "initialDelaySeconds": { + "type": "number", + "description": "Initial delay seconds for livenessProbe", + "default": 0 + }, + "periodSeconds": { + "type": "number", + "description": "Period seconds for livenessProbe", + "default": 15 + }, + "timeoutSeconds": { + "type": "number", + "description": "Timeout seconds for livenessProbe", + "default": 1 + }, + "failureThreshold": { + "type": "number", + "description": "Failure threshold for livenessProbe", + "default": 4 + }, + "successThreshold": { + "type": "number", + "description": "Success threshold for livenessProbe", + "default": 1 + } + } + }, + "readinessProbe": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable readinessProbe on worker containers. Only effective with worker.enableHelmDeployment.", + "default": false + }, + "initialDelaySeconds": { + "type": "number", + "description": "Initial delay seconds for readinessProbe", + "default": 0 + }, + "periodSeconds": { + "type": "number", + "description": "Period seconds for readinessProbe", + "default": 5 + }, + "timeoutSeconds": { + "type": "number", + "description": "Timeout seconds for readinessProbe", + "default": 1 + }, + "failureThreshold": { + "type": "number", + "description": "Failure threshold for readinessProbe", + "default": 3 + }, + "successThreshold": { + "type": "number", + "description": "Success threshold for readinessProbe", + "default": 1 + } + } + }, + "startupProbe": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable startupProbe on worker containers. Only effective with worker.enableHelmDeployment.", + "default": false + }, + "initialDelaySeconds": { + "type": "number", + "description": "Initial delay seconds for startupProbe", + "default": 0 + }, + "periodSeconds": { + "type": "number", + "description": "Period seconds for startupProbe", + "default": 1 + }, + "timeoutSeconds": { + "type": "number", + "description": "Timeout seconds for startupProbe", + "default": 1 + }, + "failureThreshold": { + "type": "number", + "description": "Failure threshold for startupProbe", + "default": 60 + }, + "successThreshold": { + "type": "number", + "description": "Success threshold for startupProbe", + "default": 1 + } + } + }, + "customLivenessProbe": { + "type": "object", + "description": "Custom livenessProbe that overrides the default one. Only effective with worker.enableHelmDeployment.", + "default": {} + }, + "customReadinessProbe": { + "type": "object", + "description": "Custom readinessProbe that overrides the default one. Only effective with worker.enableHelmDeployment.", + "default": {} + }, + "customStartupProbe": { + "type": "object", + "description": "Custom startupProbe that overrides the default one. Only effective with worker.enableHelmDeployment.", + "default": {} + }, "resourcesPreset": { "type": "string", "description": "Set worker container resources according to one common preset (allowed values: none, nano, small, medium, large, xlarge, 2xlarge). This is ignored if worker.resources is set (worker.resources is recommended for production).", @@ -2191,6 +2353,273 @@ } } }, + "command": { + "type": "array", + "description": "Override default worker container command (useful when using custom images). Only effective with worker.enableHelmDeployment.", + "default": [], + "items": {} + }, + "args": { + "type": "array", + "description": "Override default worker container args (useful when using custom images). Only effective with worker.enableHelmDeployment.", + "default": [], + "items": {} + }, + "automountServiceAccountToken": { + "type": "boolean", + "description": "Mount Service Account token in worker pods. Only effective with worker.enableHelmDeployment.", + "default": true + }, + "hostAliases": { + "type": "array", + "description": "worker pods host aliases. Only effective with worker.enableHelmDeployment.", + "default": [], + "items": {} + }, + "deploymentAnnotations": { + "type": "object", + "description": "Annotations for worker deployment. Only effective with worker.enableHelmDeployment.", + "default": {} + }, + "podLabels": { + "type": "object", + "description": "Extra labels for worker pods. Only effective with worker.enableHelmDeployment.", + "default": {} + }, + "podAnnotations": { + "type": "object", + "description": "Annotations for worker pods. Only effective with worker.enableHelmDeployment.", + "default": {} + }, + "podAffinityPreset": { + "type": "string", + "description": "Pod affinity preset. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`. Only effective with worker.enableHelmDeployment.", + "default": "" + }, + "podAntiAffinityPreset": { + "type": "string", + "description": "Pod anti-affinity preset. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`. Only effective with worker.enableHelmDeployment.", + "default": "soft" + }, + "nodeAffinityPreset": { + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "Node affinity preset type. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`. Only effective with worker.enableHelmDeployment.", + "default": "" + }, + "key": { + "type": "string", + "description": "Node label key to match. Ignored if `worker.affinity` is set", + "default": "" + }, + "values": { + "type": "array", + "description": "Node label values to match. Ignored if `worker.affinity` is set", + "default": [], + "items": {} + } + } + }, + "affinity": { + "type": "object", + "description": "Affinity for worker pods assignment. Only effective with worker.enableHelmDeployment.", + "default": {} + }, + "nodeSelector": { + "type": "object", + "description": "Node labels for worker pods assignment. Only effective with worker.enableHelmDeployment.", + "default": {} + }, + "tolerations": { + "type": "array", + "description": "Tolerations for worker pods assignment. Only effective with worker.enableHelmDeployment.", + "default": [], + "items": {} + }, + "updateStrategy": { + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "worker deployment strategy type. Only effective with worker.enableHelmDeployment.", + "default": "RollingUpdate" + } + } + }, + "priorityClassName": { + "type": "string", + "description": "worker pods' priorityClassName. Only effective with worker.enableHelmDeployment.", + "default": "" + }, + "topologySpreadConstraints": { + "type": "array", + "description": "Topology Spread Constraints for worker pod assignment spread across your cluster among failure-domains. Only effective with worker.enableHelmDeployment.", + "default": [], + "items": {} + }, + "schedulerName": { + "type": "string", + "description": "Name of the k8s scheduler (other than default) for worker pods. Only effective with worker.enableHelmDeployment.", + "default": "" + }, + "terminationGracePeriodSeconds": { + "type": "string", + "description": "Seconds worker pods need to terminate gracefully. Only effective with worker.enableHelmDeployment.", + "default": "" + }, + "lifecycleHooks": { + "type": "object", + "description": "for worker containers to automate configuration before or after startup. Only effective with worker.enableHelmDeployment.", + "default": {} + }, + "extraEnvVars": { + "type": "array", + "description": "Array with extra environment variables to add to worker containers. Only effective with worker.enableHelmDeployment.", + "default": [], + "items": {} + }, + "extraEnvVarsCM": { + "type": "string", + "description": "Name of existing ConfigMap containing extra env vars for worker containers. Only effective with worker.enableHelmDeployment.", + "default": "" + }, + "extraEnvVarsSecret": { + "type": "string", + "description": "Name of existing Secret containing extra env vars for worker containers. Only effective with worker.enableHelmDeployment.", + "default": "" + }, + "extraVolumes": { + "type": "array", + "description": "Optionally specify extra list of additional volumes for the worker pods. Only effective with worker.enableHelmDeployment.", + "default": [], + "items": {} + }, + "extraVolumeMounts": { + "type": "array", + "description": "Optionally specify extra list of additional volumeMounts for the worker containers. Only effective with worker.enableHelmDeployment.", + "default": [], + "items": {} + }, + "sidecars": { + "type": "array", + "description": "Add additional sidecar containers to the worker pods. Only effective with worker.enableHelmDeployment.", + "default": [], + "items": {} + }, + "initContainers": { + "type": "array", + "description": "Add additional init containers to the worker pods. Only effective with worker.enableHelmDeployment.", + "default": [], + "items": {} + }, + "pdb": { + "type": "object", + "properties": { + "create": { + "type": "boolean", + "description": "Enable/disable a Pod Disruption Budget creation. Only effective with worker.enableHelmDeployment.", + "default": true + }, + "minAvailable": { + "type": "string", + "description": "Minimum number/percentage of pods that should remain scheduled", + "default": "" + }, + "maxUnavailable": { + "type": "string", + "description": "Maximum number/percentage of pods that may be made unavailable. Defaults to `1` if both `worker.pdb.minAvailable` and `worker.pdb.maxUnavailable` are empty.", + "default": "" + } + } + }, + "autoscaling": { + "type": "object", + "properties": { + "vpa": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable VPA for worker pods. Only effective with worker.enableHelmDeployment.", + "default": false + }, + "annotations": { + "type": "object", + "description": "Annotations for VPA resource", + "default": {} + }, + "controlledResources": { + "type": "array", + "description": "VPA List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory", + "default": [], + "items": {} + }, + "maxAllowed": { + "type": "object", + "description": "VPA Max allowed resources for the pod", + "default": {} + }, + "minAllowed": { + "type": "object", + "description": "VPA Min allowed resources for the pod", + "default": {} + }, + "updatePolicy": { + "type": "object", + "properties": { + "updateMode": { + "type": "string", + "description": "Autoscaling update policy", + "default": "Auto" + } + } + } + } + }, + "hpa": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable HPA for worker pods. Only effective with worker.enableHelmDeployment.", + "default": false + }, + "minReplicas": { + "type": "string", + "description": "Minimum number of replicas", + "default": "" + }, + "maxReplicas": { + "type": "string", + "description": "Maximum number of replicas", + "default": "" + }, + "targetCPU": { + "type": "string", + "description": "Target CPU utilization percentage", + "default": "" + }, + "targetMemory": { + "type": "string", + "description": "Target Memory utilization percentage", + "default": "" + } + } + } + } + }, + "service": { + "type": "object", + "properties": { + "labels": { + "type": "object", + "description": "Extra labels for worker service. Only effective with worker.enableHelmDeployment.", + "default": {} + } + } + }, "serviceAccount": { "type": "object", "properties": { @@ -2263,6 +2692,11 @@ "items": {} } } + }, + "manageAppDeployments": { + "type": "boolean", + "description": "Whether apps are deployed and managed by worker. When disabled, every used app is expected to to be already deployed (see app.deployViaHelm).", + "default": true } } }, @@ -2453,6 +2887,16 @@ "type": "number", "description": "The port that shuffle app containers will listen on for new requests. ", "default": 80 + }, + "sdkTimeout": { + "type": "number", + "description": "The timeout in seconds for app actions.", + "default": 300 + }, + "disableLogs": { + "type": "boolean", + "description": "Do not capture app logs. By default, app logs are captured, so that they are visible in the frontend.", + "default": false } } }, @@ -2793,7 +3237,7 @@ "repository": { "type": "string", "description": "OS Shell + Utility image repository", - "default": "bitnami/os-shell" + "default": "bitnamilegacy/os-shell" }, "tag": { "type": "string", diff --git a/functions/kubernetes/charts/shuffle/values.yaml b/functions/kubernetes/charts/shuffle/values.yaml index 4b745749..c53e4943 100644 --- a/functions/kubernetes/charts/shuffle/values.yaml +++ b/functions/kubernetes/charts/shuffle/values.yaml @@ -74,7 +74,7 @@ diagnosticMode: shuffle: ## @param shuffle.baseUrl The external base URL under which Shuffle is reachable. ## - #baseUrl: "" + baseUrl: "" ## ref: https://shuffler.io/docs/organizations ## This chart only supports single-tenant deployments at the moment @@ -84,14 +84,18 @@ shuffle: ## @param shuffle.appRegistry The registry from / to which shuffle apps are pulled / pushed ## - appRegistry: "" # TODO: Default? + appRegistry: "docker.io" + + ## @param shuffle.appBaseImageName The base image used for shuffle apps. The final image for an app is //: + ## + appBaseImageName: "frikky" ## @param shuffle.timezone The timezone used by Shuffle ## timezone: Europe/Berlin ## @section backend Parameters -## +## backend: ## backend image ## @param backend.image.registry backend image registry @@ -525,8 +529,6 @@ backend: ## @param backend.openSearch.username The username that is used for authenticating with OpenSearch ## username: admin - - password: StrongShufflePassword321! ## @param backend.openSearch.certificateFile The path to a custom OpenSearch certificate file ## certificateFile: "" @@ -887,16 +889,6 @@ frontend: ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ ## labels: {} - type: NodePort - ports: - - name: http - port: 80 - targetPort: 3001 - nodePort: 30080 - - name: https - port: 443 - targetPort: 3443 - nodePort: 30443 ## ServiceAccount configuration ## @@ -1199,17 +1191,7 @@ orborus: ## - name: FOO ## value: "bar" ## - extraEnvVars: # TODO: Make this list empty and move to other values. Also check if these need to be set on worker or apps. - - name: SHUFFLE_APP_SDK_TIMEOUT - value: "300" - - name: SHUFFLE_ORBORUS_EXCUTION_CONCURRENCY - value: "7" - - name: SHUFFLE_STATS_DISABLED - value: "true" - - name: KUBERNETES_NAMESPACE - value: "shuffle" - - name: SHUFFLE_BASE_IMAGE_NAME - value: "frikky/shuffle" # TODO: Allow to configure, share with worker + extraEnvVars: [] ## @param orborus.extraEnvVarsCM Name of existing ConfigMap containing extra env vars for orborus containers ## extraEnvVarsCM: "" @@ -1347,6 +1329,10 @@ orborus: ## extraEgress: [] + ## @param orborus.executionConcurrency The maximum amount of concurrent workflow executions per worker + ## + executionConcurrency: 25 + ## @param orborus.manageWorkerDeployments Whether workers are deployed and managed by orborus. When disabled, every worker is expected to be already deployed (see worker.enableHelmDeployment). ## This effectively removes required RBAC permissions from the shuffle-orborus service account to create deployments and services. ## Orborus might still attempt to create kubernetes objects, resulting in an error. There is currently no way to tell orborus, that it should not manage k8s resources. @@ -1365,8 +1351,8 @@ worker: ## @param worker.image.repository worker image repository ## @param worker.image.tag worker image tag (immutable tags are recommended, defaults to appVersion) ## @param worker.image.digest worker image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended) - ## @param worker.image.pullPolicy worker image pull policy - ## @param worker.image.pullSecrets worker image pull secrets + ## @param worker.image.pullPolicy worker image pull policy. Only effective with worker.enableHelmDeployment. + ## @param worker.image.pullSecrets worker image pull secrets. Only effective with worker.enableHelmDeployment. ## image: registry: ghcr.io @@ -1386,10 +1372,10 @@ worker: ## - myRegistryKeySecretName ## pullSecrets: [] - ## @param worker.replicaCount Number of worker replicas to deploy + ## @param worker.replicaCount Number of worker replicas to deploy. Only effective with worker.enableHelmDeployment. ## replicaCount: 1 - ## @param worker.extraContainerPorts Optionally specify extra list of additional ports for worker containers + ## @param worker.extraContainerPorts Optionally specify extra list of additional ports for worker containers. Only effective with worker.enableHelmDeployment. ## e.g: ## extraContainerPorts: ## - name: myservice @@ -1398,7 +1384,7 @@ worker: extraContainerPorts: [] ## Configure extra options for worker containers' liveness and readiness probes ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes - ## @param worker.livenessProbe.enabled Enable livenessProbe on worker containers + ## @param worker.livenessProbe.enabled Enable livenessProbe on worker containers. Only effective with worker.enableHelmDeployment. ## @param worker.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe ## @param worker.livenessProbe.periodSeconds Period seconds for livenessProbe ## @param worker.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe @@ -1412,7 +1398,7 @@ worker: timeoutSeconds: 1 failureThreshold: 4 successThreshold: 1 - ## @param worker.readinessProbe.enabled Enable readinessProbe on worker containers + ## @param worker.readinessProbe.enabled Enable readinessProbe on worker containers. Only effective with worker.enableHelmDeployment. ## @param worker.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe ## @param worker.readinessProbe.periodSeconds Period seconds for readinessProbe ## @param worker.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe @@ -1426,7 +1412,7 @@ worker: timeoutSeconds: 1 failureThreshold: 3 successThreshold: 1 - ## @param worker.startupProbe.enabled Enable startupProbe on worker containers + ## @param worker.startupProbe.enabled Enable startupProbe on worker containers. Only effective with worker.enableHelmDeployment. ## @param worker.startupProbe.initialDelaySeconds Initial delay seconds for startupProbe ## @param worker.startupProbe.periodSeconds Period seconds for startupProbe ## @param worker.startupProbe.timeoutSeconds Timeout seconds for startupProbe @@ -1440,13 +1426,13 @@ worker: timeoutSeconds: 1 failureThreshold: 60 successThreshold: 1 - ## @param worker.customLivenessProbe Custom livenessProbe that overrides the default one + ## @param worker.customLivenessProbe Custom livenessProbe that overrides the default one. Only effective with worker.enableHelmDeployment. ## customLivenessProbe: {} - ## @param worker.customReadinessProbe Custom readinessProbe that overrides the default one + ## @param worker.customReadinessProbe Custom readinessProbe that overrides the default one. Only effective with worker.enableHelmDeployment. ## customReadinessProbe: {} - ## @param worker.customStartupProbe Custom startupProbe that overrides the default one + ## @param worker.customStartupProbe Custom startupProbe that overrides the default one. Only effective with worker.enableHelmDeployment. ## customStartupProbe: {} ## worker resource requests and limits @@ -1508,37 +1494,37 @@ worker: drop: ["ALL"] seccompProfile: type: "RuntimeDefault" - ## @param worker.command Override default worker container command (useful when using custom images) + ## @param worker.command Override default worker container command (useful when using custom images). Only effective with worker.enableHelmDeployment. ## command: [] - ## @param worker.args Override default worker container args (useful when using custom images) + ## @param worker.args Override default worker container args (useful when using custom images). Only effective with worker.enableHelmDeployment. ## args: [] - ## @param worker.automountServiceAccountToken Mount Service Account token in worker pods + ## @param worker.automountServiceAccountToken Mount Service Account token in worker pods. Only effective with worker.enableHelmDeployment. ## NOTE: worker requires the service account credentials to be mounted if manageAppDeployments is enabled. ## automountServiceAccountToken: true - ## @param worker.hostAliases worker pods host aliases + ## @param worker.hostAliases worker pods host aliases. Only effective with worker.enableHelmDeployment. ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ ## hostAliases: [] - ## @param worker.deploymentAnnotations Annotations for worker deployment + ## @param worker.deploymentAnnotations Annotations for worker deployment. Only effective with worker.enableHelmDeployment. ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ ## deploymentAnnotations: {} - ## @param worker.podLabels Extra labels for worker pods + ## @param worker.podLabels Extra labels for worker pods. Only effective with worker.enableHelmDeployment. ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ ## podLabels: {} - ## @param worker.podAnnotations Annotations for worker pods + ## @param worker.podAnnotations Annotations for worker pods. Only effective with worker.enableHelmDeployment. ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ ## podAnnotations: {} - ## @param worker.podAffinityPreset Pod affinity preset. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard` + ## @param worker.podAffinityPreset Pod affinity preset. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`. Only effective with worker.enableHelmDeployment. ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity ## podAffinityPreset: "" - ## @param worker.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard` + ## @param worker.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`. Only effective with worker.enableHelmDeployment. ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity ## podAntiAffinityPreset: soft @@ -1546,7 +1532,7 @@ worker: ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity ## nodeAffinityPreset: - ## @param worker.nodeAffinityPreset.type Node affinity preset type. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard` + ## @param worker.nodeAffinityPreset.type Node affinity preset type. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`. Only effective with worker.enableHelmDeployment. ## type: "" ## @param worker.nodeAffinityPreset.key Node label key to match. Ignored if `worker.affinity` is set @@ -1559,64 +1545,64 @@ worker: ## - e2e-az2 ## values: [] - ## @param worker.affinity Affinity for worker pods assignment + ## @param worker.affinity Affinity for worker pods assignment. Only effective with worker.enableHelmDeployment. ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity ## NOTE: `worker.podAffinityPreset`, `worker.podAntiAffinityPreset`, and `worker.nodeAffinityPreset` will be ignored when it's set ## affinity: {} - ## @param worker.nodeSelector Node labels for worker pods assignment + ## @param worker.nodeSelector Node labels for worker pods assignment. Only effective with worker.enableHelmDeployment. ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/ ## nodeSelector: {} - ## @param worker.tolerations Tolerations for worker pods assignment + ## @param worker.tolerations Tolerations for worker pods assignment. Only effective with worker.enableHelmDeployment. ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ ## tolerations: [] - ## @param worker.updateStrategy.type worker deployment strategy type + ## @param worker.updateStrategy.type worker deployment strategy type. Only effective with worker.enableHelmDeployment. ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy ## updateStrategy: ## Can be set to RollingUpdate or Recreate ## type: RollingUpdate - ## @param worker.priorityClassName worker pods' priorityClassName + ## @param worker.priorityClassName worker pods' priorityClassName. Only effective with worker.enableHelmDeployment. ## priorityClassName: "" - ## @param worker.topologySpreadConstraints Topology Spread Constraints for worker pod assignment spread across your cluster among failure-domains + ## @param worker.topologySpreadConstraints Topology Spread Constraints for worker pod assignment spread across your cluster among failure-domains. Only effective with worker.enableHelmDeployment. ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods ## topologySpreadConstraints: [] - ## @param worker.schedulerName Name of the k8s scheduler (other than default) for worker pods + ## @param worker.schedulerName Name of the k8s scheduler (other than default) for worker pods. Only effective with worker.enableHelmDeployment. ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ ## schedulerName: "" - ## @param worker.terminationGracePeriodSeconds Seconds worker pods need to terminate gracefully + ## @param worker.terminationGracePeriodSeconds Seconds worker pods need to terminate gracefully. Only effective with worker.enableHelmDeployment. ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods ## terminationGracePeriodSeconds: "" - ## @param worker.lifecycleHooks for worker containers to automate configuration before or after startup + ## @param worker.lifecycleHooks for worker containers to automate configuration before or after startup. Only effective with worker.enableHelmDeployment. ## lifecycleHooks: {} - ## @param worker.extraEnvVars Array with extra environment variables to add to worker containers + ## @param worker.extraEnvVars Array with extra environment variables to add to worker containers. Only effective with worker.enableHelmDeployment. ## e.g: ## extraEnvVars: ## - name: FOO ## value: "bar" ## extraEnvVars: [] - ## @param worker.extraEnvVarsCM Name of existing ConfigMap containing extra env vars for worker containers + ## @param worker.extraEnvVarsCM Name of existing ConfigMap containing extra env vars for worker containers. Only effective with worker.enableHelmDeployment. ## extraEnvVarsCM: "" - ## @param worker.extraEnvVarsSecret Name of existing Secret containing extra env vars for worker containers + ## @param worker.extraEnvVarsSecret Name of existing Secret containing extra env vars for worker containers. Only effective with worker.enableHelmDeployment. ## extraEnvVarsSecret: "" - ## @param worker.extraVolumes Optionally specify extra list of additional volumes for the worker pods + ## @param worker.extraVolumes Optionally specify extra list of additional volumes for the worker pods. Only effective with worker.enableHelmDeployment. ## extraVolumes: [] - ## @param worker.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the worker containers + ## @param worker.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the worker containers. Only effective with worker.enableHelmDeployment. ## extraVolumeMounts: [] - ## @param worker.sidecars Add additional sidecar containers to the worker pods + ## @param worker.sidecars Add additional sidecar containers to the worker pods. Only effective with worker.enableHelmDeployment. ## e.g: ## sidecars: ## - name: your-image-name @@ -1627,7 +1613,7 @@ worker: ## containerPort: 1234 ## sidecars: [] - ## @param worker.initContainers Add additional init containers to the worker pods + ## @param worker.initContainers Add additional init containers to the worker pods. Only effective with worker.enableHelmDeployment. ## ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ ## e.g: ## initContainers: @@ -1639,7 +1625,7 @@ worker: initContainers: [] ## Pod Disruption Budget configuration ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb - ## @param worker.pdb.create Enable/disable a Pod Disruption Budget creation + ## @param worker.pdb.create Enable/disable a Pod Disruption Budget creation. Only effective with worker.enableHelmDeployment. ## @param worker.pdb.minAvailable Minimum number/percentage of pods that should remain scheduled ## @param worker.pdb.maxUnavailable Maximum number/percentage of pods that may be made unavailable. Defaults to `1` if both `worker.pdb.minAvailable` and `worker.pdb.maxUnavailable` are empty. ## @@ -1651,7 +1637,7 @@ worker: ## ref: https://kubernetes.io/docs/concepts/workloads/autoscaling/ ## autoscaling: - ## @param worker.autoscaling.vpa.enabled Enable VPA for worker pods + ## @param worker.autoscaling.vpa.enabled Enable VPA for worker pods. Only effective with worker.enableHelmDeployment. ## @param worker.autoscaling.vpa.annotations Annotations for VPA resource ## @param worker.autoscaling.vpa.controlledResources VPA List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory ## @param worker.autoscaling.vpa.maxAllowed VPA Max allowed resources for the pod @@ -1669,7 +1655,7 @@ worker: ## updatePolicy: updateMode: Auto - ## @param worker.autoscaling.hpa.enabled Enable HPA for worker pods + ## @param worker.autoscaling.hpa.enabled Enable HPA for worker pods. Only effective with worker.enableHelmDeployment. ## @param worker.autoscaling.hpa.minReplicas Minimum number of replicas ## @param worker.autoscaling.hpa.maxReplicas Maximum number of replicas ## @param worker.autoscaling.hpa.targetCPU Target CPU utilization percentage @@ -1685,7 +1671,7 @@ worker: ## Service configuration ## service: - ## @param worker.service.labels Extra labels for worker service + ## @param worker.service.labels Extra labels for worker service. Only effective with worker.enableHelmDeployment. ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ ## labels: {} @@ -1870,8 +1856,14 @@ app: ## @param app.exposedContainerPort The port that shuffle app containers will listen on for new requests. ## exposedContainerPort: 80 + ## @param app.sdkTimeout The timeout in seconds for app actions. + ## + sdkTimeout: 300 + ## @param app.disableLogs Do not capture app logs. By default, app logs are captured, so that they are visible in the frontend. + ## + disableLogs: false - ## @param app.deployViaHelm A list of apps that should be deployed using helm. + ## @param app.deployViaHelm [object] A list of apps that should be deployed using helm. ## By default, workers create deployments and services for apps when they are first needed (or during startup for some selected apps). ## Deploying apps via workers has some drawbacks, such as: ## - A workflow fails when the app is not deployed when the workflow gets executed (see https://github.com/Shuffle/Shuffle/issues/1739) @@ -1879,7 +1871,11 @@ app: ## - Worker needs elevated permissions in Kubernetes ## Note that you can deploy some apps via helm, while keeping the flexibility of letting workers deploy apps if they are not already deployed. ## If you deploy all needed apps via helm and dont want workers to create additional deployments, set worker.manageAppDeployments to false. - ## + ## The key of the app does not matter. + ## @extra app.deployViaHelm.MY_APP.app [string] The name of the app (required, e.g. shuffle-tools) + ## @extra app.deployViaHelm.MY_APP.version [string] The version of the app (required, e.g. 1.2.0) + + # TODO: Convert sample value to @extra. ## Sample value: ## - app: my-app # required ## version: 1.0.0 # required @@ -1892,10 +1888,9 @@ app: ## extraEnv: [] # optional ## podSecurityContext: null # optional, defaults to app.podSecurityContext ## containerSecurityContext: null # optional, defaults to app.containerSecurityContext - ## ## TODO: allow to override resources, serviceAccount, mounts, env, security Contexts. Fall back to app defaults as defined above. ## - deployViaHelm: [] + deployViaHelm: {} ## @section Traffic Exposure Parameters ## @@ -2192,6 +2187,8 @@ volumePermissions: ## OpenSearch chart configuration ## ref: https://github.com/bitnami/charts/blob/main/bitnami/opensearch/values.yaml ## @param opensearch.enabled Switch to enable or disable the opensearch helm chart +## @skip opensearch.sysctlImage +## @skip opensearch.image ## @skip opensearch.master ## @skip opensearch.data ## @skip opensearch.coordinating @@ -2200,15 +2197,12 @@ volumePermissions: ## opensearch: enabled: true - sysctlImage: enabled: false - image: registry: docker.io repository: bitnamilegacy/opensearch tag: "3.2.0" - master: replicaCount: 1 data: From 42c5ba22dd0c06dd09ee2a85e9a1a2bcbf60c190 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Mon, 1 Dec 2025 17:19:24 +0100 Subject: [PATCH 12/41] restore orborus image tag default Signed-off-by: Pascal Sthamer --- functions/kubernetes/charts/shuffle/README.md | 3 ++- functions/kubernetes/charts/shuffle/values.schema.json | 2 +- functions/kubernetes/charts/shuffle/values.yaml | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/README.md b/functions/kubernetes/charts/shuffle/README.md index 8d516904..5e3d7728 100644 --- a/functions/kubernetes/charts/shuffle/README.md +++ b/functions/kubernetes/charts/shuffle/README.md @@ -435,7 +435,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | ----------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------- | | `orborus.image.registry` | orborus image registry | `ghcr.io` | | `orborus.image.repository` | orborus image repository | `shuffle/shuffle-orborus` | -| `orborus.image.tag` | orborus image tag (immutable tags are recommended, defaults to appVersion) | `nightly` | +| `orborus.image.tag` | orborus image tag (immutable tags are recommended, defaults to appVersion) | `""` | | `orborus.image.digest` | orborus image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended) | `""` | | `orborus.image.pullPolicy` | orborus image pull policy | `IfNotPresent` | | `orborus.image.pullSecrets` | orborus image pull secrets | `[]` | @@ -781,3 +781,4 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia + diff --git a/functions/kubernetes/charts/shuffle/values.schema.json b/functions/kubernetes/charts/shuffle/values.schema.json index d997e81e..db8e428b 100644 --- a/functions/kubernetes/charts/shuffle/values.schema.json +++ b/functions/kubernetes/charts/shuffle/values.schema.json @@ -1471,7 +1471,7 @@ "tag": { "type": "string", "description": "orborus image tag (immutable tags are recommended, defaults to appVersion)", - "default": "nightly" + "default": "" }, "digest": { "type": "string", diff --git a/functions/kubernetes/charts/shuffle/values.yaml b/functions/kubernetes/charts/shuffle/values.yaml index c53e4943..a97e5168 100644 --- a/functions/kubernetes/charts/shuffle/values.yaml +++ b/functions/kubernetes/charts/shuffle/values.yaml @@ -960,7 +960,7 @@ orborus: image: registry: ghcr.io repository: shuffle/shuffle-orborus - tag: "nightly" + tag: "" digest: "" ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' From 21de3783f127af053e3cd7b256b08dca2e879bfa Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Mon, 1 Dec 2025 17:20:38 +0100 Subject: [PATCH 13/41] change version to 0.0.0 to indicate that it is not maintained in the file Signed-off-by: Pascal Sthamer --- functions/kubernetes/charts/shuffle/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/kubernetes/charts/shuffle/Chart.yaml b/functions/kubernetes/charts/shuffle/Chart.yaml index e286e11f..9cc97a23 100644 --- a/functions/kubernetes/charts/shuffle/Chart.yaml +++ b/functions/kubernetes/charts/shuffle/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: shuffle description: A Helm chart for deploying Shuffle on Kubernetes type: application -version: 2.1.0 # Set during publishing in GitHub actions +version: 0.0.0 # Set during publishing in GitHub actions appVersion: latest # Overwritten during publishing in GitHub actions dependencies: - name: common From 7654a6e1f9420e6d823053c268f12e73a2cf5038 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Mon, 1 Dec 2025 17:37:05 +0100 Subject: [PATCH 14/41] fix WORKER_HOSTNAME when not deploying via helm Signed-off-by: Pascal Sthamer --- .../charts/shuffle/templates/shuffle-worker/_helpers.tpl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl index 4ac59321..ee7c2917 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl @@ -99,7 +99,12 @@ IS_KUBERNETES: "true" KUBERNETES_NAMESPACE: "{{ .Release.Namespace }}" BASE_URL: {{ include "shuffle.backend.baseUrl" . | quote }} SHUFFLE_APP_EXPOSED_PORT: {{ .Values.app.exposedContainerPort | quote }} + +{{- if .Values.worker.enableHelmDeployment }} WORKER_HOSTNAME: "{{ include "shuffle.worker.name" . }}.{{ .Release.Namespace }}.svc.cluster.local" +{{- else }} +WORKER_HOSTNAME: "shuffle-workers.{{ .Release.Namespace }}.svc.cluster.local" +{{- end }} {{- if .Values.worker.manageAppDeployments }} # Shuffle app images From 78f5e6c0b9ce4c938136bb2ff45e0a7eaaab1db3 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Tue, 2 Dec 2025 10:09:42 +0100 Subject: [PATCH 15/41] use top-level apps value instead of app.deployViaHelm, add enabled toggle Signed-off-by: Pascal Sthamer --- functions/kubernetes/charts/shuffle/README.md | 51 +- .../kubernetes/charts/shuffle/values.yaml | 624 +++++++++++++++++- 2 files changed, 648 insertions(+), 27 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/README.md b/functions/kubernetes/charts/shuffle/README.md index 5e3d7728..71fa2563 100644 --- a/functions/kubernetes/charts/shuffle/README.md +++ b/functions/kubernetes/charts/shuffle/README.md @@ -131,30 +131,41 @@ If you want full control, you can deploy apps using helm. This has the following - granular control per app and version (e.g. have more replicas and resources for frequently used apps) - avoid problems with on-demand started apps (see https://github.com/Shuffle/Shuffle/issues/1739) -To deploy apps using helm, use the `app.deployViaHelm` value. Here is an example, which deploys the shuffle-tools app and 2 versions of the opensearch app: +To deploy apps using helm, set `apps.enabled=true`. By default, this deploys the `shuffle-tools`, `shuffle-subflow` and `http` apps. +You can also deploy your own apps. See the following values file for an example. ```yaml app: replicas: 1 # default to 1 replica per app - deployViaHelm: - tools: - app: shuffle-tools - version: 1.2.0 - opensearchOld: - app: opensearch - version: 1.0.0 # Also deploy OpenSearch version 1.0.0, because some workflows still use it. - opensearch: - app: opensearch - version: 1.1.0 - replicas: 3 # use 3 replicas for OpenSearch app - resources: # Custom resources for OpenSearch app - limits: - memory: 512Mi -``` -The key of an app in the `deployViaHelm` map does not matter. We are not using an array here, to allow overriding values in stage-specific value files or using the command line, e.g. -`helm upgrade ... --set app.deployViaHelm.tools.replicas=3`. + resources: {} # default resources for apps +# ... configure default options for all apps here -See the app Paramters section below for a complete list of helm values, that can be used to customize app deployments. -Take a look at the `app.deployViaHelm.MY_APP.*` parameters for a complete list of helm values that can be set on every app. +apps: + enabled: true # Deploy apps using helm. + +# Configure default apps + shuffleTools: + enabled: true # default + shuffleSubflow: + enabled: true # default + http: + enabled: true # default +# optionally override defaults from app values: + replicas: 1 + resources: {} + +# Deploy additional apps (e.g. opensearch) + opensearch: + enabled: true # required to actually deploy the app + name: opensearch # required. The name and version must match the values of the `api.yaml` file of the app. + version: 1.1.0 # required. +# optionally change app configuration: + replicas: 3 + resources: {} +``` +The key of an app in the `apps` map does not matter, as long as it is unique. We are not using an array here, to allow overriding values in stage-specific value files or using the command line, e.g. +`helm upgrade ... --set apps.shuffleTools.replicas=3`. + +See the "Parameters to deploy apps using helm" section below for a complete list of helm values, that can be used to customize app deployments. It is possible to use a hybrid approach - deploy some apps using helm, while still allowing Worker to create additional apps on-demand. diff --git a/functions/kubernetes/charts/shuffle/values.yaml b/functions/kubernetes/charts/shuffle/values.yaml index a97e5168..748cdd72 100644 --- a/functions/kubernetes/charts/shuffle/values.yaml +++ b/functions/kubernetes/charts/shuffle/values.yaml @@ -1735,7 +1735,7 @@ worker: ## extraEgress: [] - ## @param worker.manageAppDeployments Whether apps are deployed and managed by worker. When disabled, every used app is expected to to be already deployed (see app.deployViaHelm). + ## @param worker.manageAppDeployments Whether apps are deployed and managed by worker. When disabled, every used app is expected to to be already deployed (see apps.enabled). ## This effectively removes required RBAC permissions from the shuffle-worker service account to create deployments and services. ## The worker might still attempt to create kubernetes objects, resulting in an error. There is currently no way to tell the worker, that it should not manage k8s resources. manageAppDeployments: true @@ -1743,6 +1743,86 @@ worker: ## @section app Parameters ## app: + ## @param app.image.pullPolicy default image pull policy for app deployments. Only effective for helm-deployed apps (see apps.enabled). + ## @param app.image.pullSecrets default image pull secrets for app deployments. Only effective for helm-deployed apps (see apps.enabled). + ## + image: + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/concepts/containers/images/#pre-pulled-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## e.g: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## @param app.replicaCount Default number of replicas to deploy for each app. Only effective for helm-deployed apps (see apps.enabled). + ## + replicaCount: 1 + ## @param app.extraContainerPorts Optionally specify extra list of additional ports for app containers. Only effective for helm-deployed apps (see apps.enabled). + ## e.g: + ## extraContainerPorts: + ## - name: myservice + ## containerPort: 9090 + ## + extraContainerPorts: [] + ## Configure extra options for app containers' liveness and readiness probes + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes + ## @param app.livenessProbe.enabled Enable livenessProbe on app containers. Only effective for helm-deployed apps (see apps.enabled). + ## @param app.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe + ## @param app.livenessProbe.periodSeconds Period seconds for livenessProbe + ## @param app.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe + ## @param app.livenessProbe.failureThreshold Failure threshold for livenessProbe + ## @param app.livenessProbe.successThreshold Success threshold for livenessProbe + ## + livenessProbe: + enabled: false + initialDelaySeconds: 0 + periodSeconds: 15 + timeoutSeconds: 1 + failureThreshold: 4 + successThreshold: 1 + ## @param app.readinessProbe.enabled Enable readinessProbe on app containers. Only effective for helm-deployed apps (see apps.enabled). + ## @param app.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe + ## @param app.readinessProbe.periodSeconds Period seconds for readinessProbe + ## @param app.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe + ## @param app.readinessProbe.failureThreshold Failure threshold for readinessProbe + ## @param app.readinessProbe.successThreshold Success threshold for readinessProbe + ## + readinessProbe: + enabled: false + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + ## @param app.startupProbe.enabled Enable startupProbe on app containers. Only effective for helm-deployed apps (see apps.enabled). + ## @param app.startupProbe.initialDelaySeconds Initial delay seconds for startupProbe + ## @param app.startupProbe.periodSeconds Period seconds for startupProbe + ## @param app.startupProbe.timeoutSeconds Timeout seconds for startupProbe + ## @param app.startupProbe.failureThreshold Failure threshold for startupProbe + ## @param app.startupProbe.successThreshold Success threshold for startupProbe + ## + startupProbe: + enabled: false + initialDelaySeconds: 0 + periodSeconds: 1 + timeoutSeconds: 1 + failureThreshold: 60 + successThreshold: 1 + ## @param app.customLivenessProbe Custom livenessProbe that overrides the default one. Only effective for helm-deployed apps (see apps.enabled). + ## + customLivenessProbe: {} + ## @param app.customReadinessProbe Custom readinessProbe that overrides the default one. Only effective for helm-deployed apps (see apps.enabled). + ## + customReadinessProbe: {} + ## @param app.customStartupProbe Custom startupProbe that overrides the default one. Only effective for helm-deployed apps (see apps.enabled). + ## + customStartupProbe: {} ## app resource requests and limits ## ref: http://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ ## @param app.resourcesPreset Set app container resources according to one common preset (allowed values: none, nano, small, medium, large, xlarge, 2xlarge). This is ignored if app.resources is set (app.resources is recommended for production). @@ -1802,7 +1882,183 @@ app: drop: ["ALL"] seccompProfile: type: "RuntimeDefault" - + + ## @param app.automountServiceAccountToken Mount Service Account token in app pods. Only effective for helm-deployed apps (see apps.enabled). + ## + automountServiceAccountToken: false + + ## @param app.hostAliases app pods host aliases. Only effective for helm-deployed apps (see apps.enabled). + ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ + ## + hostAliases: [] + ## @param app.deploymentAnnotations Annotations for app deployment. Only effective for helm-deployed apps (see apps.enabled). + ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ + ## + deploymentAnnotations: {} + ## @param app.podLabels Extra labels for app pods. Only effective for helm-deployed apps (see apps.enabled). + ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + ## + podLabels: {} + ## @param app.podAnnotations Annotations for app pods. Only effective for helm-deployed apps (see apps.enabled). + ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ + ## + podAnnotations: {} + ## @param app.podAffinityPreset Pod affinity preset. Ignored if `app.affinity` is set. Allowed values: `soft` or `hard`. Only effective for helm-deployed apps (see apps.enabled). + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity + ## + podAffinityPreset: "" + ## @param app.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `app.affinity` is set. Allowed values: `soft` or `hard`. Only effective for helm-deployed apps (see apps.enabled). + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity + ## + podAntiAffinityPreset: soft + ## Node app.affinity preset + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity + ## + nodeAffinityPreset: + ## @param app.nodeAffinityPreset.type Node affinity preset type. Ignored if `app.affinity` is set. Allowed values: `soft` or `hard`. Only effective for helm-deployed apps (see apps.enabled). + ## + type: "" + ## @param app.nodeAffinityPreset.key Node label key to match. Ignored if `app.affinity` is set + ## + key: "" + ## @param app.nodeAffinityPreset.values Node label values to match. Ignored if `app.affinity` is set + ## E.g. + ## values: + ## - e2e-az1 + ## - e2e-az2 + ## + values: [] + ## @param app.affinity Affinity for app pods assignment. Only effective for helm-deployed apps (see apps.enabled). + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## NOTE: `app.podAffinityPreset`, `app.podAntiAffinityPreset`, and `app.nodeAffinityPreset` will be ignored when it's set + ## + affinity: {} + ## @param app.nodeSelector Node labels for app pods assignment. Only effective for helm-deployed apps (see apps.enabled). + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/ + ## + nodeSelector: {} + ## @param app.tolerations Tolerations for app pods assignment. Only effective for helm-deployed apps (see apps.enabled). + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + ## @param app.updateStrategy.type app deployment strategy type. Only effective for helm-deployed apps (see apps.enabled). + ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy + ## + updateStrategy: + ## Can be set to RollingUpdate or Recreate + ## + type: RollingUpdate + ## @param app.priorityClassName app pods' priorityClassName. Only effective for helm-deployed apps (see apps.enabled). + ## + priorityClassName: "" + ## @param app.topologySpreadConstraints Topology Spread Constraints for app pod assignment spread across your cluster among failure-domains. Only effective for helm-deployed apps (see apps.enabled). + ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods + ## + topologySpreadConstraints: [] + ## @param app.schedulerName Name of the k8s scheduler (other than default) for app pods. Only effective for helm-deployed apps (see apps.enabled). + ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ + ## + schedulerName: "" + ## @param app.terminationGracePeriodSeconds Seconds app pods need to terminate gracefully. Only effective for helm-deployed apps (see apps.enabled). + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods + ## + terminationGracePeriodSeconds: "" + ## @param app.lifecycleHooks for app containers to automate configuration before or after startup. Only effective for helm-deployed apps (see apps.enabled). + ## + lifecycleHooks: {} + ## @param app.extraEnvVars Array with extra environment variables to add to app containers. Only effective for helm-deployed apps (see apps.enabled). + ## e.g: + ## extraEnvVars: + ## - name: FOO + ## value: "bar" + ## + extraEnvVars: [] + ## @param app.extraEnvVarsCM Name of existing ConfigMap containing extra env vars for app containers. Only effective for helm-deployed apps (see apps.enabled). + ## + extraEnvVarsCM: "" + ## @param app.extraEnvVarsSecret Name of existing Secret containing extra env vars for app containers. Only effective for helm-deployed apps (see apps.enabled). + ## + extraEnvVarsSecret: "" + ## @param app.extraVolumes Optionally specify extra list of additional volumes for the app pods. Only effective for helm-deployed apps (see apps.enabled). + ## + extraVolumes: [] + ## @param app.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the app containers. Only effective for helm-deployed apps (see apps.enabled). + ## + extraVolumeMounts: [] + ## @param app.sidecars Add additional sidecar containers to the app pods. Only effective for helm-deployed apps (see apps.enabled). + ## e.g: + ## sidecars: + ## - name: your-image-name + ## image: your-image + ## imagePullPolicy: Always + ## ports: + ## - name: portname + ## containerPort: 1234 + ## + sidecars: [] + ## @param app.initContainers Add additional init containers to the app pods. Only effective for helm-deployed apps (see apps.enabled). + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ + ## e.g: + ## initContainers: + ## - name: your-image-name + ## image: your-image + ## imagePullPolicy: Always + ## command: ['sh', '-c', 'echo "hello world"'] + ## + initContainers: [] + ## Pod Disruption Budget configuration + ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb + ## @param app.pdb.create Enable/disable a Pod Disruption Budget creation. Only effective for helm-deployed apps (see apps.enabled). + ## @param app.pdb.minAvailable Minimum number/percentage of pods that should remain scheduled + ## @param app.pdb.maxUnavailable Maximum number/percentage of pods that may be made unavailable. Defaults to `1` if both `app.pdb.minAvailable` and `app.pdb.maxUnavailable` are empty. + ## + pdb: + create: true + minAvailable: "" + maxUnavailable: "" + ## Autoscaling configuration + ## ref: https://kubernetes.io/docs/concepts/workloads/autoscaling/ + ## + autoscaling: + ## @param app.autoscaling.vpa.enabled Enable VPA for app pods. Only effective for helm-deployed apps (see apps.enabled). + ## @param app.autoscaling.vpa.annotations Annotations for VPA resource + ## @param app.autoscaling.vpa.controlledResources VPA List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory + ## @param app.autoscaling.vpa.maxAllowed VPA Max allowed resources for the pod + ## @param app.autoscaling.vpa.minAllowed VPA Min allowed resources for the pod + ## + vpa: + enabled: false + annotations: {} + controlledResources: [] + maxAllowed: {} + minAllowed: {} + ## @param app.autoscaling.vpa.updatePolicy.updateMode Autoscaling update policy + ## Specifies whether recommended updates are applied when a Pod is started and whether recommended updates are applied during the life of a Pod + ## Possible values are "Off", "Initial", "Recreate", and "Auto". + ## + updatePolicy: + updateMode: Auto + ## @param app.autoscaling.hpa.enabled Enable HPA for app pods. Only effective for helm-deployed apps (see apps.enabled). + ## @param app.autoscaling.hpa.minReplicas Minimum number of replicas + ## @param app.autoscaling.hpa.maxReplicas Maximum number of replicas + ## @param app.autoscaling.hpa.targetCPU Target CPU utilization percentage + ## @param app.autoscaling.hpa.targetMemory Target Memory utilization percentage + ## + hpa: + enabled: false + minReplicas: "" + maxReplicas: "" + targetCPU: "" + targetMemory: "" + + ## Service configuration + ## + service: + ## @param app.service.labels Extra labels for app service. Only effective for helm-deployed apps (see apps.enabled). + ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + ## + labels: {} + ## ServiceAccount configuration ## serviceAccount: @@ -1863,7 +2119,10 @@ app: ## disableLogs: false - ## @param app.deployViaHelm [object] A list of apps that should be deployed using helm. +## @section Parameters to deploy apps using helm +## +apps: + ## @param apps.enabled Whether apps should be deployed using helm. ## By default, workers create deployments and services for apps when they are first needed (or during startup for some selected apps). ## Deploying apps via workers has some drawbacks, such as: ## - A workflow fails when the app is not deployed when the workflow gets executed (see https://github.com/Shuffle/Shuffle/issues/1739) @@ -1871,9 +2130,361 @@ app: ## - Worker needs elevated permissions in Kubernetes ## Note that you can deploy some apps via helm, while keeping the flexibility of letting workers deploy apps if they are not already deployed. ## If you deploy all needed apps via helm and dont want workers to create additional deployments, set worker.manageAppDeployments to false. - ## The key of the app does not matter. - ## @extra app.deployViaHelm.MY_APP.app [string] The name of the app (required, e.g. shuffle-tools) - ## @extra app.deployViaHelm.MY_APP.version [string] The version of the app (required, e.g. 1.2.0) + ## + enabled: false + + shuffleTools: + ## @param apps.shuffleTools.enabled Whether the app is enabled + ## + enabled: true + ## app image + ## @param apps.shuffleTools.image.registry app image registry + ## @param apps.shuffleTools.image.repository app image repository + ## @param apps.shuffleTools.image.tag app image tag (immutable tags are recommended, defaults to appVersion) + ## @param apps.shuffleTools.image.digest app image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended) + ## @param apps.shuffleTools.image.pullPolicy app image pull policy + ## @param apps.shuffleTools.image.pullSecrets app image pull secrets + ## + image: + registry: ghcr.io + repository: shuffle/shuffle-app + tag: "" + digest: "" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/concepts/containers/images/#pre-pulled-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## e.g: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## @param apps.shuffleTools.replicaCount [number, nullable] Number of app replicas to deploy + ## + replicaCount: null + ## @param apps.shuffleTools.extraContainerPorts Optionally specify extra list of additional ports for app containers + ## e.g: + ## extraContainerPorts: + ## - name: myservice + ## containerPort: 9090 + ## + extraContainerPorts: [] + ## Configure extra options for app containers' liveness and readiness probes + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes + ## @param apps.shuffleTools.livenessProbe.enabled Enable livenessProbe on app containers + ## @param apps.shuffleTools.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe + ## @param apps.shuffleTools.livenessProbe.periodSeconds Period seconds for livenessProbe + ## @param apps.shuffleTools.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe + ## @param apps.shuffleTools.livenessProbe.failureThreshold Failure threshold for livenessProbe + ## @param apps.shuffleTools.livenessProbe.successThreshold Success threshold for livenessProbe + ## + livenessProbe: + enabled: false + initialDelaySeconds: 0 + periodSeconds: 15 + timeoutSeconds: 1 + failureThreshold: 4 + successThreshold: 1 + ## @param apps.shuffleTools.readinessProbe.enabled Enable readinessProbe on app containers + ## @param apps.shuffleTools.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe + ## @param apps.shuffleTools.readinessProbe.periodSeconds Period seconds for readinessProbe + ## @param apps.shuffleTools.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe + ## @param apps.shuffleTools.readinessProbe.failureThreshold Failure threshold for readinessProbe + ## @param apps.shuffleTools.readinessProbe.successThreshold Success threshold for readinessProbe + ## + readinessProbe: + enabled: false + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + ## @param apps.shuffleTools.startupProbe.enabled Enable startupProbe on app containers + ## @param apps.shuffleTools.startupProbe.initialDelaySeconds Initial delay seconds for startupProbe + ## @param apps.shuffleTools.startupProbe.periodSeconds Period seconds for startupProbe + ## @param apps.shuffleTools.startupProbe.timeoutSeconds Timeout seconds for startupProbe + ## @param apps.shuffleTools.startupProbe.failureThreshold Failure threshold for startupProbe + ## @param apps.shuffleTools.startupProbe.successThreshold Success threshold for startupProbe + ## + startupProbe: + enabled: false + initialDelaySeconds: 0 + periodSeconds: 1 + timeoutSeconds: 1 + failureThreshold: 60 + successThreshold: 1 + ## @param apps.shuffleTools.customLivenessProbe Custom livenessProbe that overrides the default one + ## + customLivenessProbe: {} + ## @param apps.shuffleTools.customReadinessProbe Custom readinessProbe that overrides the default one + ## + customReadinessProbe: {} + ## @param apps.shuffleTools.customStartupProbe Custom startupProbe that overrides the default one + ## + customStartupProbe: {} + ## app resource requests and limits + ## ref: http://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + ## @param apps.shuffleTools.resourcesPreset Set app container resources according to one common preset (allowed values: none, nano, small, medium, large, xlarge, 2xlarge). This is ignored if app.resources is set (app.resources is recommended for production). + ## More information: https://github.com/bitnami/charts/blob/main/bitnami/common/templates/_resources.tpl#L15 + ## Shuffle gets OOM killed with 256M memory during startup. Up to 360MiB of memory usage were observed during testing. + ## The small preset grants 512M. + ## + resourcesPreset: "small" + ## @param apps.shuffleTools.resources Set app container requests and limits for different resources like CPU or memory (essential for production workloads) + ## Example: + ## resources: + ## requests: + ## cpu: 2 + ## memory: 512Mi + ## limits: + ## cpu: 3 + ## memory: 1024Mi + ## + resources: {} + ## Configure Pods Security Context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod + ## @param apps.shuffleTools.podSecurityContext.enabled Enable app pods' Security Context + ## @param apps.shuffleTools.podSecurityContext.fsGroupChangePolicy Set filesystem group change policy for app pods + ## @param apps.shuffleTools.podSecurityContext.sysctls Set kernel settings using the sysctl interface for app pods + ## @param apps.shuffleTools.podSecurityContext.supplementalGroups Set filesystem extra groups for app pods + ## @param apps.shuffleTools.podSecurityContext.fsGroup Set fsGroup in app pods' Security Context + ## + podSecurityContext: + enabled: true + fsGroupChangePolicy: Always + sysctls: [] + supplementalGroups: [] + fsGroup: 1001 + ## Configure Container Security Context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + ## @param apps.shuffleTools.containerSecurityContext.enabled Enabled app container' Security Context + ## @param apps.shuffleTools.containerSecurityContext.seLinuxOptions [object,nullable] Set SELinux options in app container + ## @param apps.shuffleTools.containerSecurityContext.runAsUser Set runAsUser in app container' Security Context + ## @param apps.shuffleTools.containerSecurityContext.runAsGroup Set runAsGroup in app container' Security Context + ## @param apps.shuffleTools.containerSecurityContext.runAsNonRoot Set runAsNonRoot in app container' Security Context + ## @param apps.shuffleTools.containerSecurityContext.readOnlyRootFilesystem Set readOnlyRootFilesystem in app container' Security Context + ## @param apps.shuffleTools.containerSecurityContext.privileged Set privileged in app container' Security Context + ## @param apps.shuffleTools.containerSecurityContext.allowPrivilegeEscalation Set allowPrivilegeEscalation in app container' Security Context + ## @param apps.shuffleTools.containerSecurityContext.capabilities.drop List of capabilities to be dropped in app container + ## @param apps.shuffleTools.containerSecurityContext.seccompProfile.type Set seccomp profile in app container + ## + containerSecurityContext: + enabled: true + seLinuxOptions: {} + runAsUser: 1001 + runAsGroup: 1001 + runAsNonRoot: true + readOnlyRootFilesystem: true + privileged: false + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: "RuntimeDefault" + ## @param apps.shuffleTools.command Override default app container command (useful when using custom images) + ## + command: [] + ## @param apps.shuffleTools.args Override default app container args (useful when using custom images) + ## + args: [] + ## @param apps.shuffleTools.automountServiceAccountToken Mount Service Account token in app pods + ## NOTE: app requires the service account credentials to be mounted + ## + automountServiceAccountToken: true + ## @param apps.shuffleTools.hostAliases app pods host aliases + ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ + ## + hostAliases: [] + ## @param apps.shuffleTools.deploymentAnnotations Annotations for app deployment + ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ + ## + deploymentAnnotations: {} + ## @param apps.shuffleTools.podLabels Extra labels for app pods + ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + ## + podLabels: {} + ## @param apps.shuffleTools.podAnnotations Annotations for app pods + ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ + ## + podAnnotations: {} + ## @param apps.shuffleTools.podAffinityPreset Pod affinity preset. Ignored if `app.affinity` is set. Allowed values: `soft` or `hard` + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity + ## + podAffinityPreset: "" + ## @param apps.shuffleTools.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `app.affinity` is set. Allowed values: `soft` or `hard` + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity + ## + podAntiAffinityPreset: soft + ## Node app.affinity preset + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity + ## + nodeAffinityPreset: + ## @param apps.shuffleTools.nodeAffinityPreset.type Node affinity preset type. Ignored if `app.affinity` is set. Allowed values: `soft` or `hard` + ## + type: "" + ## @param apps.shuffleTools.nodeAffinityPreset.key Node label key to match. Ignored if `app.affinity` is set + ## + key: "" + ## @param apps.shuffleTools.nodeAffinityPreset.values Node label values to match. Ignored if `app.affinity` is set + ## E.g. + ## values: + ## - e2e-az1 + ## - e2e-az2 + ## + values: [] + ## @param apps.shuffleTools.affinity Affinity for app pods assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## NOTE: `app.podAffinityPreset`, `app.podAntiAffinityPreset`, and `app.nodeAffinityPreset` will be ignored when it's set + ## + affinity: {} + ## @param apps.shuffleTools.nodeSelector Node labels for app pods assignment + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/ + ## + nodeSelector: {} + ## @param apps.shuffleTools.tolerations Tolerations for app pods assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + ## @param apps.shuffleTools.updateStrategy.type app deployment strategy type + ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy + ## + updateStrategy: + ## Can be set to RollingUpdate or Recreate + ## app uses ReadWriteOnce volumes by default, which is incompatible with RollingUpdate + ## + type: Recreate + ## @param apps.shuffleTools.priorityClassName app pods' priorityClassName + ## + priorityClassName: "" + ## @param apps.shuffleTools.topologySpreadConstraints Topology Spread Constraints for app pod assignment spread across your cluster among failure-domains + ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods + ## + topologySpreadConstraints: [] + ## @param apps.shuffleTools.schedulerName Name of the k8s scheduler (other than default) for app pods + ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ + ## + schedulerName: "" + ## @param apps.shuffleTools.terminationGracePeriodSeconds Seconds app pods need to terminate gracefully + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods + ## + terminationGracePeriodSeconds: "" + ## @param apps.shuffleTools.lifecycleHooks for app containers to automate configuration before or after startup + ## + lifecycleHooks: {} + ## @param apps.shuffleTools.extraEnvVars Array with extra environment variables to add to app containers + ## e.g: + ## extraEnvVars: + ## - name: FOO + ## value: "bar" + ## + extraEnvVars: [] + ## @param apps.shuffleTools.extraEnvVarsCM Name of existing ConfigMap containing extra env vars for app containers + ## + extraEnvVarsCM: "" + ## @param apps.shuffleTools.extraEnvVarsSecret Name of existing Secret containing extra env vars for app containers + ## + extraEnvVarsSecret: "" + ## @param apps.shuffleTools.extraVolumes Optionally specify extra list of additional volumes for the app pods + ## + extraVolumes: [] + ## @param apps.shuffleTools.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the app containers + ## + extraVolumeMounts: [] + ## @param apps.shuffleTools.sidecars Add additional sidecar containers to the app pods + ## e.g: + ## sidecars: + ## - name: your-image-name + ## image: your-image + ## imagePullPolicy: Always + ## ports: + ## - name: portname + ## containerPort: 1234 + ## + sidecars: [] + ## @param apps.shuffleTools.initContainers Add additional init containers to the app pods + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ + ## e.g: + ## initContainers: + ## - name: your-image-name + ## image: your-image + ## imagePullPolicy: Always + ## command: ['sh', '-c', 'echo "hello world"'] + ## + initContainers: [] + ## Pod Disruption Budget configuration + ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb + ## @param apps.shuffleTools.pdb.create Enable/disable a Pod Disruption Budget creation + ## @param apps.shuffleTools.pdb.minAvailable Minimum number/percentage of pods that should remain scheduled + ## @param apps.shuffleTools.pdb.maxUnavailable Maximum number/percentage of pods that may be made unavailable. Defaults to `1` if both `app.pdb.minAvailable` and `app.pdb.maxUnavailable` are empty. + ## + pdb: + create: true + minAvailable: "" + maxUnavailable: "" + ## Autoscaling configuration + ## ref: https://kubernetes.io/docs/concepts/workloads/autoscaling/ + ## + autoscaling: + ## @param apps.shuffleTools.autoscaling.vpa.enabled Enable VPA for app pods + ## @param apps.shuffleTools.autoscaling.vpa.annotations Annotations for VPA resource + ## @param apps.shuffleTools.autoscaling.vpa.controlledResources VPA List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory + ## @param apps.shuffleTools.autoscaling.vpa.maxAllowed VPA Max allowed resources for the pod + ## @param apps.shuffleTools.autoscaling.vpa.minAllowed VPA Min allowed resources for the pod + ## + vpa: + enabled: false + annotations: {} + controlledResources: [] + maxAllowed: {} + minAllowed: {} + ## @param apps.shuffleTools.autoscaling.vpa.updatePolicy.updateMode Autoscaling update policy + ## Specifies whether recommended updates are applied when a Pod is started and whether recommended updates are applied during the life of a Pod + ## Possible values are "Off", "Initial", "Recreate", and "Auto". + ## + updatePolicy: + updateMode: Auto + ## @param apps.shuffleTools.autoscaling.hpa.enabled Enable HPA for app pods + ## @param apps.shuffleTools.autoscaling.hpa.minReplicas Minimum number of replicas + ## @param apps.shuffleTools.autoscaling.hpa.maxReplicas Maximum number of replicas + ## @param apps.shuffleTools.autoscaling.hpa.targetCPU Target CPU utilization percentage + ## @param apps.shuffleTools.autoscaling.hpa.targetMemory Target Memory utilization percentage + ## + hpa: + enabled: false + minReplicas: "" + maxReplicas: "" + targetCPU: "" + targetMemory: "" + + ## Service configuration + ## + service: + ## @param apps.shuffleTools.service.labels Extra labels for app service + ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + ## + labels: {} + + shuffleSubflow: + enabled: true + + http: + enabled: true + + # replicas, image, etc. + + ## @extra apps.MY_APP.app [string] The name of the app (required, e.g. shuffle-tools) + ## @extra apps.MY_APP.version [string] The version of the app (required, e.g. 1.2.0) + ## TODO: Add more extras + ## Add your own apps here. The key of the app does not matter, as long as it is unique. + ## myApp: + ## enabled: true + ## name: my-app + ## version: 1.0.0 + ## ... # TODO: Convert sample value to @extra. ## Sample value: @@ -1890,7 +2501,6 @@ app: ## containerSecurityContext: null # optional, defaults to app.containerSecurityContext ## TODO: allow to override resources, serviceAccount, mounts, env, security Contexts. Fall back to app defaults as defined above. ## - deployViaHelm: {} ## @section Traffic Exposure Parameters ## From b6376cf967c6be7edf4aeedb24eba1a6f8fbd440 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Tue, 2 Dec 2025 12:57:51 +0100 Subject: [PATCH 16/41] app deployment using helm with full per-app customization Signed-off-by: Pascal Sthamer --- functions/kubernetes/charts/shuffle/README.md | 101 +++- .../templates/shuffle-app/_helpers.tpl | 2 + .../shuffle-app/shuffle-app-deployments.yaml | 17 - .../templates/shuffle-app/shuffle-apps.yaml | 306 +++++++++++ .../templates/shuffle-worker/_helpers.tpl | 3 + .../charts/shuffle/values.schema.json | 491 +++++++++++++++++- .../kubernetes/charts/shuffle/values.yaml | 387 ++------------ 7 files changed, 933 insertions(+), 374 deletions(-) delete mode 100644 functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-deployments.yaml create mode 100644 functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml diff --git a/functions/kubernetes/charts/shuffle/README.md b/functions/kubernetes/charts/shuffle/README.md index 71fa2563..0535cb9f 100644 --- a/functions/kubernetes/charts/shuffle/README.md +++ b/functions/kubernetes/charts/shuffle/README.md @@ -135,7 +135,7 @@ To deploy apps using helm, set `apps.enabled=true`. By default, this deploys the You can also deploy your own apps. See the following values file for an example. ```yaml app: - replicas: 1 # default to 1 replica per app + replicaCount: 1 # default to 1 replica per app resources: {} # default resources for apps # ... configure default options for all apps here @@ -150,7 +150,7 @@ apps: http: enabled: true # default # optionally override defaults from app values: - replicas: 1 + replicaCount: 1 resources: {} # Deploy additional apps (e.g. opensearch) @@ -159,13 +159,14 @@ apps: name: opensearch # required. The name and version must match the values of the `api.yaml` file of the app. version: 1.1.0 # required. # optionally change app configuration: - replicas: 3 + replicaCount: 3 resources: {} ``` The key of an app in the `apps` map does not matter, as long as it is unique. We are not using an array here, to allow overriding values in stage-specific value files or using the command line, e.g. `helm upgrade ... --set apps.shuffleTools.replicas=3`. -See the "Parameters to deploy apps using helm" section below for a complete list of helm values, that can be used to customize app deployments. +You can override any value set in `app.*` (e.g. `app.image`, `app.replicaCount`, `app.resources`, `app.podSecurityContext`) for each app +(e.g. for the `shuffle-tools` app using `apps.shuffleTools.image`, `apps.shuffleTools.replicaCount`, ...). It is possible to use a hybrid approach - deploy some apps using helm, while still allowing Worker to create additional apps on-demand. @@ -651,12 +652,40 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `worker.networkPolicy.allowExternalEgress` | Allow the pod to access any range of port and all destinations. | `true` | | `worker.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` | | `worker.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | -| `worker.manageAppDeployments` | Whether apps are deployed and managed by worker. When disabled, every used app is expected to to be already deployed (see app.deployViaHelm). | `true` | +| `worker.manageAppDeployments` | Whether apps are deployed and managed by worker. When disabled, every used app is expected to to be already deployed (see apps.enabled). | `true` | ### app Parameters | Name | Description | Value | | ------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | +| `app.image.registry` | app image registry (defaults to shuffle.appRegistry) | `""` | +| `app.image.repository` | app image repository (defaults to shuffle.appBaseImageName) | `""` | +| `app.image.tag` | app image tag (defaults to the apps version) | `""` | +| `app.image.pullPolicy` | default image pull policy for app deployments. Only effective for helm-deployed apps (see apps.enabled). | `IfNotPresent` | +| `app.image.pullSecrets` | default image pull secrets for app deployments. Only effective for helm-deployed apps (see apps.enabled). | `[]` | +| `app.replicaCount` | Default number of replicas to deploy for each app. Only effective for helm-deployed apps (see apps.enabled). | `1` | +| `app.extraContainerPorts` | Optionally specify extra list of additional ports for app containers. Only effective for helm-deployed apps (see apps.enabled). | `[]` | +| `app.livenessProbe.enabled` | Enable livenessProbe on app containers. Only effective for helm-deployed apps (see apps.enabled). | `false` | +| `app.livenessProbe.initialDelaySeconds` | Initial delay seconds for livenessProbe | `0` | +| `app.livenessProbe.periodSeconds` | Period seconds for livenessProbe | `15` | +| `app.livenessProbe.timeoutSeconds` | Timeout seconds for livenessProbe | `1` | +| `app.livenessProbe.failureThreshold` | Failure threshold for livenessProbe | `4` | +| `app.livenessProbe.successThreshold` | Success threshold for livenessProbe | `1` | +| `app.readinessProbe.enabled` | Enable readinessProbe on app containers. Only effective for helm-deployed apps (see apps.enabled). | `false` | +| `app.readinessProbe.initialDelaySeconds` | Initial delay seconds for readinessProbe | `0` | +| `app.readinessProbe.periodSeconds` | Period seconds for readinessProbe | `5` | +| `app.readinessProbe.timeoutSeconds` | Timeout seconds for readinessProbe | `1` | +| `app.readinessProbe.failureThreshold` | Failure threshold for readinessProbe | `3` | +| `app.readinessProbe.successThreshold` | Success threshold for readinessProbe | `1` | +| `app.startupProbe.enabled` | Enable startupProbe on app containers. Only effective for helm-deployed apps (see apps.enabled). | `false` | +| `app.startupProbe.initialDelaySeconds` | Initial delay seconds for startupProbe | `0` | +| `app.startupProbe.periodSeconds` | Period seconds for startupProbe | `1` | +| `app.startupProbe.timeoutSeconds` | Timeout seconds for startupProbe | `1` | +| `app.startupProbe.failureThreshold` | Failure threshold for startupProbe | `60` | +| `app.startupProbe.successThreshold` | Success threshold for startupProbe | `1` | +| `app.customLivenessProbe` | Custom livenessProbe that overrides the default one. Only effective for helm-deployed apps (see apps.enabled). | `{}` | +| `app.customReadinessProbe` | Custom readinessProbe that overrides the default one. Only effective for helm-deployed apps (see apps.enabled). | `{}` | +| `app.customStartupProbe` | Custom startupProbe that overrides the default one. Only effective for helm-deployed apps (see apps.enabled). | `{}` | | `app.resourcesPreset` | Set app container resources according to one common preset (allowed values: none, nano, small, medium, large, xlarge, 2xlarge). This is ignored if app.resources is set (app.resources is recommended for production). | `nano` | | `app.resources` | Set app container requests and limits for different resources like CPU or memory (essential for production workloads) | `{}` | | `app.podSecurityContext.enabled` | Enable app pods' Security Context | `true` | @@ -674,6 +703,49 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `app.containerSecurityContext.allowPrivilegeEscalation` | Set allowPrivilegeEscalation in app container' Security Context | `false` | | `app.containerSecurityContext.capabilities.drop` | List of capabilities to be dropped in app container | `["ALL"]` | | `app.containerSecurityContext.seccompProfile.type` | Set seccomp profile in app container | `RuntimeDefault` | +| `app.command` | Override default app container command (useful when using custom images) | `[]` | +| `app.args` | Override default app container args (useful when using custom images) | `[]` | +| `app.automountServiceAccountToken` | Mount Service Account token in app pods. Only effective for helm-deployed apps (see apps.enabled). | `false` | +| `app.hostAliases` | app pods host aliases. Only effective for helm-deployed apps (see apps.enabled). | `[]` | +| `app.deploymentAnnotations` | Annotations for app deployment. Only effective for helm-deployed apps (see apps.enabled). | `{}` | +| `app.podLabels` | Extra labels for app pods. Only effective for helm-deployed apps (see apps.enabled). | `{}` | +| `app.podAnnotations` | Annotations for app pods. Only effective for helm-deployed apps (see apps.enabled). | `{}` | +| `app.podAffinityPreset` | Pod affinity preset. Ignored if `app.affinity` is set. Allowed values: `soft` or `hard`. Only effective for helm-deployed apps (see apps.enabled). | `""` | +| `app.podAntiAffinityPreset` | Pod anti-affinity preset. Ignored if `app.affinity` is set. Allowed values: `soft` or `hard`. Only effective for helm-deployed apps (see apps.enabled). | `soft` | +| `app.nodeAffinityPreset.type` | Node affinity preset type. Ignored if `app.affinity` is set. Allowed values: `soft` or `hard`. Only effective for helm-deployed apps (see apps.enabled). | `""` | +| `app.nodeAffinityPreset.key` | Node label key to match. Ignored if `app.affinity` is set | `""` | +| `app.nodeAffinityPreset.values` | Node label values to match. Ignored if `app.affinity` is set | `[]` | +| `app.affinity` | Affinity for app pods assignment. Only effective for helm-deployed apps (see apps.enabled). | `{}` | +| `app.nodeSelector` | Node labels for app pods assignment. Only effective for helm-deployed apps (see apps.enabled). | `{}` | +| `app.tolerations` | Tolerations for app pods assignment. Only effective for helm-deployed apps (see apps.enabled). | `[]` | +| `app.updateStrategy.type` | app deployment strategy type. Only effective for helm-deployed apps (see apps.enabled). | `RollingUpdate` | +| `app.priorityClassName` | app pods' priorityClassName. Only effective for helm-deployed apps (see apps.enabled). | `""` | +| `app.topologySpreadConstraints` | Topology Spread Constraints for app pod assignment spread across your cluster among failure-domains. Only effective for helm-deployed apps (see apps.enabled). | `[]` | +| `app.schedulerName` | Name of the k8s scheduler (other than default) for app pods. Only effective for helm-deployed apps (see apps.enabled). | `""` | +| `app.terminationGracePeriodSeconds` | Seconds app pods need to terminate gracefully. Only effective for helm-deployed apps (see apps.enabled). | `""` | +| `app.lifecycleHooks` | for app containers to automate configuration before or after startup. Only effective for helm-deployed apps (see apps.enabled). | `{}` | +| `app.extraEnvVars` | Array with extra environment variables to add to app containers. Only effective for helm-deployed apps (see apps.enabled). | `[]` | +| `app.extraEnvVarsCM` | Name of existing ConfigMap containing extra env vars for app containers. Only effective for helm-deployed apps (see apps.enabled). | `""` | +| `app.extraEnvVarsSecret` | Name of existing Secret containing extra env vars for app containers. Only effective for helm-deployed apps (see apps.enabled). | `""` | +| `app.extraVolumes` | Optionally specify extra list of additional volumes for the app pods. Only effective for helm-deployed apps (see apps.enabled). | `[]` | +| `app.extraVolumeMounts` | Optionally specify extra list of additional volumeMounts for the app containers. Only effective for helm-deployed apps (see apps.enabled). | `[]` | +| `app.sidecars` | Add additional sidecar containers to the app pods. Only effective for helm-deployed apps (see apps.enabled). | `[]` | +| `app.initContainers` | Add additional init containers to the app pods. Only effective for helm-deployed apps (see apps.enabled). | `[]` | +| `app.pdb.create` | Enable/disable a Pod Disruption Budget creation. Only effective for helm-deployed apps (see apps.enabled). | `true` | +| `app.pdb.minAvailable` | Minimum number/percentage of pods that should remain scheduled | `""` | +| `app.pdb.maxUnavailable` | Maximum number/percentage of pods that may be made unavailable. Defaults to `1` if both `app.pdb.minAvailable` and `app.pdb.maxUnavailable` are empty. | `""` | +| `app.autoscaling.vpa.enabled` | Enable VPA for app pods. Only effective for helm-deployed apps (see apps.enabled). | `false` | +| `app.autoscaling.vpa.annotations` | Annotations for VPA resource | `{}` | +| `app.autoscaling.vpa.controlledResources` | VPA List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory | `[]` | +| `app.autoscaling.vpa.maxAllowed` | VPA Max allowed resources for the pod | `{}` | +| `app.autoscaling.vpa.minAllowed` | VPA Min allowed resources for the pod | `{}` | +| `app.autoscaling.vpa.updatePolicy.updateMode` | Autoscaling update policy | `Auto` | +| `app.autoscaling.hpa.enabled` | Enable HPA for app pods. Only effective for helm-deployed apps (see apps.enabled). | `false` | +| `app.autoscaling.hpa.minReplicas` | Minimum number of replicas | `""` | +| `app.autoscaling.hpa.maxReplicas` | Maximum number of replicas | `""` | +| `app.autoscaling.hpa.targetCPU` | Target CPU utilization percentage | `""` | +| `app.autoscaling.hpa.targetMemory` | Target Memory utilization percentage | `""` | +| `app.service.labels` | Extra labels for app service. Only effective for helm-deployed apps (see apps.enabled). | `{}` | | `app.serviceAccount.create` | Specifies whether a ServiceAccount should be created | `true` | | `app.serviceAccount.name` | The name of the ServiceAccount to use. | `""` | | `app.serviceAccount.annotations` | Additional Service Account annotations (evaluated as a template) | `{}` | @@ -685,12 +757,24 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `app.networkPolicy.allowExternalEgress` | Allow the pod to access any range of port and all destinations. | `true` | | `app.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` | | `app.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | +| `app.mountTmpVolume` | Whether a writable /tmp emptyDir volume should be mounted to the app. | `false` | | `app.exposedContainerPort` | The port that shuffle app containers will listen on for new requests. | `80` | | `app.sdkTimeout` | The timeout in seconds for app actions. | `300` | | `app.disableLogs` | Do not capture app logs. By default, app logs are captured, so that they are visible in the frontend. | `false` | -| `app.deployViaHelm` | A list of apps that should be deployed using helm. | `{}` | -| `app.deployViaHelm.MY_APP.app` | The name of the app (required, e.g. shuffle-tools) | | -| `app.deployViaHelm.MY_APP.version` | The version of the app (required, e.g. 1.2.0) | | + +### Parameters to deploy apps using helm + +| Name | Description | Value | +| ----------------------------- | -------------------------------------------------- | ------- | +| `apps.enabled` | Whether apps should be deployed using helm. | `false` | +| `apps.shuffleTools.enabled` | Whether the shuffle-tools app is enabled | `true` | +| `apps.shuffleTools.version` | The version of the shuffle-tools app to deploy. | `1.2.0` | +| `apps.shuffleSubflow.enabled` | Whether the shuffle-subflow app is enabled | `true` | +| `apps.shuffleSubflow.version` | The version of the shuffle-subflow app to deploy. | `1.1.0` | +| `apps.http.enabled` | Whether the http app is enabled | `true` | +| `apps.http.version` | The version of the http app to deploy. | `1.4.0` | +| `apps.MY_APP.app` | The name of the app (required, e.g. shuffle-tools) | | +| `apps.MY_APP.version` | The version of the app (required, e.g. 1.2.0) | | ### Traffic Exposure Parameters @@ -793,3 +877,4 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia + diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl index 4a482335..027a49a0 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl @@ -87,9 +87,11 @@ app.kubernetes.io/managed-by: {{ .context.Release.Service }} app.kubernetes.io/part-of: shuffle app.shuffler.io/name: {{ include "shuffle.appInstance.name" .app }} app.shuffler.io/version: {{ .app.version | quote }} +{{- if .customValues }} {{- range $key, $value := .customLabels }} {{ $key }}: {{ $value }} {{- end }} +{{- end }} {{- end -}} {{/* diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-deployments.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-deployments.yaml deleted file mode 100644 index a1679e40..00000000 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-deployments.yaml +++ /dev/null @@ -1,17 +0,0 @@ -TODO: - - Deploy each app (configurable in values) here. - - Have defaults for all apps - - Allow overriding per app (serviceAccount, securityContext, volumes , env, resources, replicas, ...) - -{{- range $index, $app := .Values.app.deployViaHelm }} ---- -apiVersion: v1 -kind: Service -metadata: -spec: ---- -apiVersion: apps/v1 -kind: Deployment -metadata: -spec: -{{- end }} \ No newline at end of file diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml new file mode 100644 index 00000000..d1af8c02 --- /dev/null +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml @@ -0,0 +1,306 @@ +{{- if .Values.apps.enabled }} +{{- range $key, $app := .Values.apps }} +{{- $ignoredKeys := list "enabled" -}} +{{- if and (not ($ignoredKeys | has $key)) $app.enabled }} + +{{/* Merge .Values.app into $app, giving precedence to the values defined on the app. */}} +{{- $appValues := mustMerge $app $.Values.app -}} + +{{/* use shuffle.appRegistry as default image registry */}} +{{- $_ := set $appValues.image "registry" ($appValues.image.registry | default $.Values.shuffle.appRegistry) -}} +{{/* use shuffle.appBaseImageName as default image repository */}} +{{- $_ := set $appValues.image "repository" ($appValues.image.repository | default (printf "%s/%s" $.Values.shuffle.appBaseImageName $appValues.name)) -}} +{{/* use app version as default tag */}} +{{- $_ := set $appValues.image "tag" ($appValues.image.tag | default $appValues.version) -}} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "shuffle.appInstance.fullname" $app }} + namespace: {{ include "common.names.namespace" $ | quote }} + {{- $serviceLabels := include "common.tplvalues.merge" (dict "values" (list $appValues.service.labels $.Values.commonLabels) "context" $) }} + labels: {{- include "shuffle.appInstance.labels" ( dict "app" $app "customLabels" $serviceLabels "context" $ ) | nindent 4 }} + {{- if $.Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" (dict "value" $.Values.commonAnnotations "context" $) | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + ports: + - name: http + port: 80 + targetPort: {{ $.Values.app.exposedContainerPort }} + protocol: TCP + appProtocol: http +--- +apiVersion: {{ include "common.capabilities.deployment.apiVersion" $ }} +kind: Deployment +metadata: + name: {{ include "shuffle.appInstance.fullname" $app }} + namespace: {{ include "common.names.namespace" $ | quote }} + labels: {{- include "shuffle.appInstance.labels" ( dict "app" $app "customLabels" $.Values.commonLabels "context" $ ) | nindent 4 }} + {{- if or $appValues.deploymentAnnotations $.Values.commonAnnotations }} + {{- $annotations := include "common.tplvalues.merge" (dict "values" (list $appValues.deploymentAnnotations $.Values.commonAnnotations) "context" $) }} + annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + {{- if not $appValues.autoscaling.hpa.enabled }} + replicas: {{ $appValues.replicaCount }} + {{- end }} + {{- if $appValues.updateStrategy }} + strategy: {{- toYaml $appValues.updateStrategy | nindent 4 }} + {{- end }} + {{- $podLabels := include "common.tplvalues.merge" (dict "values" (list $appValues.podLabels $.Values.commonLabels) "context" $) }} + selector: + matchLabels: {{- include "shuffle.appInstance.matchLabels" ( dict "app" $app "customLabels" $podLabels "context" $ ) | nindent 6 }} + template: + metadata: + {{- if $appValues.podAnnotations }} + annotations: {{- include "common.tplvalues.render" (dict "value" $appValues.podAnnotations "context" $) | nindent 8 }} + {{- end }} + labels: {{- include "shuffle.appInstance.labels" ( dict "app" $app "customLabels" $podLabels "context" $ ) | nindent 8 }} + spec: + {{- include "common.images.renderPullSecrets" (dict "images" (list $appValues.image) "context" $) | nindent 6 }} + {{- if $appValues.serviceAccount.create }} + serviceAccountName: {{ default (include "shuffle.app.name" $) $appValues.serviceAccount.name | trunc 63 | trimSuffix "-" }} + {{- else }} + serviceAccountName: {{ default "default" $appValues.serviceAccount.name }} + {{- end }} + automountServiceAccountToken: {{ $appValues.automountServiceAccountToken }} + {{- if $appValues.hostAliases }} + hostAliases: {{- include "common.tplvalues.render" (dict "value" $appValues.hostAliases "context" $) | nindent 8 }} + {{- end }} + {{- if $appValues.affinity }} + affinity: {{- include "common.tplvalues.render" ( dict "value" $appValues.affinity "context" $) | nindent 8 }} + {{- else }} + affinity: + podAffinity: {{- include "common.affinities.pods" (dict "type" $appValues.podAffinityPreset "customLabels" $podLabels "context" $) | nindent 10 }} + podAntiAffinity: {{- include "common.affinities.pods" (dict "type" $appValues.podAntiAffinityPreset "customLabels" $podLabels "context" $) | nindent 10 }} + nodeAffinity: {{- include "common.affinities.nodes" (dict "type" $appValues.nodeAffinityPreset.type "key" $appValues.nodeAffinityPreset.key "values" $appValues.nodeAffinityPreset.values) | nindent 10 }} + {{- end }} + {{- if $appValues.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" ( dict "value" $appValues.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if $appValues.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" $appValues.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if $appValues.priorityClassName }} + priorityClassName: {{ $appValues.priorityClassName | quote }} + {{- end }} + {{- if $appValues.schedulerName }} + schedulerName: {{ $appValues.schedulerName | quote }} + {{- end }} + {{- if $appValues.topologySpreadConstraints }} + topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" $appValues.topologySpreadConstraints "context" $) | nindent 8 }} + {{- end }} + {{- if $appValues.podSecurityContext.enabled }} + securityContext: {{- omit $appValues.podSecurityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + {{- if $appValues.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ $appValues.terminationGracePeriodSeconds }} + {{- end }} + initContainers: + {{- if $appValues.initContainers }} + {{- include "common.tplvalues.render" (dict "value" $appValues.initContainers "context" $) | nindent 8 }} + {{- end }} + containers: + - name: {{ include "shuffle.appInstance.fullname" $app }} + image: {{ include "common.images.image" ( dict "imageRoot" $appValues.image "global" $.Values.global "chart" $.Chart ) }} + imagePullPolicy: {{ $appValues.image.pullPolicy }} + {{- if $appValues.containerSecurityContext.enabled }} + securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" $appValues.containerSecurityContext "context" $) | nindent 12 }} + {{- end }} + {{- if $.Values.diagnosticMode.enabled }} + command: {{- include "common.tplvalues.render" (dict "value" $.Values.diagnosticMode.command "context" $) | nindent 12 }} + {{- else if $appValues.command }} + command: {{- include "common.tplvalues.render" (dict "value" $appValues.command "context" $) | nindent 12 }} + {{- end }} + {{- if $.Values.diagnosticMode.enabled }} + args: {{- include "common.tplvalues.render" (dict "value" $.Values.diagnosticMode.args "context" $) | nindent 12 }} + {{- else if $appValues.args }} + args: {{- include "common.tplvalues.render" (dict "value" $appValues.args "context" $) | nindent 12 }} + {{- end }} + env: + {{- $env := include "shuffle.appInstance.env" $ | fromYaml }} + {{- range $key, $val := $env }} + - name: {{ $key | quote }} + value: {{ $val | quote }} + {{- end }} + {{- if $appValues.extraEnvVars }} + {{- include "common.tplvalues.render" (dict "value" $appValues.extraEnvVars "context" $) | nindent 12 }} + {{- end }} + envFrom: + {{- if $appValues.extraEnvVarsCM }} + - configMapRef: + name: {{ include "common.tplvalues.render" (dict "value" $appValues.extraEnvVarsCM "context" $) }} + {{- end }} + {{- if $appValues.extraEnvVarsSecret }} + - secretRef: + name: {{ include "common.tplvalues.render" (dict "value" $appValues.extraEnvVarsSecret "context" $) }} + {{- end }} + {{- if $appValues.resources }} + resources: {{- toYaml $appValues.resources | nindent 12 }} + {{- else if ne $appValues.resourcesPreset "none" }} + resources: {{- include "common.resources.preset" (dict "type" $appValues.resourcesPreset) | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: {{ $appValues.exposedContainerPort }} + {{- if $appValues.extraContainerPorts }} + {{- include "common.tplvalues.render" (dict "value" $appValues.extraContainerPorts "context" $) | nindent 12 }} + {{- end }} + {{- if not $.Values.diagnosticMode.enabled }} + {{- if $appValues.customLivenessProbe }} + livenessProbe: {{- include "common.tplvalues.render" (dict "value" $appValues.customLivenessProbe "context" $) | nindent 12 }} + {{- else if $appValues.livenessProbe.enabled }} + livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit $appValues.livenessProbe "enabled") "context" $) | nindent 12 }} + httpGet: + path: /api/v1/health + port: {{ $appValues.containerPorts.http }} + {{- end }} + {{- if $appValues.customReadinessProbe }} + readinessProbe: {{- include "common.tplvalues.render" (dict "value" $appValues.customReadinessProbe "context" $) | nindent 12 }} + {{- else if $appValues.readinessProbe.enabled }} + readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit $appValues.readinessProbe "enabled") "context" $) | nindent 12 }} + httpGet: + path: /api/v1/health + port: {{ $appValues.containerPorts.http }} + {{- end }} + {{- if $appValues.customStartupProbe }} + startupProbe: {{- include "common.tplvalues.render" (dict "value" $appValues.customStartupProbe "context" $) | nindent 12 }} + {{- else if $appValues.startupProbe.enabled }} + startupProbe: {{- include "common.tplvalues.render" (dict "value" (omit $appValues.startupProbe "enabled") "context" $) | nindent 12 }} + httpGet: + path: /api/v1/health + port: {{ $appValues.containerPorts.http }} + {{- end }} + {{- end }} + {{- if $appValues.lifecycleHooks }} + lifecycle: {{- include "common.tplvalues.render" (dict "value" $appValues.lifecycleHooks "context" $) | nindent 12 }} + {{- end }} + volumeMounts: + {{- if $appValues.mountTmpVolume }} + - name: empty-dir + mountPath: /tmp + subPath: tmp-dir + {{- end }} + {{- if $appValues.extraVolumeMounts }} + {{- include "common.tplvalues.render" (dict "value" $appValues.extraVolumeMounts "context" $) | nindent 12 }} + {{- end }} + {{- if $appValues.sidecars }} + {{- include "common.tplvalues.render" ( dict "value" $appValues.sidecars "context" $) | nindent 8 }} + {{- end }} + volumes: + - name: empty-dir + emptyDir: {} + {{- if $appValues.extraVolumes }} + {{- include "common.tplvalues.render" (dict "value" $appValues.extraVolumes "context" $) | nindent 8 }} + {{- end }} + +--- +{{- if $appValues.pdb.create }} +apiVersion: {{ include "common.capabilities.policy.apiVersion" $ }} +kind: PodDisruptionBudget +metadata: + name: {{ include "shuffle.appInstance.fullname" $app }} + namespace: {{ include "common.names.namespace" $ | quote }} + labels: {{- include "shuffle.appInstance.labels" ( dict "app" $app "customLabels" $.Values.commonLabels "context" $ ) | nindent 4 }} + {{- if $.Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" $.Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + {{- if $appValues.pdb.minAvailable }} + minAvailable: {{ $appValues.pdb.minAvailable }} + {{- end }} + {{- if or $appValues.pdb.maxUnavailable ( not $appValues.pdb.minAvailable ) }} + maxUnavailable: {{ $appValues.pdb.maxUnavailable | default 1 }} + {{- end }} + {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list $appValues.podLabels $.Values.commonLabels ) "context" $ ) }} + selector: + matchLabels: {{- include "shuffle.appInstance.matchLabels" ( dict "app" $app "customLabels" $podLabels "context" $ ) | nindent 6 }} +{{- end }} +--- +{{- if $appValues.autoscaling.hpa.enabled }} +apiVersion: {{ include "common.capabilities.hpa.apiVersion" $ }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "shuffle.appInstance.fullname" $app }} + namespace: {{ include "common.names.namespace" $ | quote }} + labels: {{- include "shuffle.appInstance.labels" ( dict "app" $app "customLabels" $.Values.commonLabels "context" $ ) | nindent 4 }} + {{- if $.Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" $.Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + scaleTargetRef: + apiVersion: {{ include "common.capabilities.deployment.apiVersion" $ }} + kind: Deployment + name: {{ include "shuffle.appInstance.fullname" $app }} + minReplicas: {{ $appValues.autoscaling.hpa.minReplicas }} + maxReplicas: {{ $appValues.autoscaling.hpa.maxReplicas }} + metrics: + {{- if $appValues.autoscaling.hpa.targetMemory }} + - type: Resource + resource: + name: memory + {{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" $) }} + targetAverageUtilization: {{ $appValues.autoscaling.hpa.targetMemory }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ $appValues.autoscaling.hpa.targetMemory }} + {{- end }} + {{- end }} + {{- if $appValues.autoscaling.hpa.targetCPU }} + - type: Resource + resource: + name: cpu + {{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" $) }} + targetAverageUtilization: {{ $appValues.autoscaling.hpa.targetCPU }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ $appValues.autoscaling.hpa.targetCPU }} + {{- end }} + {{- end }} +{{- end }} +--- +{{- if and ($.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1/VerticalPodAutoscaler") $appValues.autoscaling.vpa.enabled }} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ include "shuffle.appInstance.fullname" $app }} + namespace: {{ include "common.names.namespace" $ | quote }} + labels: {{- include "shuffle.appInstance.labels" ( dict "app" $app "customLabels" $.Values.commonLabels "context" $ ) | nindent 4 }} + {{- if or $appValues.autoscaling.vpa.annotations $.Values.commonAnnotations }} + {{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list $appValues.autoscaling.vpa.annotations $.Values.commonAnnotations ) "context" $ ) }} + annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }} + {{- end }} +spec: + resourcePolicy: + containerPolicies: + - containerName: TODO + {{- with $appValues.autoscaling.vpa.controlledResources }} + controlledResources: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $appValues.autoscaling.vpa.maxAllowed }} + maxAllowed: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $appValues.autoscaling.vpa.minAllowed }} + minAllowed: + {{- toYaml . | nindent 8 }} + {{- end }} + targetRef: + apiVersion: {{ include "common.capabilities.deployment.apiVersion" $ }} + kind: Deployment + name: {{ include "shuffle.appInstance.fullname" $app }} + {{- if $appValues.autoscaling.vpa.updatePolicy }} + updatePolicy: + {{- with $appValues.autoscaling.vpa.updatePolicy.updateMode }} + updateMode: {{ . }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl index ee7c2917..d9e16fea 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl @@ -76,9 +76,11 @@ app.kubernetes.io/part-of: shuffle {{- with .context.Chart.AppVersion }} app.kubernetes.io/version: {{ . | replace "+" "_" | quote }} {{- end -}} +{{- if .customValues }} {{- range $key, $value := .customLabels }} {{ $key }}: {{ $value }} {{- end }} +{{- end }} {{- end -}} {{/* @@ -113,6 +115,7 @@ SHUFFLE_BASE_IMAGE_REGISTRY: "{{ .Values.shuffle.appRegistry }}" SHUFFLE_BASE_IMAGE_NAME: "{{ .Values.shuffle.appBaseImageName }}" # Shuffle app deployment configuration +SHUFFLE_APP_MOUNT_TMP_VOLUME: {{ .Values.app.mountTmpVolume | quote }} SHUFFLE_APP_SERVICE_ACCOUNT_NAME: {{ include "shuffle.app.serviceAccount.name" . | quote }} {{- if .Values.app.podSecurityContext.enabled }} SHUFFLE_APP_POD_SECURITY_CONTEXT: {{ omit .Values.app.podSecurityContext "enabled" | mustToJson | quote }} diff --git a/functions/kubernetes/charts/shuffle/values.schema.json b/functions/kubernetes/charts/shuffle/values.schema.json index db8e428b..90596374 100644 --- a/functions/kubernetes/charts/shuffle/values.schema.json +++ b/functions/kubernetes/charts/shuffle/values.schema.json @@ -2695,7 +2695,7 @@ }, "manageAppDeployments": { "type": "boolean", - "description": "Whether apps are deployed and managed by worker. When disabled, every used app is expected to to be already deployed (see app.deployViaHelm).", + "description": "Whether apps are deployed and managed by worker. When disabled, every used app is expected to to be already deployed (see apps.enabled).", "default": true } } @@ -2703,6 +2703,168 @@ "app": { "type": "object", "properties": { + "image": { + "type": "object", + "properties": { + "registry": { + "type": "string", + "description": "app image registry (defaults to shuffle.appRegistry)", + "default": "" + }, + "repository": { + "type": "string", + "description": "app image repository (defaults to shuffle.appBaseImageName)", + "default": "" + }, + "tag": { + "type": "string", + "description": "app image tag (defaults to the apps version)", + "default": "" + }, + "pullPolicy": { + "type": "string", + "description": "default image pull policy for app deployments. Only effective for helm-deployed apps (see apps.enabled).", + "default": "IfNotPresent" + }, + "pullSecrets": { + "type": "array", + "description": "default image pull secrets for app deployments. Only effective for helm-deployed apps (see apps.enabled).", + "default": [], + "items": {} + } + } + }, + "replicaCount": { + "type": "number", + "description": "Default number of replicas to deploy for each app. Only effective for helm-deployed apps (see apps.enabled).", + "default": 1 + }, + "extraContainerPorts": { + "type": "array", + "description": "Optionally specify extra list of additional ports for app containers. Only effective for helm-deployed apps (see apps.enabled).", + "default": [], + "items": {} + }, + "livenessProbe": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable livenessProbe on app containers. Only effective for helm-deployed apps (see apps.enabled).", + "default": false + }, + "initialDelaySeconds": { + "type": "number", + "description": "Initial delay seconds for livenessProbe", + "default": 0 + }, + "periodSeconds": { + "type": "number", + "description": "Period seconds for livenessProbe", + "default": 15 + }, + "timeoutSeconds": { + "type": "number", + "description": "Timeout seconds for livenessProbe", + "default": 1 + }, + "failureThreshold": { + "type": "number", + "description": "Failure threshold for livenessProbe", + "default": 4 + }, + "successThreshold": { + "type": "number", + "description": "Success threshold for livenessProbe", + "default": 1 + } + } + }, + "readinessProbe": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable readinessProbe on app containers. Only effective for helm-deployed apps (see apps.enabled).", + "default": false + }, + "initialDelaySeconds": { + "type": "number", + "description": "Initial delay seconds for readinessProbe", + "default": 0 + }, + "periodSeconds": { + "type": "number", + "description": "Period seconds for readinessProbe", + "default": 5 + }, + "timeoutSeconds": { + "type": "number", + "description": "Timeout seconds for readinessProbe", + "default": 1 + }, + "failureThreshold": { + "type": "number", + "description": "Failure threshold for readinessProbe", + "default": 3 + }, + "successThreshold": { + "type": "number", + "description": "Success threshold for readinessProbe", + "default": 1 + } + } + }, + "startupProbe": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable startupProbe on app containers. Only effective for helm-deployed apps (see apps.enabled).", + "default": false + }, + "initialDelaySeconds": { + "type": "number", + "description": "Initial delay seconds for startupProbe", + "default": 0 + }, + "periodSeconds": { + "type": "number", + "description": "Period seconds for startupProbe", + "default": 1 + }, + "timeoutSeconds": { + "type": "number", + "description": "Timeout seconds for startupProbe", + "default": 1 + }, + "failureThreshold": { + "type": "number", + "description": "Failure threshold for startupProbe", + "default": 60 + }, + "successThreshold": { + "type": "number", + "description": "Success threshold for startupProbe", + "default": 1 + } + } + }, + "customLivenessProbe": { + "type": "object", + "description": "Custom livenessProbe that overrides the default one. Only effective for helm-deployed apps (see apps.enabled).", + "default": {} + }, + "customReadinessProbe": { + "type": "object", + "description": "Custom readinessProbe that overrides the default one. Only effective for helm-deployed apps (see apps.enabled).", + "default": {} + }, + "customStartupProbe": { + "type": "object", + "description": "Custom startupProbe that overrides the default one. Only effective for helm-deployed apps (see apps.enabled).", + "default": {} + }, "resourcesPreset": { "type": "string", "description": "Set app container resources according to one common preset (allowed values: none, nano, small, medium, large, xlarge, 2xlarge). This is ignored if app.resources is set (app.resources is recommended for production).", @@ -2810,6 +2972,273 @@ } } }, + "command": { + "type": "array", + "description": "Override default app container command (useful when using custom images)", + "default": [], + "items": {} + }, + "args": { + "type": "array", + "description": "Override default app container args (useful when using custom images)", + "default": [], + "items": {} + }, + "automountServiceAccountToken": { + "type": "boolean", + "description": "Mount Service Account token in app pods. Only effective for helm-deployed apps (see apps.enabled).", + "default": false + }, + "hostAliases": { + "type": "array", + "description": "app pods host aliases. Only effective for helm-deployed apps (see apps.enabled).", + "default": [], + "items": {} + }, + "deploymentAnnotations": { + "type": "object", + "description": "Annotations for app deployment. Only effective for helm-deployed apps (see apps.enabled).", + "default": {} + }, + "podLabels": { + "type": "object", + "description": "Extra labels for app pods. Only effective for helm-deployed apps (see apps.enabled).", + "default": {} + }, + "podAnnotations": { + "type": "object", + "description": "Annotations for app pods. Only effective for helm-deployed apps (see apps.enabled).", + "default": {} + }, + "podAffinityPreset": { + "type": "string", + "description": "Pod affinity preset. Ignored if `app.affinity` is set. Allowed values: `soft` or `hard`. Only effective for helm-deployed apps (see apps.enabled).", + "default": "" + }, + "podAntiAffinityPreset": { + "type": "string", + "description": "Pod anti-affinity preset. Ignored if `app.affinity` is set. Allowed values: `soft` or `hard`. Only effective for helm-deployed apps (see apps.enabled).", + "default": "soft" + }, + "nodeAffinityPreset": { + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "Node affinity preset type. Ignored if `app.affinity` is set. Allowed values: `soft` or `hard`. Only effective for helm-deployed apps (see apps.enabled).", + "default": "" + }, + "key": { + "type": "string", + "description": "Node label key to match. Ignored if `app.affinity` is set", + "default": "" + }, + "values": { + "type": "array", + "description": "Node label values to match. Ignored if `app.affinity` is set", + "default": [], + "items": {} + } + } + }, + "affinity": { + "type": "object", + "description": "Affinity for app pods assignment. Only effective for helm-deployed apps (see apps.enabled).", + "default": {} + }, + "nodeSelector": { + "type": "object", + "description": "Node labels for app pods assignment. Only effective for helm-deployed apps (see apps.enabled).", + "default": {} + }, + "tolerations": { + "type": "array", + "description": "Tolerations for app pods assignment. Only effective for helm-deployed apps (see apps.enabled).", + "default": [], + "items": {} + }, + "updateStrategy": { + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "app deployment strategy type. Only effective for helm-deployed apps (see apps.enabled).", + "default": "RollingUpdate" + } + } + }, + "priorityClassName": { + "type": "string", + "description": "app pods' priorityClassName. Only effective for helm-deployed apps (see apps.enabled).", + "default": "" + }, + "topologySpreadConstraints": { + "type": "array", + "description": "Topology Spread Constraints for app pod assignment spread across your cluster among failure-domains. Only effective for helm-deployed apps (see apps.enabled).", + "default": [], + "items": {} + }, + "schedulerName": { + "type": "string", + "description": "Name of the k8s scheduler (other than default) for app pods. Only effective for helm-deployed apps (see apps.enabled).", + "default": "" + }, + "terminationGracePeriodSeconds": { + "type": "string", + "description": "Seconds app pods need to terminate gracefully. Only effective for helm-deployed apps (see apps.enabled).", + "default": "" + }, + "lifecycleHooks": { + "type": "object", + "description": "for app containers to automate configuration before or after startup. Only effective for helm-deployed apps (see apps.enabled).", + "default": {} + }, + "extraEnvVars": { + "type": "array", + "description": "Array with extra environment variables to add to app containers. Only effective for helm-deployed apps (see apps.enabled).", + "default": [], + "items": {} + }, + "extraEnvVarsCM": { + "type": "string", + "description": "Name of existing ConfigMap containing extra env vars for app containers. Only effective for helm-deployed apps (see apps.enabled).", + "default": "" + }, + "extraEnvVarsSecret": { + "type": "string", + "description": "Name of existing Secret containing extra env vars for app containers. Only effective for helm-deployed apps (see apps.enabled).", + "default": "" + }, + "extraVolumes": { + "type": "array", + "description": "Optionally specify extra list of additional volumes for the app pods. Only effective for helm-deployed apps (see apps.enabled).", + "default": [], + "items": {} + }, + "extraVolumeMounts": { + "type": "array", + "description": "Optionally specify extra list of additional volumeMounts for the app containers. Only effective for helm-deployed apps (see apps.enabled).", + "default": [], + "items": {} + }, + "sidecars": { + "type": "array", + "description": "Add additional sidecar containers to the app pods. Only effective for helm-deployed apps (see apps.enabled).", + "default": [], + "items": {} + }, + "initContainers": { + "type": "array", + "description": "Add additional init containers to the app pods. Only effective for helm-deployed apps (see apps.enabled).", + "default": [], + "items": {} + }, + "pdb": { + "type": "object", + "properties": { + "create": { + "type": "boolean", + "description": "Enable/disable a Pod Disruption Budget creation. Only effective for helm-deployed apps (see apps.enabled).", + "default": true + }, + "minAvailable": { + "type": "string", + "description": "Minimum number/percentage of pods that should remain scheduled", + "default": "" + }, + "maxUnavailable": { + "type": "string", + "description": "Maximum number/percentage of pods that may be made unavailable. Defaults to `1` if both `app.pdb.minAvailable` and `app.pdb.maxUnavailable` are empty.", + "default": "" + } + } + }, + "autoscaling": { + "type": "object", + "properties": { + "vpa": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable VPA for app pods. Only effective for helm-deployed apps (see apps.enabled).", + "default": false + }, + "annotations": { + "type": "object", + "description": "Annotations for VPA resource", + "default": {} + }, + "controlledResources": { + "type": "array", + "description": "VPA List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory", + "default": [], + "items": {} + }, + "maxAllowed": { + "type": "object", + "description": "VPA Max allowed resources for the pod", + "default": {} + }, + "minAllowed": { + "type": "object", + "description": "VPA Min allowed resources for the pod", + "default": {} + }, + "updatePolicy": { + "type": "object", + "properties": { + "updateMode": { + "type": "string", + "description": "Autoscaling update policy", + "default": "Auto" + } + } + } + } + }, + "hpa": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable HPA for app pods. Only effective for helm-deployed apps (see apps.enabled).", + "default": false + }, + "minReplicas": { + "type": "string", + "description": "Minimum number of replicas", + "default": "" + }, + "maxReplicas": { + "type": "string", + "description": "Maximum number of replicas", + "default": "" + }, + "targetCPU": { + "type": "string", + "description": "Target CPU utilization percentage", + "default": "" + }, + "targetMemory": { + "type": "string", + "description": "Target Memory utilization percentage", + "default": "" + } + } + } + } + }, + "service": { + "type": "object", + "properties": { + "labels": { + "type": "object", + "description": "Extra labels for app service. Only effective for helm-deployed apps (see apps.enabled).", + "default": {} + } + } + }, "serviceAccount": { "type": "object", "properties": { @@ -2883,6 +3312,11 @@ } } }, + "mountTmpVolume": { + "type": "boolean", + "description": "Whether a writable /tmp emptyDir volume should be mounted to the app.", + "default": false + }, "exposedContainerPort": { "type": "number", "description": "The port that shuffle app containers will listen on for new requests. ", @@ -2900,6 +3334,61 @@ } } }, + "apps": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Whether apps should be deployed using helm.", + "default": false + }, + "shuffleTools": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Whether the shuffle-tools app is enabled", + "default": true + }, + "version": { + "type": "string", + "description": "The version of the shuffle-tools app to deploy.", + "default": "1.2.0" + } + } + }, + "shuffleSubflow": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Whether the shuffle-subflow app is enabled", + "default": true + }, + "version": { + "type": "string", + "description": "The version of the shuffle-subflow app to deploy.", + "default": "1.1.0" + } + } + }, + "http": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Whether the http app is enabled", + "default": true + }, + "version": { + "type": "string", + "description": "The version of the http app to deploy.", + "default": "1.4.0" + } + } + } + } + }, "ingress": { "type": "object", "properties": { diff --git a/functions/kubernetes/charts/shuffle/values.yaml b/functions/kubernetes/charts/shuffle/values.yaml index 748cdd72..25e7654c 100644 --- a/functions/kubernetes/charts/shuffle/values.yaml +++ b/functions/kubernetes/charts/shuffle/values.yaml @@ -1743,10 +1743,16 @@ worker: ## @section app Parameters ## app: + ## @param app.image.registry app image registry (defaults to shuffle.appRegistry) + ## @param app.image.repository app image repository (defaults to shuffle.appBaseImageName) + ## @param app.image.tag app image tag (defaults to the apps version) ## @param app.image.pullPolicy default image pull policy for app deployments. Only effective for helm-deployed apps (see apps.enabled). ## @param app.image.pullSecrets default image pull secrets for app deployments. Only effective for helm-deployed apps (see apps.enabled). ## image: + registry: "" + repository: "" + tag: "" ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' ## ref: https://kubernetes.io/docs/concepts/containers/images/#pre-pulled-images @@ -1882,7 +1888,12 @@ app: drop: ["ALL"] seccompProfile: type: "RuntimeDefault" - + ## @param app.command Override default app container command (useful when using custom images) + ## + command: [] + ## @param app.args Override default app container args (useful when using custom images) + ## + args: [] ## @param app.automountServiceAccountToken Mount Service Account token in app pods. Only effective for helm-deployed apps (see apps.enabled). ## automountServiceAccountToken: false @@ -2109,6 +2120,9 @@ app: ## extraEgress: [] + ## @param app.mountTmpVolume Whether a writable /tmp emptyDir volume should be mounted to the app. + ## + mountTmpVolume: false ## @param app.exposedContainerPort The port that shuffle app containers will listen on for new requests. ## exposedContainerPort: 80 @@ -2134,373 +2148,50 @@ apps: enabled: false shuffleTools: - ## @param apps.shuffleTools.enabled Whether the app is enabled + ## @param apps.shuffleTools.enabled Whether the shuffle-tools app is enabled ## enabled: true - ## app image - ## @param apps.shuffleTools.image.registry app image registry - ## @param apps.shuffleTools.image.repository app image repository - ## @param apps.shuffleTools.image.tag app image tag (immutable tags are recommended, defaults to appVersion) - ## @param apps.shuffleTools.image.digest app image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended) - ## @param apps.shuffleTools.image.pullPolicy app image pull policy - ## @param apps.shuffleTools.image.pullSecrets app image pull secrets + ## @skip apps.shuffleTools.name ## - image: - registry: ghcr.io - repository: shuffle/shuffle-app - tag: "" - digest: "" - ## Specify a imagePullPolicy - ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: https://kubernetes.io/docs/concepts/containers/images/#pre-pulled-images - ## - pullPolicy: IfNotPresent - ## Optionally specify an array of imagePullSecrets. - ## Secrets must be manually created in the namespace. - ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ - ## e.g: - ## pullSecrets: - ## - myRegistryKeySecretName - ## - pullSecrets: [] - ## @param apps.shuffleTools.replicaCount [number, nullable] Number of app replicas to deploy + name: shuffle-tools + ## @param apps.shuffleTools.version The version of the shuffle-tools app to deploy. ## - replicaCount: null - ## @param apps.shuffleTools.extraContainerPorts Optionally specify extra list of additional ports for app containers - ## e.g: - ## extraContainerPorts: - ## - name: myservice - ## containerPort: 9090 - ## - extraContainerPorts: [] - ## Configure extra options for app containers' liveness and readiness probes - ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes - ## @param apps.shuffleTools.livenessProbe.enabled Enable livenessProbe on app containers - ## @param apps.shuffleTools.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe - ## @param apps.shuffleTools.livenessProbe.periodSeconds Period seconds for livenessProbe - ## @param apps.shuffleTools.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe - ## @param apps.shuffleTools.livenessProbe.failureThreshold Failure threshold for livenessProbe - ## @param apps.shuffleTools.livenessProbe.successThreshold Success threshold for livenessProbe - ## - livenessProbe: - enabled: false - initialDelaySeconds: 0 - periodSeconds: 15 - timeoutSeconds: 1 - failureThreshold: 4 - successThreshold: 1 - ## @param apps.shuffleTools.readinessProbe.enabled Enable readinessProbe on app containers - ## @param apps.shuffleTools.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe - ## @param apps.shuffleTools.readinessProbe.periodSeconds Period seconds for readinessProbe - ## @param apps.shuffleTools.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe - ## @param apps.shuffleTools.readinessProbe.failureThreshold Failure threshold for readinessProbe - ## @param apps.shuffleTools.readinessProbe.successThreshold Success threshold for readinessProbe - ## - readinessProbe: - enabled: false - initialDelaySeconds: 0 - periodSeconds: 5 - timeoutSeconds: 1 - failureThreshold: 3 - successThreshold: 1 - ## @param apps.shuffleTools.startupProbe.enabled Enable startupProbe on app containers - ## @param apps.shuffleTools.startupProbe.initialDelaySeconds Initial delay seconds for startupProbe - ## @param apps.shuffleTools.startupProbe.periodSeconds Period seconds for startupProbe - ## @param apps.shuffleTools.startupProbe.timeoutSeconds Timeout seconds for startupProbe - ## @param apps.shuffleTools.startupProbe.failureThreshold Failure threshold for startupProbe - ## @param apps.shuffleTools.startupProbe.successThreshold Success threshold for startupProbe - ## - startupProbe: - enabled: false - initialDelaySeconds: 0 - periodSeconds: 1 - timeoutSeconds: 1 - failureThreshold: 60 - successThreshold: 1 - ## @param apps.shuffleTools.customLivenessProbe Custom livenessProbe that overrides the default one - ## - customLivenessProbe: {} - ## @param apps.shuffleTools.customReadinessProbe Custom readinessProbe that overrides the default one - ## - customReadinessProbe: {} - ## @param apps.shuffleTools.customStartupProbe Custom startupProbe that overrides the default one - ## - customStartupProbe: {} - ## app resource requests and limits - ## ref: http://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ - ## @param apps.shuffleTools.resourcesPreset Set app container resources according to one common preset (allowed values: none, nano, small, medium, large, xlarge, 2xlarge). This is ignored if app.resources is set (app.resources is recommended for production). - ## More information: https://github.com/bitnami/charts/blob/main/bitnami/common/templates/_resources.tpl#L15 - ## Shuffle gets OOM killed with 256M memory during startup. Up to 360MiB of memory usage were observed during testing. - ## The small preset grants 512M. - ## - resourcesPreset: "small" - ## @param apps.shuffleTools.resources Set app container requests and limits for different resources like CPU or memory (essential for production workloads) - ## Example: - ## resources: - ## requests: - ## cpu: 2 - ## memory: 512Mi - ## limits: - ## cpu: 3 - ## memory: 1024Mi - ## - resources: {} - ## Configure Pods Security Context - ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod - ## @param apps.shuffleTools.podSecurityContext.enabled Enable app pods' Security Context - ## @param apps.shuffleTools.podSecurityContext.fsGroupChangePolicy Set filesystem group change policy for app pods - ## @param apps.shuffleTools.podSecurityContext.sysctls Set kernel settings using the sysctl interface for app pods - ## @param apps.shuffleTools.podSecurityContext.supplementalGroups Set filesystem extra groups for app pods - ## @param apps.shuffleTools.podSecurityContext.fsGroup Set fsGroup in app pods' Security Context - ## - podSecurityContext: - enabled: true - fsGroupChangePolicy: Always - sysctls: [] - supplementalGroups: [] - fsGroup: 1001 - ## Configure Container Security Context - ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container - ## @param apps.shuffleTools.containerSecurityContext.enabled Enabled app container' Security Context - ## @param apps.shuffleTools.containerSecurityContext.seLinuxOptions [object,nullable] Set SELinux options in app container - ## @param apps.shuffleTools.containerSecurityContext.runAsUser Set runAsUser in app container' Security Context - ## @param apps.shuffleTools.containerSecurityContext.runAsGroup Set runAsGroup in app container' Security Context - ## @param apps.shuffleTools.containerSecurityContext.runAsNonRoot Set runAsNonRoot in app container' Security Context - ## @param apps.shuffleTools.containerSecurityContext.readOnlyRootFilesystem Set readOnlyRootFilesystem in app container' Security Context - ## @param apps.shuffleTools.containerSecurityContext.privileged Set privileged in app container' Security Context - ## @param apps.shuffleTools.containerSecurityContext.allowPrivilegeEscalation Set allowPrivilegeEscalation in app container' Security Context - ## @param apps.shuffleTools.containerSecurityContext.capabilities.drop List of capabilities to be dropped in app container - ## @param apps.shuffleTools.containerSecurityContext.seccompProfile.type Set seccomp profile in app container - ## - containerSecurityContext: - enabled: true - seLinuxOptions: {} - runAsUser: 1001 - runAsGroup: 1001 - runAsNonRoot: true - readOnlyRootFilesystem: true - privileged: false - allowPrivilegeEscalation: false - capabilities: - drop: ["ALL"] - seccompProfile: - type: "RuntimeDefault" - ## @param apps.shuffleTools.command Override default app container command (useful when using custom images) - ## - command: [] - ## @param apps.shuffleTools.args Override default app container args (useful when using custom images) - ## - args: [] - ## @param apps.shuffleTools.automountServiceAccountToken Mount Service Account token in app pods - ## NOTE: app requires the service account credentials to be mounted - ## - automountServiceAccountToken: true - ## @param apps.shuffleTools.hostAliases app pods host aliases - ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ - ## - hostAliases: [] - ## @param apps.shuffleTools.deploymentAnnotations Annotations for app deployment - ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ - ## - deploymentAnnotations: {} - ## @param apps.shuffleTools.podLabels Extra labels for app pods - ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ - ## - podLabels: {} - ## @param apps.shuffleTools.podAnnotations Annotations for app pods - ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ - ## - podAnnotations: {} - ## @param apps.shuffleTools.podAffinityPreset Pod affinity preset. Ignored if `app.affinity` is set. Allowed values: `soft` or `hard` - ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity - ## - podAffinityPreset: "" - ## @param apps.shuffleTools.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `app.affinity` is set. Allowed values: `soft` or `hard` - ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity - ## - podAntiAffinityPreset: soft - ## Node app.affinity preset - ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity - ## - nodeAffinityPreset: - ## @param apps.shuffleTools.nodeAffinityPreset.type Node affinity preset type. Ignored if `app.affinity` is set. Allowed values: `soft` or `hard` - ## - type: "" - ## @param apps.shuffleTools.nodeAffinityPreset.key Node label key to match. Ignored if `app.affinity` is set - ## - key: "" - ## @param apps.shuffleTools.nodeAffinityPreset.values Node label values to match. Ignored if `app.affinity` is set - ## E.g. - ## values: - ## - e2e-az1 - ## - e2e-az2 - ## - values: [] - ## @param apps.shuffleTools.affinity Affinity for app pods assignment - ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity - ## NOTE: `app.podAffinityPreset`, `app.podAntiAffinityPreset`, and `app.nodeAffinityPreset` will be ignored when it's set - ## - affinity: {} - ## @param apps.shuffleTools.nodeSelector Node labels for app pods assignment - ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/ - ## - nodeSelector: {} - ## @param apps.shuffleTools.tolerations Tolerations for app pods assignment - ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ - ## - tolerations: [] - ## @param apps.shuffleTools.updateStrategy.type app deployment strategy type - ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy - ## - updateStrategy: - ## Can be set to RollingUpdate or Recreate - ## app uses ReadWriteOnce volumes by default, which is incompatible with RollingUpdate - ## - type: Recreate - ## @param apps.shuffleTools.priorityClassName app pods' priorityClassName - ## - priorityClassName: "" - ## @param apps.shuffleTools.topologySpreadConstraints Topology Spread Constraints for app pod assignment spread across your cluster among failure-domains - ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods - ## - topologySpreadConstraints: [] - ## @param apps.shuffleTools.schedulerName Name of the k8s scheduler (other than default) for app pods - ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ - ## - schedulerName: "" - ## @param apps.shuffleTools.terminationGracePeriodSeconds Seconds app pods need to terminate gracefully - ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods - ## - terminationGracePeriodSeconds: "" - ## @param apps.shuffleTools.lifecycleHooks for app containers to automate configuration before or after startup - ## - lifecycleHooks: {} - ## @param apps.shuffleTools.extraEnvVars Array with extra environment variables to add to app containers - ## e.g: - ## extraEnvVars: - ## - name: FOO - ## value: "bar" - ## - extraEnvVars: [] - ## @param apps.shuffleTools.extraEnvVarsCM Name of existing ConfigMap containing extra env vars for app containers - ## - extraEnvVarsCM: "" - ## @param apps.shuffleTools.extraEnvVarsSecret Name of existing Secret containing extra env vars for app containers - ## - extraEnvVarsSecret: "" - ## @param apps.shuffleTools.extraVolumes Optionally specify extra list of additional volumes for the app pods - ## - extraVolumes: [] - ## @param apps.shuffleTools.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the app containers - ## - extraVolumeMounts: [] - ## @param apps.shuffleTools.sidecars Add additional sidecar containers to the app pods - ## e.g: - ## sidecars: - ## - name: your-image-name - ## image: your-image - ## imagePullPolicy: Always - ## ports: - ## - name: portname - ## containerPort: 1234 - ## - sidecars: [] - ## @param apps.shuffleTools.initContainers Add additional init containers to the app pods - ## ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ - ## e.g: - ## initContainers: - ## - name: your-image-name - ## image: your-image - ## imagePullPolicy: Always - ## command: ['sh', '-c', 'echo "hello world"'] - ## - initContainers: [] - ## Pod Disruption Budget configuration - ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb - ## @param apps.shuffleTools.pdb.create Enable/disable a Pod Disruption Budget creation - ## @param apps.shuffleTools.pdb.minAvailable Minimum number/percentage of pods that should remain scheduled - ## @param apps.shuffleTools.pdb.maxUnavailable Maximum number/percentage of pods that may be made unavailable. Defaults to `1` if both `app.pdb.minAvailable` and `app.pdb.maxUnavailable` are empty. - ## - pdb: - create: true - minAvailable: "" - maxUnavailable: "" - ## Autoscaling configuration - ## ref: https://kubernetes.io/docs/concepts/workloads/autoscaling/ - ## - autoscaling: - ## @param apps.shuffleTools.autoscaling.vpa.enabled Enable VPA for app pods - ## @param apps.shuffleTools.autoscaling.vpa.annotations Annotations for VPA resource - ## @param apps.shuffleTools.autoscaling.vpa.controlledResources VPA List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory - ## @param apps.shuffleTools.autoscaling.vpa.maxAllowed VPA Max allowed resources for the pod - ## @param apps.shuffleTools.autoscaling.vpa.minAllowed VPA Min allowed resources for the pod - ## - vpa: - enabled: false - annotations: {} - controlledResources: [] - maxAllowed: {} - minAllowed: {} - ## @param apps.shuffleTools.autoscaling.vpa.updatePolicy.updateMode Autoscaling update policy - ## Specifies whether recommended updates are applied when a Pod is started and whether recommended updates are applied during the life of a Pod - ## Possible values are "Off", "Initial", "Recreate", and "Auto". - ## - updatePolicy: - updateMode: Auto - ## @param apps.shuffleTools.autoscaling.hpa.enabled Enable HPA for app pods - ## @param apps.shuffleTools.autoscaling.hpa.minReplicas Minimum number of replicas - ## @param apps.shuffleTools.autoscaling.hpa.maxReplicas Maximum number of replicas - ## @param apps.shuffleTools.autoscaling.hpa.targetCPU Target CPU utilization percentage - ## @param apps.shuffleTools.autoscaling.hpa.targetMemory Target Memory utilization percentage - ## - hpa: - enabled: false - minReplicas: "" - maxReplicas: "" - targetCPU: "" - targetMemory: "" - - ## Service configuration - ## - service: - ## @param apps.shuffleTools.service.labels Extra labels for app service - ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ - ## - labels: {} + version: 1.2.0 shuffleSubflow: + ## @param apps.shuffleSubflow.enabled Whether the shuffle-subflow app is enabled + ## enabled: true + ## @skip apps.shuffleSubflow.name + ## + name: shuffle-subflow + ## @param apps.shuffleSubflow.version The version of the shuffle-subflow app to deploy. + ## + version: 1.1.0 http: + ## @param apps.http.enabled Whether the http app is enabled + ## enabled: true + ## @skip apps.http.name + ## + name: http + ## @param apps.http.version The version of the http app to deploy. + ## + version: 1.4.0 # replicas, image, etc. ## @extra apps.MY_APP.app [string] The name of the app (required, e.g. shuffle-tools) ## @extra apps.MY_APP.version [string] The version of the app (required, e.g. 1.2.0) - ## TODO: Add more extras ## Add your own apps here. The key of the app does not matter, as long as it is unique. ## myApp: ## enabled: true ## name: my-app ## version: 1.0.0 - ## ... - - # TODO: Convert sample value to @extra. - ## Sample value: - ## - app: my-app # required - ## version: 1.0.0 # required - ## replicas: 1 # optional, default to 1 - ## image: "" # optional, default to appregistry/baseimage/appname:appversion - ## imagePullSecrets: null # optional, defaults to global.imagePullSecrets - ## resources: null # optional, defaults to app.resources (or app.resourcePreset) - ## extraVolumes: [] # optional - ## extraVolumeMounts: [] # optional - ## extraEnv: [] # optional - ## podSecurityContext: null # optional, defaults to app.podSecurityContext - ## containerSecurityContext: null # optional, defaults to app.containerSecurityContext - ## TODO: allow to override resources, serviceAccount, mounts, env, security Contexts. Fall back to app defaults as defined above. - ## + ## ... Overwrite .app.* values here, e.g.: + ## replicaCount: 3 + ## resources: {} ## @section Traffic Exposure Parameters ## From 69c7c7bb06c7ec50700e20ef9720ed4a32a6464c Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Wed, 3 Dec 2025 11:58:46 +0100 Subject: [PATCH 17/41] fix label matching for worker and apps Signed-off-by: Pascal Sthamer --- .../backend/backend-network-policy.yaml | 4 ++-- .../orborus/orborus-network-policy.yaml | 2 +- .../templates/shuffle-app/_helpers.tpl | 23 ++++++++++++++++--- .../shuffle-app-network-policy.yaml | 6 ++--- .../templates/shuffle-app/shuffle-apps.yaml | 1 - .../templates/shuffle-worker/_helpers.tpl | 21 ++++++++++++++--- .../shuffle-worker-network-policy.yaml | 7 +++--- .../shuffle-worker/shuffle-worker-svc.yaml | 3 ++- 8 files changed, 49 insertions(+), 18 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/templates/backend/backend-network-policy.yaml b/functions/kubernetes/charts/shuffle/templates/backend/backend-network-policy.yaml index 516c9f33..48010706 100644 --- a/functions/kubernetes/charts/shuffle/templates/backend/backend-network-policy.yaml +++ b/functions/kubernetes/charts/shuffle/templates/backend/backend-network-policy.yaml @@ -51,14 +51,14 @@ spec: matchLabels: kubernetes.io/metadata.name: {{ .Release.Namespace }} podSelector: - matchLabels: {{ include "shuffle.workerInstance.matchLabels" . | nindent 14 }} + matchLabels: {{ include "shuffle.worker.matchLabels" . | nindent 14 }} # Allow traffic from apps - namespaceSelector: matchLabels: kubernetes.io/metadata.name: {{ .Release.Namespace }} podSelector: - matchLabels: {{ include "shuffle.appInstance.matchLabels" . | nindent 14 }} + matchLabels: {{ include "shuffle.app.matchLabels" . | nindent 14 }} {{- end }} {{- if .Values.backend.networkPolicy.extraIngress }} {{- include "common.tplvalues.render" ( dict "value" .Values.backend.networkPolicy.extraIngress "context" $ ) | nindent 4 }} diff --git a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-network-policy.yaml b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-network-policy.yaml index 68ea9aa5..f55bf957 100644 --- a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-network-policy.yaml +++ b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-network-policy.yaml @@ -48,7 +48,7 @@ spec: matchLabels: kubernetes.io/metadata.name: {{ .Release.Namespace }} podSelector: - matchLabels: {{ include "shuffle.workerInstance.matchLabels" . | nindent 14 }} + matchLabels: {{ include "shuffle.worker.matchLabels" . | nindent 14 }} {{- if .Values.orborus.networkPolicy.extraEgress }} {{- include "common.tplvalues.render" ( dict "value" .Values.orborus.networkPolicy.extraEgress "context" $ ) | nindent 4 }} {{- end }} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl index 027a49a0..f47f073e 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl @@ -87,7 +87,7 @@ app.kubernetes.io/managed-by: {{ .context.Release.Service }} app.kubernetes.io/part-of: shuffle app.shuffler.io/name: {{ include "shuffle.appInstance.name" .app }} app.shuffler.io/version: {{ .app.version | quote }} -{{- if .customValues }} +{{- if .customLabels }} {{- range $key, $value := .customLabels }} {{ $key }}: {{ $value }} {{- end }} @@ -95,12 +95,29 @@ app.shuffler.io/version: {{ .app.version | quote }} {{- end -}} {{/* -Return the match labels for apps. -These must match the labels of helm-deployed apps (shuffle.appInstance.labels), +Return the match labels for ALL apps. +These match the labels of helm-deployed apps (shuffle.appInstance.labels), as well as worker-deployed apps (deployK8sApp). */}} +{{- define "shuffle.app.matchLabels" -}} +app.kubernetes.io/name: shuffle-app +{{- end -}} + + +{{/* +Return the match labels of a single app, deployed via helm. +Usage: +{{ include "shuffle.appInstance.matchLabels" (dict "app" $app "customLabels" .Values.commonLabels "context" $) }} +*/}} {{- define "shuffle.appInstance.matchLabels" -}} app.kubernetes.io/name: shuffle-app +app.shuffler.io/name: {{ include "shuffle.appInstance.name" .app }} +app.shuffler.io/version: {{ .app.version | quote }} +{{- if .customLabels }} +{{- range $key, $value := .customLabels }} +{{ $key }}: {{ $value }} +{{- end }} +{{- end }} {{- end -}} {{/* diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-network-policy.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-network-policy.yaml index 73c0572e..a8493002 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-network-policy.yaml +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-network-policy.yaml @@ -11,7 +11,7 @@ metadata: spec: {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.app.podLabels .Values.commonLabels ) "context" . ) }} podSelector: - matchLabels: {{- include "shuffle.appInstance.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 6 }} + matchLabels: {{- include "shuffle.app.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 6 }} policyTypes: - Ingress - Egress @@ -48,7 +48,7 @@ spec: matchLabels: kubernetes.io/metadata.name: {{ .Release.Namespace }} podSelector: - matchLabels: {{ include "shuffle.workerInstance.matchLabels" . | nindent 14 }} + matchLabels: {{ include "shuffle.worker.matchLabels" . | nindent 14 }} {{- if .Values.app.networkPolicy.extraEgress }} {{- include "common.tplvalues.render" ( dict "value" .Values.app.networkPolicy.extraEgress "context" $ ) | nindent 4 }} {{- end }} @@ -64,7 +64,7 @@ spec: matchLabels: kubernetes.io/metadata.name: {{ .Release.Namespace }} podSelector: - matchLabels: {{ include "shuffle.workerInstance.matchLabels" . | nindent 14 }} + matchLabels: {{ include "shuffle.worker.matchLabels" . | nindent 14 }} {{- end }} {{- if .Values.app.networkPolicy.extraIngress }} {{- include "common.tplvalues.render" ( dict "value" .Values.app.networkPolicy.extraIngress "context" $ ) | nindent 4 }} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml index d1af8c02..112c6241 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml @@ -214,7 +214,6 @@ spec: {{- if or $appValues.pdb.maxUnavailable ( not $appValues.pdb.minAvailable ) }} maxUnavailable: {{ $appValues.pdb.maxUnavailable | default 1 }} {{- end }} - {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list $appValues.podLabels $.Values.commonLabels ) "context" $ ) }} selector: matchLabels: {{- include "shuffle.appInstance.matchLabels" ( dict "app" $app "customLabels" $podLabels "context" $ ) | nindent 6 }} {{- end }} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl index d9e16fea..b9f223fd 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl @@ -76,7 +76,7 @@ app.kubernetes.io/part-of: shuffle {{- with .context.Chart.AppVersion }} app.kubernetes.io/version: {{ . | replace "+" "_" | quote }} {{- end -}} -{{- if .customValues }} +{{- if .customLabels }} {{- range $key, $value := .customLabels }} {{ $key }}: {{ $value }} {{- end }} @@ -84,12 +84,27 @@ app.kubernetes.io/version: {{ . | replace "+" "_" | quote }} {{- end -}} {{/* -Return the match labels for workers. -These must match the labels of helm-deployed workers (shuffle.workerInstance.labels), +Return the labels to match ALL workers. +These match the labels of helm-deployed workers (shuffle.workerInstance.labels), as well as orborus-deployed workers (deployk8sworker). */}} +{{- define "shuffle.worker.matchLabels" -}} +app.kubernetes.io/name: shuffle-worker +{{- end -}} + +{{/* +Return the labels to match a helm-deployed worker. +Usage: +{{ include "shuffle.workerInstance.matchLabels" (dict "customLabels" .Values.commonLabels "context" $) -}} +*/}} {{- define "shuffle.workerInstance.matchLabels" -}} app.kubernetes.io/name: shuffle-worker +app.kubernetes.io/instance: {{ .context.Release.Name }} +{{- if .customLabels }} +{{- range $key, $value := .customLabels }} +{{ $key }}: {{ $value }} +{{- end }} +{{- end }} {{- end -}} {{/* diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-network-policy.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-network-policy.yaml index 259300e1..514f3815 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-network-policy.yaml +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-network-policy.yaml @@ -9,9 +9,8 @@ metadata: annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} {{- end }} spec: - {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.worker.podLabels .Values.commonLabels ) "context" . ) }} podSelector: - matchLabels: {{- include "shuffle.workerInstance.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 6 }} + matchLabels: {{- include "shuffle.worker.matchLabels" . | nindent 6 }} policyTypes: - Ingress - Egress @@ -35,7 +34,7 @@ spec: matchLabels: kubernetes.io/metadata.name: {{ .Release.Namespace }} podSelector: - matchLabels: {{ include "shuffle.appInstance.matchLabels" . | nindent 14 }} + matchLabels: {{- include "shuffle.app.matchLabels" . | nindent 14 }} {{- if .Values.worker.networkPolicy.extraEgress }} {{- include "common.tplvalues.render" ( dict "value" .Values.worker.networkPolicy.extraEgress "context" $ ) | nindent 4 }} {{- end }} @@ -57,7 +56,7 @@ spec: matchLabels: kubernetes.io/metadata.name: {{ .Release.Namespace }} podSelector: - matchLabels: {{ include "shuffle.appInstance.matchLabels" . | nindent 14 }} + matchLabels: {{- include "shuffle.app.matchLabels" . | nindent 14 }} {{- end }} {{- if .Values.worker.networkPolicy.extraIngress }} {{- include "common.tplvalues.render" ( dict "value" .Values.worker.networkPolicy.extraIngress "context" $ ) | nindent 4 }} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-svc.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-svc.yaml index 79f5be00..b9f355a5 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-svc.yaml +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-svc.yaml @@ -17,5 +17,6 @@ spec: targetPort: 33333 protocol: TCP appProtocol: http - selector: {{- include "shuffle.workerInstance.matchLabels" $ | nindent 4 }} + {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.worker.podLabels .Values.commonLabels ) "context" . ) }} + selector: {{- include "shuffle.workerInstance.matchLabels" (dict "customLabels" $podLabels "context" .) | nindent 4 }} {{- end }} \ No newline at end of file From e3d78fbf0ce5b0014b96bc5a44e49f55bc742cc7 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Wed, 3 Dec 2025 12:02:26 +0100 Subject: [PATCH 18/41] add vpa, hpa, and pdb for workers Signed-off-by: Pascal Sthamer --- .../shuffle-worker/shuffle-worker-hpa.yaml | 43 +++++++++++++++++++ .../shuffle-worker/shuffle-worker-pdb.yaml | 21 +++++++++ .../shuffle-worker/shuffle-worker-vpa.yaml | 39 +++++++++++++++++ 3 files changed, 103 insertions(+) create mode 100644 functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-hpa.yaml create mode 100644 functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-pdb.yaml create mode 100644 functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-vpa.yaml diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-hpa.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-hpa.yaml new file mode 100644 index 00000000..4f620f83 --- /dev/null +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-hpa.yaml @@ -0,0 +1,43 @@ +{{- if and .Values.worker.enableHelmDeployment .Values.worker.autoscaling.hpa.enabled }} +apiVersion: {{ include "common.capabilities.hpa.apiVersion" . }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "shuffle.worker.name" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "shuffle.worker.labels" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + scaleTargetRef: + apiVersion: {{ include "common.capabilities.deployment.apiVersion" . }} + kind: Deployment + name: {{ include "shuffle.worker.name" . }} + minReplicas: {{ .Values.worker.autoscaling.hpa.minReplicas }} + maxReplicas: {{ .Values.worker.autoscaling.hpa.maxReplicas }} + metrics: + {{- if .Values.worker.autoscaling.hpa.targetMemory }} + - type: Resource + resource: + name: memory + {{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .) }} + targetAverageUtilization: {{ .Values.worker.autoscaling.hpa.targetMemory }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.worker.autoscaling.hpa.targetMemory }} + {{- end }} + {{- end }} + {{- if .Values.worker.autoscaling.hpa.targetCPU }} + - type: Resource + resource: + name: cpu + {{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .) }} + targetAverageUtilization: {{ .Values.worker.autoscaling.hpa.targetCPU }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.worker.autoscaling.hpa.targetCPU }} + {{- end }} + {{- end }} +{{- end }} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-pdb.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-pdb.yaml new file mode 100644 index 00000000..7bb2218e --- /dev/null +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-pdb.yaml @@ -0,0 +1,21 @@ +{{- if .Values.worker.pdb.create }} +apiVersion: {{ include "common.capabilities.policy.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "shuffle.worker.name" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "shuffle.worker.labels" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + {{- if .Values.worker.pdb.minAvailable }} + minAvailable: {{ .Values.worker.pdb.minAvailable }} + {{- end }} + {{- if or .Values.worker.pdb.maxUnavailable ( not .Values.worker.pdb.minAvailable ) }} + maxUnavailable: {{ .Values.worker.pdb.maxUnavailable | default 1 }} + {{- end }} + selector: + {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.worker.podLabels .Values.commonLabels ) "context" . ) }} + matchLabels: {{- include "shuffle.workerInstance.matchLabels" (dict "customLabels" $podLabels "context" .) | nindent 6 }} +{{- end }} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-vpa.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-vpa.yaml new file mode 100644 index 00000000..d8101831 --- /dev/null +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-vpa.yaml @@ -0,0 +1,39 @@ +{{- if and (.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1/VerticalPodAutoscaler") (and .Values.worker.enableHelmDeployment .Values.worker.autoscaling.vpa.enabled) }} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ include "shuffle.worker.name" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "shuffle.worker.labels" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if or .Values.worker.autoscaling.vpa.annotations .Values.commonAnnotations }} + {{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.worker.autoscaling.vpa.annotations .Values.commonAnnotations ) "context" . ) }} + annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }} + {{- end }} +spec: + resourcePolicy: + containerPolicies: + - containerName: worker + {{- with .Values.worker.autoscaling.vpa.controlledResources }} + controlledResources: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.autoscaling.vpa.maxAllowed }} + maxAllowed: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.autoscaling.vpa.minAllowed }} + minAllowed: + {{- toYaml . | nindent 8 }} + {{- end }} + targetRef: + apiVersion: {{ include "common.capabilities.deployment.apiVersion" . }} + kind: Deployment + name: {{ include "shuffle.worker.name" . }} + {{- end }} + {{- if .Values.worker.autoscaling.vpa.updatePolicy }} + updatePolicy: + {{- with .Values.worker.autoscaling.vpa.updatePolicy.updateMode }} + updateMode: {{ . }} + {{- end }} + {{- end }} +{{- end }} From d63db15c42167484a0e852f8e199f5e8a25343a1 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Wed, 3 Dec 2025 12:43:29 +0100 Subject: [PATCH 19/41] fix affinity rules for worker and apps Signed-off-by: Pascal Sthamer --- .../templates/shuffle-app/_helpers.tpl | 52 +++++++++---------- .../templates/shuffle-app/shuffle-apps.yaml | 4 +- .../templates/shuffle-worker/_helpers.tpl | 52 ++++++++----------- .../shuffle-worker/shuffle-worker-dpl.yaml | 6 +-- 4 files changed, 53 insertions(+), 61 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl index f47f073e..d8718f3e 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl @@ -55,6 +55,16 @@ imagePullSecrets: {{- end }} {{- end -}} +{{/* +Return the match labels for ALL apps. +These match the labels of helm-deployed apps (shuffle.appInstance.labels), +as well as worker-deployed apps (deployK8sApp). +*/}} +{{- define "shuffle.app.matchLabels" -}} +app.kubernetes.io/name: shuffle-app +{{- end -}} + + {{/* Return the sanitized name of a shuffle app. Usage: @@ -79,45 +89,33 @@ Usage: {{ include "shuffle.appInstance.labels" (dict "app" $app "customLabels" .Values.commonLabels "context" $) }} */}} {{- define "shuffle.appInstance.labels" -}} -app.kubernetes.io/name: shuffle-app -app.kubernetes.io/instance: {{ include "shuffle.appInstance.fullname" .app }} -helm.sh/chart: {{ include "common.names.chart" .context }} -app.kubernetes.io/instance: {{ .context.Release.Name }} -app.kubernetes.io/managed-by: {{ .context.Release.Service }} -app.kubernetes.io/part-of: shuffle +{{- $customLabels := mustMerge (dict "app.kubernetes.io/name" "shuffle-app" "app.kubernetes.io/part-of" "shuffle") $customLabels -}} +{{ include "common.labels.standard" (dict "customLabels" $customLabels "context" $) }} app.shuffler.io/name: {{ include "shuffle.appInstance.name" .app }} app.shuffler.io/version: {{ .app.version | quote }} -{{- if .customLabels }} -{{- range $key, $value := .customLabels }} -{{ $key }}: {{ $value }} -{{- end }} -{{- end }} {{- end -}} -{{/* -Return the match labels for ALL apps. -These match the labels of helm-deployed apps (shuffle.appInstance.labels), -as well as worker-deployed apps (deployK8sApp). -*/}} -{{- define "shuffle.app.matchLabels" -}} -app.kubernetes.io/name: shuffle-app -{{- end -}} - - {{/* Return the match labels of a single app, deployed via helm. Usage: {{ include "shuffle.appInstance.matchLabels" (dict "app" $app "customLabels" .Values.commonLabels "context" $) }} */}} {{- define "shuffle.appInstance.matchLabels" -}} -app.kubernetes.io/name: shuffle-app +{{- $customLabels := mustMerge (dict "app.kubernetes.io/name" "shuffle-app" "app.kubernetes.io/part-of" "shuffle") $customLabels -}} +{{ include "common.labels.matchLabels" (dict "customLabels" $customLabels "context" $) }} app.shuffler.io/name: {{ include "shuffle.appInstance.name" .app }} app.shuffler.io/version: {{ .app.version | quote }} -{{- if .customLabels }} -{{- range $key, $value := .customLabels }} -{{ $key }}: {{ $value }} -{{- end }} -{{- end }} +{{- end -}} + +{{/* +Return a podAffinity/podAntiAffinity definition. +Usage: +{{ include "shuffle.appInstance.affinities.pods" (dict "type" "soft" "app" $app "customLabels" $podLabels "context" $) -}} +*/}} +{{- define "shuffle.appInstance.affinities.pods" -}} +{{- $customLabels := mustMerge (dict "app.kubernetes.io/name" "shuffle-app" "app.kubernetes.io/part-of" "shuffle") .customLabels -}} +{{- $extraMatchLabels := dict "app.shuffler.io/name" (include "shuffle.appInstance.name" .app) "app.shuffler.io/version" (.app.version | quote) }} +{{ include "common.affinities.pods" (dict "type" .type "customLabels" $customLabels "context" .context "extraMatchLabels" $extraMatchLabels )}} {{- end -}} {{/* diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml index 112c6241..df16f4c7 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml @@ -73,8 +73,8 @@ spec: affinity: {{- include "common.tplvalues.render" ( dict "value" $appValues.affinity "context" $) | nindent 8 }} {{- else }} affinity: - podAffinity: {{- include "common.affinities.pods" (dict "type" $appValues.podAffinityPreset "customLabels" $podLabels "context" $) | nindent 10 }} - podAntiAffinity: {{- include "common.affinities.pods" (dict "type" $appValues.podAntiAffinityPreset "customLabels" $podLabels "context" $) | nindent 10 }} + podAffinity: {{- include "shuffle.appInstance.affinities.pods" (dict "app" $app "type" $appValues.podAffinityPreset "customLabels" $podLabels "context" $) | nindent 10 }} + podAntiAffinity: {{- include "shuffle.appInstance.affinities.pods" (dict "app" $app "type" $appValues.podAntiAffinityPreset "customLabels" $podLabels "context" $) | nindent 10 }} nodeAffinity: {{- include "common.affinities.nodes" (dict "type" $appValues.nodeAffinityPreset.type "key" $appValues.nodeAffinityPreset.key "values" $appValues.nodeAffinityPreset.values) | nindent 10 }} {{- end }} {{- if $appValues.nodeSelector }} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl index b9f223fd..d58b3912 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl @@ -62,27 +62,6 @@ imagePullSecrets: {{- end }} {{- end -}} -{{/* -Return the labels for a specific worker instance deployed via helm. -Usage: -{{ include "shuffle.workerInstance.labels" (dict "customLabels" .Values.commonLabels "context" $) -}} -*/}} -{{- define "shuffle.workerInstance.labels" -}} -app.kubernetes.io/name: shuffle-worker -helm.sh/chart: {{ include "common.names.chart" .context }} -app.kubernetes.io/instance: {{ .context.Release.Name }} -app.kubernetes.io/managed-by: {{ .context.Release.Service }} -app.kubernetes.io/part-of: shuffle -{{- with .context.Chart.AppVersion }} -app.kubernetes.io/version: {{ . | replace "+" "_" | quote }} -{{- end -}} -{{- if .customLabels }} -{{- range $key, $value := .customLabels }} -{{ $key }}: {{ $value }} -{{- end }} -{{- end }} -{{- end -}} - {{/* Return the labels to match ALL workers. These match the labels of helm-deployed workers (shuffle.workerInstance.labels), @@ -92,19 +71,34 @@ as well as orborus-deployed workers (deployk8sworker). app.kubernetes.io/name: shuffle-worker {{- end -}} +{{/* +Return the labels for a specific worker instance deployed via helm. +Usage: +{{ include "shuffle.workerInstance.labels" (dict "customLabels" $podLabels "context" $) -}} +*/}} +{{- define "shuffle.workerInstance.labels" -}} +{{- $customLabels := mustMerge (dict "app.kubernetes.io/name" "shuffle-worker" "app.kubernetes.io/part-of" "shuffle") $customLabels -}} +{{ include "common.labels.standard" (dict "customLabels" $customLabels "context" $) }} +{{- end -}} + {{/* Return the labels to match a helm-deployed worker. Usage: -{{ include "shuffle.workerInstance.matchLabels" (dict "customLabels" .Values.commonLabels "context" $) -}} +{{ include "shuffle.workerInstance.matchLabels" (dict "customLabels" $podLabels "context" $) -}} */}} {{- define "shuffle.workerInstance.matchLabels" -}} -app.kubernetes.io/name: shuffle-worker -app.kubernetes.io/instance: {{ .context.Release.Name }} -{{- if .customLabels }} -{{- range $key, $value := .customLabels }} -{{ $key }}: {{ $value }} -{{- end }} -{{- end }} +{{- $customLabels := mustMerge (dict "app.kubernetes.io/name" "shuffle-worker" "app.kubernetes.io/part-of" "shuffle") $customLabels -}} +{{ include "common.labels.matchLabels" (dict "customLabels" $customLabels "context" $) }} +{{- end -}} + +{{/* +Return a podAffinity/podAntiAffinity definition. +Usage: +{{ include "shuffle.workerInstance.affinities.pods" (dict "type" "soft" "customLabels" $podLabels "context" $) -}} +*/}} +{{- define "shuffle.workerInstance.affinities.pods" -}} +{{- $customLabels := mustMerge (dict "app.kubernetes.io/name" "shuffle-worker" "app.kubernetes.io/part-of" "shuffle") .customLabels -}} +{{ include "common.affinities.pods" (dict "type" .type "customLabels" $customLabels "context" .context )}} {{- end -}} {{/* diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml index 09d00269..748fe232 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml @@ -36,8 +36,8 @@ spec: affinity: {{- include "common.tplvalues.render" ( dict "value" .Values.worker.affinity "context" $) | nindent 8 }} {{- else }} affinity: - podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.worker.podAffinityPreset "component" "orborus" "customLabels" $podLabels "context" $) | nindent 10 }} - podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.worker.podAntiAffinityPreset "component" "orborus" "customLabels" $podLabels "context" $) | nindent 10 }} + podAffinity: {{- include "shuffle.workerInstance.affinities.pods" (dict "type" .Values.worker.podAffinityPreset "customLabels" $podLabels "context" $) | nindent 10 }} + podAntiAffinity: {{- include "shuffle.workerInstance.affinities.pods" (dict "type" .Values.worker.podAntiAffinityPreset "customLabels" $podLabels "context" $) | nindent 10 }} nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.worker.nodeAffinityPreset.type "key" .Values.worker.nodeAffinityPreset.key "values" .Values.worker.nodeAffinityPreset.values) | nindent 10 }} {{- end }} {{- if .Values.worker.nodeSelector }} @@ -66,7 +66,7 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.worker.initContainers "context" $) | nindent 8 }} {{- end }} containers: - - name: orborus + - name: worker image: {{ template "shuffle.worker.image" . }} imagePullPolicy: {{ .Values.worker.image.pullPolicy }} {{- if .Values.worker.containerSecurityContext.enabled }} From 43030316e699216262c1580e8f88d27407483054 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Wed, 3 Dec 2025 12:56:05 +0100 Subject: [PATCH 20/41] fix template issues Signed-off-by: Pascal Sthamer --- .../shuffle/templates/shuffle-app/_helpers.tpl | 12 ++++++------ .../shuffle/templates/shuffle-worker/_helpers.tpl | 10 +++++----- .../templates/shuffle-worker/shuffle-worker-vpa.yaml | 1 - 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl index d8718f3e..6195023c 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl @@ -89,8 +89,8 @@ Usage: {{ include "shuffle.appInstance.labels" (dict "app" $app "customLabels" .Values.commonLabels "context" $) }} */}} {{- define "shuffle.appInstance.labels" -}} -{{- $customLabels := mustMerge (dict "app.kubernetes.io/name" "shuffle-app" "app.kubernetes.io/part-of" "shuffle") $customLabels -}} -{{ include "common.labels.standard" (dict "customLabels" $customLabels "context" $) }} +{{- $customLabels := mustMerge (dict "app.kubernetes.io/name" "shuffle-app" "app.kubernetes.io/part-of" "shuffle") (include "common.tplvalues.render" (dict "value" .customLabels "context" .context) | fromYaml) -}} +{{ include "common.labels.standard" (dict "customLabels" $customLabels "context" .context) }} app.shuffler.io/name: {{ include "shuffle.appInstance.name" .app }} app.shuffler.io/version: {{ .app.version | quote }} {{- end -}} @@ -101,8 +101,8 @@ Usage: {{ include "shuffle.appInstance.matchLabels" (dict "app" $app "customLabels" .Values.commonLabels "context" $) }} */}} {{- define "shuffle.appInstance.matchLabels" -}} -{{- $customLabels := mustMerge (dict "app.kubernetes.io/name" "shuffle-app" "app.kubernetes.io/part-of" "shuffle") $customLabels -}} -{{ include "common.labels.matchLabels" (dict "customLabels" $customLabels "context" $) }} +{{- $customLabels := mustMerge (dict "app.kubernetes.io/name" "shuffle-app" "app.kubernetes.io/part-of" "shuffle") (include "common.tplvalues.render" (dict "value" .customLabels "context" .context) | fromYaml) -}} +{{ include "common.labels.matchLabels" (dict "customLabels" $customLabels "context" .context) }} app.shuffler.io/name: {{ include "shuffle.appInstance.name" .app }} app.shuffler.io/version: {{ .app.version | quote }} {{- end -}} @@ -113,8 +113,8 @@ Usage: {{ include "shuffle.appInstance.affinities.pods" (dict "type" "soft" "app" $app "customLabels" $podLabels "context" $) -}} */}} {{- define "shuffle.appInstance.affinities.pods" -}} -{{- $customLabels := mustMerge (dict "app.kubernetes.io/name" "shuffle-app" "app.kubernetes.io/part-of" "shuffle") .customLabels -}} -{{- $extraMatchLabels := dict "app.shuffler.io/name" (include "shuffle.appInstance.name" .app) "app.shuffler.io/version" (.app.version | quote) }} +{{- $customLabels := mustMerge (dict "app.kubernetes.io/name" "shuffle-app" "app.kubernetes.io/part-of" "shuffle") (include "common.tplvalues.render" (dict "value" .customLabels "context" .context) | fromYaml) -}} +{{- $extraMatchLabels := dict "app.shuffler.io/name" (include "shuffle.appInstance.name" .app) "app.shuffler.io/version" (.app.version) }} {{ include "common.affinities.pods" (dict "type" .type "customLabels" $customLabels "context" .context "extraMatchLabels" $extraMatchLabels )}} {{- end -}} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl index d58b3912..ae8565af 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl @@ -77,8 +77,8 @@ Usage: {{ include "shuffle.workerInstance.labels" (dict "customLabels" $podLabels "context" $) -}} */}} {{- define "shuffle.workerInstance.labels" -}} -{{- $customLabels := mustMerge (dict "app.kubernetes.io/name" "shuffle-worker" "app.kubernetes.io/part-of" "shuffle") $customLabels -}} -{{ include "common.labels.standard" (dict "customLabels" $customLabels "context" $) }} +{{- $customLabels := mustMerge (dict "app.kubernetes.io/name" "shuffle-worker" "app.kubernetes.io/part-of" "shuffle") (include "common.tplvalues.render" (dict "value" .customLabels "context" .context) | fromYaml) -}} +{{ include "common.labels.standard" (dict "customLabels" $customLabels "context" .context) }} {{- end -}} {{/* @@ -87,8 +87,8 @@ Usage: {{ include "shuffle.workerInstance.matchLabels" (dict "customLabels" $podLabels "context" $) -}} */}} {{- define "shuffle.workerInstance.matchLabels" -}} -{{- $customLabels := mustMerge (dict "app.kubernetes.io/name" "shuffle-worker" "app.kubernetes.io/part-of" "shuffle") $customLabels -}} -{{ include "common.labels.matchLabels" (dict "customLabels" $customLabels "context" $) }} +{{- $customLabels := mustMerge (dict "app.kubernetes.io/name" "shuffle-worker" "app.kubernetes.io/part-of" "shuffle") (include "common.tplvalues.render" (dict "value" .customLabels "context" .context) | fromYaml) -}} +{{ include "common.labels.matchLabels" (dict "customLabels" $customLabels "context" .context) }} {{- end -}} {{/* @@ -97,7 +97,7 @@ Usage: {{ include "shuffle.workerInstance.affinities.pods" (dict "type" "soft" "customLabels" $podLabels "context" $) -}} */}} {{- define "shuffle.workerInstance.affinities.pods" -}} -{{- $customLabels := mustMerge (dict "app.kubernetes.io/name" "shuffle-worker" "app.kubernetes.io/part-of" "shuffle") .customLabels -}} +{{- $customLabels := mustMerge (dict "app.kubernetes.io/name" "shuffle-worker" "app.kubernetes.io/part-of" "shuffle") (include "common.tplvalues.render" (dict "value" .customLabels "context" .context) | fromYaml) -}} {{ include "common.affinities.pods" (dict "type" .type "customLabels" $customLabels "context" .context )}} {{- end -}} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-vpa.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-vpa.yaml index d8101831..fbad0237 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-vpa.yaml +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-vpa.yaml @@ -29,7 +29,6 @@ spec: apiVersion: {{ include "common.capabilities.deployment.apiVersion" . }} kind: Deployment name: {{ include "shuffle.worker.name" . }} - {{- end }} {{- if .Values.worker.autoscaling.vpa.updatePolicy }} updatePolicy: {{- with .Values.worker.autoscaling.vpa.updatePolicy.updateMode }} From 79f25e08223629d6147cbde4f10e3adaec401215 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Wed, 3 Dec 2025 12:56:47 +0100 Subject: [PATCH 21/41] set app container name Signed-off-by: Pascal Sthamer --- .../charts/shuffle/templates/shuffle-app/shuffle-apps.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml index df16f4c7..30f06e87 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml @@ -276,7 +276,7 @@ metadata: spec: resourcePolicy: containerPolicies: - - containerName: TODO + - containerName: app {{- with $appValues.autoscaling.vpa.controlledResources }} controlledResources: {{- toYaml . | nindent 8 }} From 7d43e6f5a86ac2a92a5570b1f5f4befc5ce0d1a3 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Wed, 3 Dec 2025 14:15:57 +0100 Subject: [PATCH 22/41] remove unneeded argument to app.matchLabels Signed-off-by: Pascal Sthamer --- .../templates/shuffle-app/shuffle-app-network-policy.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-network-policy.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-network-policy.yaml index a8493002..9c83a6a7 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-network-policy.yaml +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-network-policy.yaml @@ -9,9 +9,8 @@ metadata: annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} {{- end }} spec: - {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.app.podLabels .Values.commonLabels ) "context" . ) }} podSelector: - matchLabels: {{- include "shuffle.app.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 6 }} + matchLabels: {{- include "shuffle.app.matchLabels" . | nindent 6 }} policyTypes: - Ingress - Egress From 1f824ed19e731d92a1a67699c737ecd48c3cae68 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Wed, 3 Dec 2025 14:52:49 +0100 Subject: [PATCH 23/41] fix app images having a duplicate registry Signed-off-by: Pascal Sthamer --- .../charts/shuffle/templates/shuffle-worker/_helpers.tpl | 1 - 1 file changed, 1 deletion(-) diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl index ae8565af..9ccf46d6 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl @@ -119,7 +119,6 @@ WORKER_HOSTNAME: "shuffle-workers.{{ .Release.Namespace }}.svc.cluster.local" {{- if .Values.worker.manageAppDeployments }} # Shuffle app images -REGISTRY_URL: "{{ .Values.shuffle.appRegistry }}" SHUFFLE_BASE_IMAGE_REGISTRY: "{{ .Values.shuffle.appRegistry }}" SHUFFLE_BASE_IMAGE_NAME: "{{ .Values.shuffle.appBaseImageName }}" From df45fa35e1ed71be42613788f3dbe05b15c276e1 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Wed, 3 Dec 2025 14:53:20 +0100 Subject: [PATCH 24/41] enhance notes after installing / upgrading helm chart Signed-off-by: Pascal Sthamer --- functions/kubernetes/charts/shuffle/templates/NOTES.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/templates/NOTES.txt b/functions/kubernetes/charts/shuffle/templates/NOTES.txt index 10e38886..6a207525 100644 --- a/functions/kubernetes/charts/shuffle/templates/NOTES.txt +++ b/functions/kubernetes/charts/shuffle/templates/NOTES.txt @@ -22,6 +22,6 @@ Access the pod you want to debug by executing To access shuffle using port-forwarding: -1. Run `kubectl port-forward -n shuffle svc/shuffle-frontend 8080:http` +1. Run `kubectl port-forward -n {{ .Release.Namespace }} svc/{{ include "shuffle.frontend.name" . }} 8080:http` 2. Visit http://localhost:8080 with your browser - +3. Create the administrator account, if not already done From 48168c8b3410e0c82df288153c2a3fc142bfdaaa Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Wed, 3 Dec 2025 14:55:16 +0100 Subject: [PATCH 25/41] another attempt to fix shuffle app images Signed-off-by: Pascal Sthamer --- .../charts/shuffle/templates/shuffle-worker/_helpers.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl index 9ccf46d6..3d852d55 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl @@ -119,7 +119,7 @@ WORKER_HOSTNAME: "shuffle-workers.{{ .Release.Namespace }}.svc.cluster.local" {{- if .Values.worker.manageAppDeployments }} # Shuffle app images -SHUFFLE_BASE_IMAGE_REGISTRY: "{{ .Values.shuffle.appRegistry }}" +REGISTRY_URL: "{{ .Values.shuffle.appRegistry }}" SHUFFLE_BASE_IMAGE_NAME: "{{ .Values.shuffle.appBaseImageName }}" # Shuffle app deployment configuration From 135a2dab36b49b1fa0ba60bb9ea241a758c3958a Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Wed, 3 Dec 2025 15:06:27 +0100 Subject: [PATCH 26/41] add missing helper shuffle.worker.imagePullSecrets Signed-off-by: Pascal Sthamer --- .../charts/shuffle/templates/shuffle-worker/_helpers.tpl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl index 3d852d55..4bc851a2 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl @@ -21,6 +21,13 @@ Return the proper Shuffle worker image name {{- include "common.images.image" ( dict "imageRoot" .Values.worker.image "global" .Values.global "chart" .Chart ) -}} {{- end -}} +{{/* +Return the proper Docker Image Registry Secret Names for the worker pod +*/}} +{{- define "shuffle.worker.imagePullSecrets" -}} +{{- include "common.images.renderPullSecrets" (dict "images" (list .Values.worker.image) "context" $) -}} +{{- end -}} + {{/* Create the name of the service account to use for Shuffle workers */}} From 454ca0020e78495bebb905e6524fe07f9ce50493 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Wed, 3 Dec 2025 15:11:40 +0100 Subject: [PATCH 27/41] fix authentication failing for helm-deployed worker Signed-off-by: Pascal Sthamer --- .../charts/shuffle/templates/shuffle-worker/_helpers.tpl | 1 + 1 file changed, 1 insertion(+) diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl index 4bc851a2..4caf4f57 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl @@ -115,6 +115,7 @@ KEY: VALUE {{- define "shuffle.workerInstance.env" -}} IS_KUBERNETES: "true" KUBERNETES_NAMESPACE: "{{ .Release.Namespace }}" +SHUFFLE_SWARM_CONFIG: "run" # Shuffle Worker requires this to be set even when using K8s instead of swarm BASE_URL: {{ include "shuffle.backend.baseUrl" . | quote }} SHUFFLE_APP_EXPOSED_PORT: {{ .Values.app.exposedContainerPort | quote }} From 7e4aee548ba1e2243e83640448b93c950f75ccb4 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Fri, 5 Dec 2025 15:15:46 +0100 Subject: [PATCH 28/41] replace dots in app version Signed-off-by: Pascal Sthamer --- .../charts/shuffle/templates/shuffle-app/_helpers.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl index 6195023c..5963f9c1 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl @@ -80,7 +80,7 @@ Usage: {{ include "shuffle.appInstance.fullname" $app }} */}} {{- define "shuffle.appInstance.fullname" -}} -{{ printf "%s-%s" .name .version | replace "_" "-" | lower }} +{{ printf "%s-%s" .name .version | replace "." "-" | replace "_" "-" | lower }} {{- end -}} {{/* From 49c996bcf2c19f44a2e68696788cec8c774f2a1e Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Fri, 5 Dec 2025 15:58:29 +0100 Subject: [PATCH 29/41] allow to configure shuffle worker port Signed-off-by: Pascal Sthamer --- functions/kubernetes/charts/shuffle/README.md | 128 ++++++++++++++++-- .../shuffle-worker/shuffle-worker-dpl.yaml | 2 + .../shuffle-worker/shuffle-worker-svc.yaml | 6 +- .../charts/shuffle/values.schema.json | 10 ++ .../kubernetes/charts/shuffle/values.yaml | 4 + 5 files changed, 138 insertions(+), 12 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/README.md b/functions/kubernetes/charts/shuffle/README.md index 0535cb9f..058b4f68 100644 --- a/functions/kubernetes/charts/shuffle/README.md +++ b/functions/kubernetes/charts/shuffle/README.md @@ -184,7 +184,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia ## Parameters -### Global parameters +#### Global parameters | Name | Description | Value | | ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | @@ -194,7 +194,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `global.compatibility.openshift.adaptSecurityContext` | Adapt the securityContext sections of the deployment to make them compatible with Openshift restricted-v2 SCC: remove runAsUser, runAsGroup and fsGroup and let the platform use their allowed default IDs. Possible values: auto (apply if the detected running cluster is Openshift), force (perform the adaptation always), disabled (do not perform adaptation) | `auto` | | `global.compatibility.omitEmptySeLinuxOptions` | If set to true, removes the seLinuxOptions from the securityContexts when it is set to an empty object | `false` | -### Common parameters +#### Common parameters | Name | Description | Value | | ------------------------ | --------------------------------------------------------------------------------------- | --------------- | @@ -210,7 +210,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `diagnosticMode.command` | Command to override all containers in the chart release | `["sleep"]` | | `diagnosticMode.args` | Args to override all containers in the chart release | `["infinity"]` | -### Shared Shuffle Parameters +#### Shared Shuffle Parameters | Name | Description | Value | | -------------------------- | -------------------------------------------------------------------------------------------------------------------------- | --------------- | @@ -220,7 +220,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `shuffle.appBaseImageName` | The base image used for shuffle apps. The final image for an app is //: | `frikky` | | `shuffle.timezone` | The timezone used by Shuffle | `Europe/Berlin` | -### backend Parameters +#### backend Parameters | Name | Description | Value | | ----------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------- | @@ -335,7 +335,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `backend.apps.downloadBranch` | The branch from which apps should be downloaded on startup. | `master` | | `backend.apps.forceUpdate` | Force an update of apps on startup. | `false` | -### frontend Parameters +#### frontend Parameters | Name | Description | Value | | ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------- | @@ -441,7 +441,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `frontend.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` | | `frontend.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | -### orborus Parameters +#### orborus Parameters | Name | Description | Value | | ----------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------- | @@ -547,7 +547,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `orborus.executionConcurrency` | The maximum amount of concurrent workflow executions per worker | `25` | | `orborus.manageWorkerDeployments` | Whether workers are deployed and managed by orborus. When disabled, every worker is expected to be already deployed (see worker.enableHelmDeployment). | `true` | -### worker Parameters +#### worker Parameters | Name | Description | Value | | ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------ | @@ -559,6 +559,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `worker.image.pullPolicy` | worker image pull policy. Only effective with worker.enableHelmDeployment. | `IfNotPresent` | | `worker.image.pullSecrets` | worker image pull secrets. Only effective with worker.enableHelmDeployment. | `[]` | | `worker.replicaCount` | Number of worker replicas to deploy. Only effective with worker.enableHelmDeployment. | `1` | +| `worker.containerPorts.http` | backend HTTP container port | `33333` | | `worker.extraContainerPorts` | Optionally specify extra list of additional ports for worker containers. Only effective with worker.enableHelmDeployment. | `[]` | | `worker.livenessProbe.enabled` | Enable livenessProbe on worker containers. Only effective with worker.enableHelmDeployment. | `false` | | `worker.livenessProbe.initialDelaySeconds` | Initial delay seconds for livenessProbe | `0` | @@ -654,7 +655,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `worker.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | | `worker.manageAppDeployments` | Whether apps are deployed and managed by worker. When disabled, every used app is expected to to be already deployed (see apps.enabled). | `true` | -### app Parameters +#### app Parameters | Name | Description | Value | | ------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | @@ -762,7 +763,116 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `app.sdkTimeout` | The timeout in seconds for app actions. | `300` | | `app.disableLogs` | Do not capture app logs. By default, app logs are captured, so that they are visible in the frontend. | `false` | -### Parameters to deploy apps using helm +#### Parameters to deploy apps using helm + +| Name | Description | Value | +| ----------------------------- | -------------------------------------------------- | ------- | +| `apps.enabled` | Whether apps should be deployed using helm. | `false` | +| `apps.shuffleTools.enabled` | Whether the shuffle-tools app is enabled | `true` | +| `apps.shuffleTools.version` | The version of the shuffle-tools app to deploy. | `1.2.0` | +| `apps.shuffleSubflow.enabled` | Whether the shuffle-subflow app is enabled | `true` | +| `apps.shuffleSubflow.version` | The version of the shuffle-subflow app to deploy. | `1.1.0` | +| `apps.http.enabled` | Whether the http app is enabled | `true` | +| `apps.http.version` | The version of the http app to deploy. | `1.4.0` | +| `apps.MY_APP.app` | The name of the app (required, e.g. shuffle-tools) | | +| `apps.MY_APP.version` | The version of the app (required, e.g. 1.2.0) | | + +#### Traffic Exposure Parameters + +| Name | Description | Value | +| -------------------------- | ----------------------------------------------------------------------------------------------------- | --------------- | +| `ingress.enabled` | Enable ingress record generation for frontend and backend | `false` | +| `ingress.pathType` | Ingress path type for the frontend path | `Prefix` | +| `ingress.backendPathType` | Ingress path type for the backend path | `Prefix` | +| `ingress.apiVersion` | Force Ingress API version (automatically detected if not set) | `""` | +| `ingress.hostname` | Default host for the ingress record | `shuffle.local` | +| `ingress.ingressClassName` | IngressClass that will be be used to implement the Ingress (Kubernetes 1.18+) | `nginx` | +| `ingress.path` | Ingress path for Shuffle frontend | `"/"` | +| `ingress.backendPath` | Ingress path for Shuffle backend | `"/api/"` | +| `ingress.annotations` | Additional annotations for the Ingress resource. | `{}` | +| `ingress.tls` | Enable TLS configuration for the host defined at `ingress.hostname` parameter | `false` | +| `ingress.selfSigned` | Create a TLS secret for this ingress record using self-signed certificates generated by Helm | `false` | +| `ingress.extraHosts` | An array with additional hostname(s) to be covered with the ingress record | `[]` | +| `ingress.extraPaths` | An array with additional arbitrary paths that may need to be added to the ingress under the main host | `[]` | +| `ingress.extraTls` | TLS configuration for additional hostname(s) to be covered with this ingress record | `[]` | +| `ingress.secrets` | Custom TLS certificates as secrets | `[]` | +| `ingress.extraRules` | Additional rules to be covered with this ingress record | `[]` | + +#### Istio Parameters + +| Name | Description | Value | +| --------------------------------------- | ------------------------------------------------------------------------------- | ------------------------ | +| `istio.enabled` | Enable creation of an Istio Gateway and VirtualService for frontend and backend | `false` | +| `istio.apiVersion` | The istio apiVersion to use for Gateway and VirtualService resources | `networking.istio.io/v1` | +| `istio.hosts` | One or more hosts exposed by Istio | `[]` | +| `istio.gateway.annotations` | Additional annotations for the Gateway resource | `{}` | +| `istio.gateway.selector` | The selector matches the ingress gateway pod labels | `{ istio: ingress }` | +| `istio.gateway.http.enabled` | Enable HTTP server port 80 | `true` | +| `istio.gateway.http.httpsRedirect` | If set to true, a 301 redirect is send for all HTTP connections | `false` | +| `istio.gateway.https.enabled` | Enable HTTPS server on port 443 | `false` | +| `istio.gateway.https.tlsCredentialName` | The name of the secret that holds the TLS certs including the CA certificates. | `""` | +| `istio.gateway.https.tlsCipherSuites` | If specified, only support the specified cipher list. | `[]` | +| `istio.gateway.extraServers` | Additional servers for the Gateway resource | `[]` | +| `istio.virtualService.annotations` | Additional annotations for the VirtualService resource. | `{}` | +| `istio.virtualService.backendHeaders` | Header manipulation rules for backend traffic | `{}` | +| `istio.virtualService.frontendHeaders` | Header manipulation rules for frontend traffic | `{}` | + +#### Persistence Parameters + +| Name | Description | Value | +| ------------------------------------- | ------------------------------------------------- | ------------------- | +| `persistence.enabled` | Enable persistence using Persistent Volume Claims | `true` | +| `persistence.apps.existingClaim` | Name of an existing PVC to use | `""` | +| `persistence.apps.storageClass` | PVC Storage Class for shuffle-apps volume | `""` | +| `persistence.apps.subPath` | The sub path used in the volume | `""` | +| `persistence.apps.accessModes` | The access mode of the volume | `["ReadWriteOnce"]` | +| `persistence.apps.size` | The size of the volume | `5Gi` | +| `persistence.apps.annotations` | Annotations for the PVC | `{}` | +| `persistence.apps.selector` | Selector to match an existing Persistent Volume | `{}` | +| `persistence.appBuilder.storageClass` | PVC Storage Class for backend-apps-claim volume | `""` | +| `persistence.appBuilder.accessModes` | The access mode of the volume | `["ReadWriteOnce"]` | +| `persistence.appBuilder.size` | The size of the volume | `5Gi` | +| `persistence.appBuilder.annotations` | Annotations for the PVC | `{}` | +| `persistence.appBuilder.selector` | Selector to match an existing Persistent Volume | `{}` | +| `persistence.files.existingClaim` | Name of an existing PVC to use | `""` | +| `persistence.files.storageClass` | PVC Storage Class for shuffle-files volume | `""` | +| `persistence.files.subPath` | The sub path used in the volume | `""` | +| `persistence.files.accessModes` | The access mode of the volume | `["ReadWriteOnce"]` | +| `persistence.files.size` | The size of the volume | `5Gi` | +| `persistence.files.annotations` | Annotations for the PVC | `{}` | +| `persistence.files.selector` | Selector to match an existing Persistent Volume | `{}` | + +#### Init Container Parameters + +| Name | Description | Value | +| ----------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------ | +| `volumePermissions.enabled` | Enable init container that changes the owner/group of the PV mount point to `runAsUser:fsGroup` | `false` | +| `volumePermissions.image.registry` | OS Shell + Utility image registry | `docker.io` | +| `volumePermissions.image.repository` | OS Shell + Utility image repository | `bitnamilegacy/os-shell` | +| `volumePermissions.image.tag` | OS Shell + Utility image tag (immutable tags are recommended) | `12-debian-12-r30` | +| `volumePermissions.image.pullPolicy` | OS Shell + Utility image pull policy | `IfNotPresent` | +| `volumePermissions.image.pullSecrets` | OS Shell + Utility image pull secrets | `[]` | +| `volumePermissions.resourcesPreset` | Set init container resources according to one common preset (allowed values: none, nano, small, medium, large, xlarge, 2xlarge). This is ignored if volumePermissions.resources is set (volumePermissions.resources is recommended for production). | `nano` | +| `volumePermissions.resources` | Set init container requests and limits for different resources like CPU or memory (essential for production workloads) | `{}` | +| `volumePermissions.containerSecurityContext.enabled` | Enabled init container' Security Context | `true` | +| `volumePermissions.containerSecurityContext.seLinuxOptions` | Set SELinux options in init container | `{}` | +| `volumePermissions.containerSecurityContext.runAsUser` | Set init container's Security Context runAsUser | `0` | + +#### OpenSearch Parameters + +| Name | Description | Value | +| -------------------- | ----------------------------------------------------- | ------ | +| `opensearch.enabled` | Switch to enable or disable the opensearch helm chart | `true` | + +#### Vault Parameters + +| Name | Description | Value | +| --------------- | -------------------------------------------------------------------------- | ----- | +| `vault.role` | Specify the Vault role, which should be used to get the secret from Vault. | `""` | +| `vault.secrets` | A list of VaultSecrets to create | `[]` | + +#### Other Parameters + | Name | Description | Value | | ----------------------------- | -------------------------------------------------- | ------- | diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml index 748fe232..c60a42e0 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml @@ -106,6 +106,8 @@ spec: resources: {{- include "common.resources.preset" (dict "type" .Values.worker.resourcesPreset) | nindent 12 }} {{- end }} ports: + - name: http + containerPort: {{ .Values.worker.containerPorts.http }} {{- if .Values.worker.extraContainerPorts }} {{- include "common.tplvalues.render" (dict "value" .Values.worker.extraContainerPorts "context" $) | nindent 12 }} {{- end }} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-svc.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-svc.yaml index b9f355a5..3f116f27 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-svc.yaml +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-svc.yaml @@ -13,10 +13,10 @@ spec: type: ClusterIP ports: - name: http - port: 33333 - targetPort: 33333 + port: {{ .Values.worker.containerPorts.http }} + targetPort: http protocol: TCP appProtocol: http {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.worker.podLabels .Values.commonLabels ) "context" . ) }} selector: {{- include "shuffle.workerInstance.matchLabels" (dict "customLabels" $podLabels "context" .) | nindent 4 }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/functions/kubernetes/charts/shuffle/values.schema.json b/functions/kubernetes/charts/shuffle/values.schema.json index 90596374..8bc0ef85 100644 --- a/functions/kubernetes/charts/shuffle/values.schema.json +++ b/functions/kubernetes/charts/shuffle/values.schema.json @@ -2120,6 +2120,16 @@ "description": "Number of worker replicas to deploy. Only effective with worker.enableHelmDeployment.", "default": 1 }, + "containerPorts": { + "type": "object", + "properties": { + "http": { + "type": "number", + "description": "backend HTTP container port", + "default": 33333 + } + } + }, "extraContainerPorts": { "type": "array", "description": "Optionally specify extra list of additional ports for worker containers. Only effective with worker.enableHelmDeployment.", diff --git a/functions/kubernetes/charts/shuffle/values.yaml b/functions/kubernetes/charts/shuffle/values.yaml index 25e7654c..27aeca90 100644 --- a/functions/kubernetes/charts/shuffle/values.yaml +++ b/functions/kubernetes/charts/shuffle/values.yaml @@ -1375,6 +1375,10 @@ worker: ## @param worker.replicaCount Number of worker replicas to deploy. Only effective with worker.enableHelmDeployment. ## replicaCount: 1 + ## @param worker.containerPorts.http backend HTTP container port + ## + containerPorts: + http: 33333 ## @param worker.extraContainerPorts Optionally specify extra list of additional ports for worker containers. Only effective with worker.enableHelmDeployment. ## e.g: ## extraContainerPorts: From 82edd747ad58bbfca647d4c79104dee1f0aa4ddf Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Fri, 5 Dec 2025 15:59:28 +0100 Subject: [PATCH 30/41] disable cleanup in k8s Signed-off-by: Pascal Sthamer --- .../charts/shuffle/templates/orborus/orborus-dpl.yaml | 2 ++ .../shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml index 88f19bed..da5f502e 100644 --- a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml +++ b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml @@ -82,6 +82,8 @@ spec: args: {{- include "common.tplvalues.render" (dict "value" .Values.orborus.args "context" $) | nindent 12 }} {{- end }} env: + - name: CLEANUP + value: "false" # Do not remove resources when restarting orborus {{- $env := include "shuffle.orborus.env" . | fromYaml }} {{- range $key, $val := $env }} - name: {{ $key | quote }} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml index c60a42e0..1e724ab3 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml @@ -83,6 +83,8 @@ spec: args: {{- include "common.tplvalues.render" (dict "value" .Values.worker.args "context" $) | nindent 12 }} {{- end }} env: + - name: CLEANUP + value: "false" # Do not remove resources when restarting worker {{- $env := include "shuffle.workerInstance.env" . | fromYaml }} {{- range $key, $val := $env }} - name: {{ $key | quote }} From fb07662d7a869bf007ba5ea3e9e034b5c2a800c4 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Fri, 5 Dec 2025 16:00:13 +0100 Subject: [PATCH 31/41] set env variables on helm-deployed apps so that they work as expected Signed-off-by: Pascal Sthamer --- .../templates/shuffle-app/shuffle-apps.yaml | 12 ++++++++++- .../templates/shuffle-worker/_helpers.tpl | 21 ++++++++++++------- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml index 30f06e87..e15ae74b 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml @@ -120,6 +120,16 @@ spec: args: {{- include "common.tplvalues.render" (dict "value" $appValues.args "context" $) | nindent 12 }} {{- end }} env: + - name: AUTHORIZATION + value: "" + - name: EXECUTIONID + value: "" + - name: BASE_URL + value: {{ include "shuffle.worker.baseUrl" $ | quote }} + - name: CALLBACK_URL + value: {{ include "shuffle.backend.baseUrl" $ | quote }} + - name: SHUFFLE_SWARM_CONFIG + value: run # Shuffle Worker requires this to be set even when using K8s instead of swarm {{- $env := include "shuffle.appInstance.env" $ | fromYaml }} {{- range $key, $val := $env }} - name: {{ $key | quote }} @@ -302,4 +312,4 @@ spec: {{- end }} {{- end }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl index 4caf4f57..692fd6f2 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl @@ -108,6 +108,18 @@ Usage: {{ include "common.affinities.pods" (dict "type" .type "customLabels" $customLabels "context" .context )}} {{- end -}} +{{- define "shuffle.worker.hostname" -}} +{{- if .Values.worker.enableHelmDeployment -}} +http://{{ include "shuffle.worker.name" . }}.{{ .Release.Namespace }}.svc.cluster.local +{{- else -}} +http://shuffle-workers.{{ .Release.Namespace }}.svc.cluster.local +{{- end -}} +{{- end -}} + +{{- define "shuffle.worker.baseUrl" -}} +{{ include "shuffle.worker.hostname" . }}:{{ .Values.worker.containerPorts.http }} +{{- end -}} + {{/* Return the environment variables of shuffle-worker in the format KEY: VALUE @@ -118,12 +130,7 @@ KUBERNETES_NAMESPACE: "{{ .Release.Namespace }}" SHUFFLE_SWARM_CONFIG: "run" # Shuffle Worker requires this to be set even when using K8s instead of swarm BASE_URL: {{ include "shuffle.backend.baseUrl" . | quote }} SHUFFLE_APP_EXPOSED_PORT: {{ .Values.app.exposedContainerPort | quote }} - -{{- if .Values.worker.enableHelmDeployment }} -WORKER_HOSTNAME: "{{ include "shuffle.worker.name" . }}.{{ .Release.Namespace }}.svc.cluster.local" -{{- else }} -WORKER_HOSTNAME: "shuffle-workers.{{ .Release.Namespace }}.svc.cluster.local" -{{- end }} +WORKER_HOSTNAME: {{ include "shuffle.worker.hostname" . }} {{- if .Values.worker.manageAppDeployments }} # Shuffle app images @@ -164,4 +171,4 @@ SHUFFLE_APP_EPHEMERAL_STORAGE_LIMIT: {{ (index $appResources.limits "ephemeral-s # Include shuffle app environment variables. Worker passes them down to apps, when creating their deployment. {{ include "shuffle.appInstance.env" . }} {{- end }} -{{- end -}} \ No newline at end of file +{{- end -}} From 825b55c3e916454fff33ee52968e20bd940e2c01 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Fri, 5 Dec 2025 16:35:02 +0100 Subject: [PATCH 32/41] use shuffle-workers for worker name Signed-off-by: Pascal Sthamer --- .../charts/shuffle/templates/shuffle-worker/_helpers.tpl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl index 692fd6f2..8feafa76 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl @@ -1,9 +1,17 @@ {{/* Return the common name for worker components */}} +{{/* +Shuffle currently hardcodes the shuffle-workers:33333 address at some places. +Until that can be properly configured, we make sure that the worker deployment and service +are named exactly the same as the deployment and service that orborus would create. {{- define "shuffle.worker.name" -}} {{- printf "%s-worker" (include "common.names.fullname" .) | trunc 63 -}} {{- end -}} +*/}} +{{- define "shuffle.worker.name" -}} +shuffle-workers +{{- end -}} {{/* Return the common labels for worker components deployed via helm. From e664f6571e06793e44bd23e1c4b72fe3992a5fc8 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Fri, 5 Dec 2025 16:35:17 +0100 Subject: [PATCH 33/41] fix app services and images Signed-off-by: Pascal Sthamer --- .../charts/shuffle/templates/shuffle-app/shuffle-apps.yaml | 5 +++-- functions/kubernetes/charts/shuffle/values.yaml | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml index e15ae74b..975dd8f8 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml @@ -4,7 +4,7 @@ {{- if and (not ($ignoredKeys | has $key)) $app.enabled }} {{/* Merge .Values.app into $app, giving precedence to the values defined on the app. */}} -{{- $appValues := mustMerge $app $.Values.app -}} +{{- $appValues := mustDeepCopy (mustMerge $app $.Values.app) -}} {{/* use shuffle.appRegistry as default image registry */}} {{- $_ := set $appValues.image "registry" ($appValues.image.registry | default $.Values.shuffle.appRegistry) -}} @@ -31,6 +31,8 @@ spec: targetPort: {{ $.Values.app.exposedContainerPort }} protocol: TCP appProtocol: http + {{- $podLabels := include "common.tplvalues.merge" (dict "values" (list $appValues.podLabels $.Values.commonLabels) "context" $) }} + selector: {{- include "shuffle.appInstance.matchLabels" ( dict "app" $app "customLabels" $podLabels "context" $ ) | nindent 6 }} --- apiVersion: {{ include "common.capabilities.deployment.apiVersion" $ }} kind: Deployment @@ -49,7 +51,6 @@ spec: {{- if $appValues.updateStrategy }} strategy: {{- toYaml $appValues.updateStrategy | nindent 4 }} {{- end }} - {{- $podLabels := include "common.tplvalues.merge" (dict "values" (list $appValues.podLabels $.Values.commonLabels) "context" $) }} selector: matchLabels: {{- include "shuffle.appInstance.matchLabels" ( dict "app" $app "customLabels" $podLabels "context" $ ) | nindent 6 }} template: diff --git a/functions/kubernetes/charts/shuffle/values.yaml b/functions/kubernetes/charts/shuffle/values.yaml index 27aeca90..a24d9708 100644 --- a/functions/kubernetes/charts/shuffle/values.yaml +++ b/functions/kubernetes/charts/shuffle/values.yaml @@ -2162,7 +2162,7 @@ apps: ## version: 1.2.0 - shuffleSubflow: + shuffleSubflow: ## @param apps.shuffleSubflow.enabled Whether the shuffle-subflow app is enabled ## enabled: true @@ -2183,7 +2183,7 @@ apps: ## @param apps.http.version The version of the http app to deploy. ## version: 1.4.0 - + # replicas, image, etc. ## @extra apps.MY_APP.app [string] The name of the app (required, e.g. shuffle-tools) From 25e73a3105b1303356b38833e9ef84a31e5f63c9 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Wed, 14 Jan 2026 09:44:57 +0100 Subject: [PATCH 34/41] formating, fix comment Signed-off-by: Pascal Sthamer --- .../kubernetes/charts/shuffle/values.yaml | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/values.yaml b/functions/kubernetes/charts/shuffle/values.yaml index a24d9708..43c612b3 100644 --- a/functions/kubernetes/charts/shuffle/values.yaml +++ b/functions/kubernetes/charts/shuffle/values.yaml @@ -95,7 +95,7 @@ shuffle: timezone: Europe/Berlin ## @section backend Parameters -## +## backend: ## backend image ## @param backend.image.registry backend image registry @@ -321,7 +321,7 @@ backend: ## Can be set to RollingUpdate or Recreate ## Backend uses ReadWriteOnce volumes by default, which is incompatible with RollingUpdate ## - type: Recreate + type: Recreate ## @param backend.priorityClassName backend pods' priorityClassName ## priorityClassName: "" @@ -1328,15 +1328,15 @@ orborus: ## istio: pilot ## extraEgress: [] - + ## @param orborus.executionConcurrency The maximum amount of concurrent workflow executions per worker ## executionConcurrency: 25 - + ## @param orborus.manageWorkerDeployments Whether workers are deployed and managed by orborus. When disabled, every worker is expected to be already deployed (see worker.enableHelmDeployment). ## This effectively removes required RBAC permissions from the shuffle-orborus service account to create deployments and services. ## Orborus might still attempt to create kubernetes objects, resulting in an error. There is currently no way to tell orborus, that it should not manage k8s resources. - ## You likely want to enable worker.enableHelmDeployment when enabling this. + ## You likely want to disable worker.enableHelmDeployment when enabling this. manageWorkerDeployments: true ## @section worker Parameters @@ -1471,7 +1471,7 @@ worker: sysctls: [] supplementalGroups: [] fsGroup: 1001 - + ## Configure Container Security Context ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container ## @param worker.containerSecurityContext.enabled Enabled worker container' Security Context @@ -1671,7 +1671,7 @@ worker: maxReplicas: "" targetCPU: "" targetMemory: "" - + ## Service configuration ## service: @@ -1738,7 +1738,7 @@ worker: ## istio: pilot ## extraEgress: [] - + ## @param worker.manageAppDeployments Whether apps are deployed and managed by worker. When disabled, every used app is expected to to be already deployed (see apps.enabled). ## This effectively removes required RBAC permissions from the shuffle-worker service account to create deployments and services. ## The worker might still attempt to create kubernetes objects, resulting in an error. There is currently no way to tell the worker, that it should not manage k8s resources. @@ -1865,7 +1865,7 @@ app: sysctls: [] supplementalGroups: [] fsGroup: 1001 - + ## Configure Container Security Context ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container ## @param app.containerSecurityContext.enabled Enabled app container' Security Context @@ -1962,7 +1962,7 @@ app: updateStrategy: ## Can be set to RollingUpdate or Recreate ## - type: RollingUpdate + type: RollingUpdate ## @param app.priorityClassName app pods' priorityClassName. Only effective for helm-deployed apps (see apps.enabled). ## priorityClassName: "" @@ -2127,7 +2127,7 @@ app: ## @param app.mountTmpVolume Whether a writable /tmp emptyDir volume should be mounted to the app. ## mountTmpVolume: false - ## @param app.exposedContainerPort The port that shuffle app containers will listen on for new requests. + ## @param app.exposedContainerPort The port that shuffle app containers will listen on for new requests. ## exposedContainerPort: 80 ## @param app.sdkTimeout The timeout in seconds for app actions. From 104f4a3e1464f27ee9d17005cbe3e89c0cceaf32 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Wed, 14 Jan 2026 09:57:16 +0100 Subject: [PATCH 35/41] improve docs Signed-off-by: Pascal Sthamer --- functions/kubernetes/charts/shuffle/values.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/values.yaml b/functions/kubernetes/charts/shuffle/values.yaml index 43c612b3..f64b2852 100644 --- a/functions/kubernetes/charts/shuffle/values.yaml +++ b/functions/kubernetes/charts/shuffle/values.yaml @@ -2161,6 +2161,7 @@ apps: ## @param apps.shuffleTools.version The version of the shuffle-tools app to deploy. ## version: 1.2.0 + # You can override .app.* values here, e.g. replicaCount, resources or image. shuffleSubflow: ## @param apps.shuffleSubflow.enabled Whether the shuffle-subflow app is enabled @@ -2172,6 +2173,7 @@ apps: ## @param apps.shuffleSubflow.version The version of the shuffle-subflow app to deploy. ## version: 1.1.0 + # You can override .app.* values here, e.g. replicaCount, resources or image. http: ## @param apps.http.enabled Whether the http app is enabled @@ -2183,8 +2185,7 @@ apps: ## @param apps.http.version The version of the http app to deploy. ## version: 1.4.0 - - # replicas, image, etc. + # You can override .app.* values here, e.g. replicaCount, resources or image. ## @extra apps.MY_APP.app [string] The name of the app (required, e.g. shuffle-tools) ## @extra apps.MY_APP.version [string] The version of the app (required, e.g. 1.2.0) From 1703145100eee63c1158832204a14cffde3ed26a Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Fri, 16 Jan 2026 17:13:44 +0100 Subject: [PATCH 36/41] mount tmp volume to apps by default required by waitress under high load Signed-off-by: Pascal Sthamer --- functions/kubernetes/charts/shuffle/README.md | 129 ++++++++++++++++-- .../charts/shuffle/values.schema.json | 4 +- .../kubernetes/charts/shuffle/values.yaml | 2 +- 3 files changed, 122 insertions(+), 13 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/README.md b/functions/kubernetes/charts/shuffle/README.md index 058b4f68..7dddb248 100644 --- a/functions/kubernetes/charts/shuffle/README.md +++ b/functions/kubernetes/charts/shuffle/README.md @@ -184,7 +184,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia ## Parameters -#### Global parameters +##### Global parameters | Name | Description | Value | | ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | @@ -194,7 +194,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `global.compatibility.openshift.adaptSecurityContext` | Adapt the securityContext sections of the deployment to make them compatible with Openshift restricted-v2 SCC: remove runAsUser, runAsGroup and fsGroup and let the platform use their allowed default IDs. Possible values: auto (apply if the detected running cluster is Openshift), force (perform the adaptation always), disabled (do not perform adaptation) | `auto` | | `global.compatibility.omitEmptySeLinuxOptions` | If set to true, removes the seLinuxOptions from the securityContexts when it is set to an empty object | `false` | -#### Common parameters +##### Common parameters | Name | Description | Value | | ------------------------ | --------------------------------------------------------------------------------------- | --------------- | @@ -210,7 +210,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `diagnosticMode.command` | Command to override all containers in the chart release | `["sleep"]` | | `diagnosticMode.args` | Args to override all containers in the chart release | `["infinity"]` | -#### Shared Shuffle Parameters +##### Shared Shuffle Parameters | Name | Description | Value | | -------------------------- | -------------------------------------------------------------------------------------------------------------------------- | --------------- | @@ -220,7 +220,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `shuffle.appBaseImageName` | The base image used for shuffle apps. The final image for an app is //: | `frikky` | | `shuffle.timezone` | The timezone used by Shuffle | `Europe/Berlin` | -#### backend Parameters +##### backend Parameters | Name | Description | Value | | ----------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------- | @@ -335,7 +335,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `backend.apps.downloadBranch` | The branch from which apps should be downloaded on startup. | `master` | | `backend.apps.forceUpdate` | Force an update of apps on startup. | `false` | -#### frontend Parameters +##### frontend Parameters | Name | Description | Value | | ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------- | @@ -441,7 +441,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `frontend.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` | | `frontend.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | -#### orborus Parameters +##### orborus Parameters | Name | Description | Value | | ----------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------- | @@ -547,7 +547,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `orborus.executionConcurrency` | The maximum amount of concurrent workflow executions per worker | `25` | | `orborus.manageWorkerDeployments` | Whether workers are deployed and managed by orborus. When disabled, every worker is expected to be already deployed (see worker.enableHelmDeployment). | `true` | -#### worker Parameters +##### worker Parameters | Name | Description | Value | | ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------ | @@ -655,7 +655,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `worker.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | | `worker.manageAppDeployments` | Whether apps are deployed and managed by worker. When disabled, every used app is expected to to be already deployed (see apps.enabled). | `true` | -#### app Parameters +##### app Parameters | Name | Description | Value | | ------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | @@ -758,12 +758,121 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `app.networkPolicy.allowExternalEgress` | Allow the pod to access any range of port and all destinations. | `true` | | `app.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` | | `app.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | -| `app.mountTmpVolume` | Whether a writable /tmp emptyDir volume should be mounted to the app. | `false` | +| `app.mountTmpVolume` | Whether a writable /tmp emptyDir volume should be mounted to the app. | `true` | | `app.exposedContainerPort` | The port that shuffle app containers will listen on for new requests. | `80` | | `app.sdkTimeout` | The timeout in seconds for app actions. | `300` | | `app.disableLogs` | Do not capture app logs. By default, app logs are captured, so that they are visible in the frontend. | `false` | -#### Parameters to deploy apps using helm +##### Parameters to deploy apps using helm + +| Name | Description | Value | +| ----------------------------- | -------------------------------------------------- | ------- | +| `apps.enabled` | Whether apps should be deployed using helm. | `false` | +| `apps.shuffleTools.enabled` | Whether the shuffle-tools app is enabled | `true` | +| `apps.shuffleTools.version` | The version of the shuffle-tools app to deploy. | `1.2.0` | +| `apps.shuffleSubflow.enabled` | Whether the shuffle-subflow app is enabled | `true` | +| `apps.shuffleSubflow.version` | The version of the shuffle-subflow app to deploy. | `1.1.0` | +| `apps.http.enabled` | Whether the http app is enabled | `true` | +| `apps.http.version` | The version of the http app to deploy. | `1.4.0` | +| `apps.MY_APP.app` | The name of the app (required, e.g. shuffle-tools) | | +| `apps.MY_APP.version` | The version of the app (required, e.g. 1.2.0) | | + +##### Traffic Exposure Parameters + +| Name | Description | Value | +| -------------------------- | ----------------------------------------------------------------------------------------------------- | --------------- | +| `ingress.enabled` | Enable ingress record generation for frontend and backend | `false` | +| `ingress.pathType` | Ingress path type for the frontend path | `Prefix` | +| `ingress.backendPathType` | Ingress path type for the backend path | `Prefix` | +| `ingress.apiVersion` | Force Ingress API version (automatically detected if not set) | `""` | +| `ingress.hostname` | Default host for the ingress record | `shuffle.local` | +| `ingress.ingressClassName` | IngressClass that will be be used to implement the Ingress (Kubernetes 1.18+) | `nginx` | +| `ingress.path` | Ingress path for Shuffle frontend | `"/"` | +| `ingress.backendPath` | Ingress path for Shuffle backend | `"/api/"` | +| `ingress.annotations` | Additional annotations for the Ingress resource. | `{}` | +| `ingress.tls` | Enable TLS configuration for the host defined at `ingress.hostname` parameter | `false` | +| `ingress.selfSigned` | Create a TLS secret for this ingress record using self-signed certificates generated by Helm | `false` | +| `ingress.extraHosts` | An array with additional hostname(s) to be covered with the ingress record | `[]` | +| `ingress.extraPaths` | An array with additional arbitrary paths that may need to be added to the ingress under the main host | `[]` | +| `ingress.extraTls` | TLS configuration for additional hostname(s) to be covered with this ingress record | `[]` | +| `ingress.secrets` | Custom TLS certificates as secrets | `[]` | +| `ingress.extraRules` | Additional rules to be covered with this ingress record | `[]` | + +##### Istio Parameters + +| Name | Description | Value | +| --------------------------------------- | ------------------------------------------------------------------------------- | ------------------------ | +| `istio.enabled` | Enable creation of an Istio Gateway and VirtualService for frontend and backend | `false` | +| `istio.apiVersion` | The istio apiVersion to use for Gateway and VirtualService resources | `networking.istio.io/v1` | +| `istio.hosts` | One or more hosts exposed by Istio | `[]` | +| `istio.gateway.annotations` | Additional annotations for the Gateway resource | `{}` | +| `istio.gateway.selector` | The selector matches the ingress gateway pod labels | `{ istio: ingress }` | +| `istio.gateway.http.enabled` | Enable HTTP server port 80 | `true` | +| `istio.gateway.http.httpsRedirect` | If set to true, a 301 redirect is send for all HTTP connections | `false` | +| `istio.gateway.https.enabled` | Enable HTTPS server on port 443 | `false` | +| `istio.gateway.https.tlsCredentialName` | The name of the secret that holds the TLS certs including the CA certificates. | `""` | +| `istio.gateway.https.tlsCipherSuites` | If specified, only support the specified cipher list. | `[]` | +| `istio.gateway.extraServers` | Additional servers for the Gateway resource | `[]` | +| `istio.virtualService.annotations` | Additional annotations for the VirtualService resource. | `{}` | +| `istio.virtualService.backendHeaders` | Header manipulation rules for backend traffic | `{}` | +| `istio.virtualService.frontendHeaders` | Header manipulation rules for frontend traffic | `{}` | + +##### Persistence Parameters + +| Name | Description | Value | +| ------------------------------------- | ------------------------------------------------- | ------------------- | +| `persistence.enabled` | Enable persistence using Persistent Volume Claims | `true` | +| `persistence.apps.existingClaim` | Name of an existing PVC to use | `""` | +| `persistence.apps.storageClass` | PVC Storage Class for shuffle-apps volume | `""` | +| `persistence.apps.subPath` | The sub path used in the volume | `""` | +| `persistence.apps.accessModes` | The access mode of the volume | `["ReadWriteOnce"]` | +| `persistence.apps.size` | The size of the volume | `5Gi` | +| `persistence.apps.annotations` | Annotations for the PVC | `{}` | +| `persistence.apps.selector` | Selector to match an existing Persistent Volume | `{}` | +| `persistence.appBuilder.storageClass` | PVC Storage Class for backend-apps-claim volume | `""` | +| `persistence.appBuilder.accessModes` | The access mode of the volume | `["ReadWriteOnce"]` | +| `persistence.appBuilder.size` | The size of the volume | `5Gi` | +| `persistence.appBuilder.annotations` | Annotations for the PVC | `{}` | +| `persistence.appBuilder.selector` | Selector to match an existing Persistent Volume | `{}` | +| `persistence.files.existingClaim` | Name of an existing PVC to use | `""` | +| `persistence.files.storageClass` | PVC Storage Class for shuffle-files volume | `""` | +| `persistence.files.subPath` | The sub path used in the volume | `""` | +| `persistence.files.accessModes` | The access mode of the volume | `["ReadWriteOnce"]` | +| `persistence.files.size` | The size of the volume | `5Gi` | +| `persistence.files.annotations` | Annotations for the PVC | `{}` | +| `persistence.files.selector` | Selector to match an existing Persistent Volume | `{}` | + +##### Init Container Parameters + +| Name | Description | Value | +| ----------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------ | +| `volumePermissions.enabled` | Enable init container that changes the owner/group of the PV mount point to `runAsUser:fsGroup` | `false` | +| `volumePermissions.image.registry` | OS Shell + Utility image registry | `docker.io` | +| `volumePermissions.image.repository` | OS Shell + Utility image repository | `bitnamilegacy/os-shell` | +| `volumePermissions.image.tag` | OS Shell + Utility image tag (immutable tags are recommended) | `12-debian-12-r30` | +| `volumePermissions.image.pullPolicy` | OS Shell + Utility image pull policy | `IfNotPresent` | +| `volumePermissions.image.pullSecrets` | OS Shell + Utility image pull secrets | `[]` | +| `volumePermissions.resourcesPreset` | Set init container resources according to one common preset (allowed values: none, nano, small, medium, large, xlarge, 2xlarge). This is ignored if volumePermissions.resources is set (volumePermissions.resources is recommended for production). | `nano` | +| `volumePermissions.resources` | Set init container requests and limits for different resources like CPU or memory (essential for production workloads) | `{}` | +| `volumePermissions.containerSecurityContext.enabled` | Enabled init container' Security Context | `true` | +| `volumePermissions.containerSecurityContext.seLinuxOptions` | Set SELinux options in init container | `{}` | +| `volumePermissions.containerSecurityContext.runAsUser` | Set init container's Security Context runAsUser | `0` | + +##### OpenSearch Parameters + +| Name | Description | Value | +| -------------------- | ----------------------------------------------------- | ------ | +| `opensearch.enabled` | Switch to enable or disable the opensearch helm chart | `true` | + +##### Vault Parameters + +| Name | Description | Value | +| --------------- | -------------------------------------------------------------------------- | ----- | +| `vault.role` | Specify the Vault role, which should be used to get the secret from Vault. | `""` | +| `vault.secrets` | A list of VaultSecrets to create | `[]` | + +##### Other Parameters + | Name | Description | Value | | ----------------------------- | -------------------------------------------------- | ------- | diff --git a/functions/kubernetes/charts/shuffle/values.schema.json b/functions/kubernetes/charts/shuffle/values.schema.json index 8bc0ef85..ca262866 100644 --- a/functions/kubernetes/charts/shuffle/values.schema.json +++ b/functions/kubernetes/charts/shuffle/values.schema.json @@ -3325,11 +3325,11 @@ "mountTmpVolume": { "type": "boolean", "description": "Whether a writable /tmp emptyDir volume should be mounted to the app.", - "default": false + "default": true }, "exposedContainerPort": { "type": "number", - "description": "The port that shuffle app containers will listen on for new requests. ", + "description": "The port that shuffle app containers will listen on for new requests.", "default": 80 }, "sdkTimeout": { diff --git a/functions/kubernetes/charts/shuffle/values.yaml b/functions/kubernetes/charts/shuffle/values.yaml index f64b2852..590d2c10 100644 --- a/functions/kubernetes/charts/shuffle/values.yaml +++ b/functions/kubernetes/charts/shuffle/values.yaml @@ -2126,7 +2126,7 @@ app: ## @param app.mountTmpVolume Whether a writable /tmp emptyDir volume should be mounted to the app. ## - mountTmpVolume: false + mountTmpVolume: true ## @param app.exposedContainerPort The port that shuffle app containers will listen on for new requests. ## exposedContainerPort: 80 From 138f66e27a280936f2dcf1938035d34c952a22df Mon Sep 17 00:00:00 2001 From: Pascal Sthamer Date: Mon, 2 Feb 2026 10:36:11 +0100 Subject: [PATCH 37/41] allow to create a service account per app Signed-off-by: Pascal Sthamer --- functions/kubernetes/charts/shuffle/README.md | 52 +++++++++++++------ .../templates/shuffle-app/shuffle-apps.yaml | 39 +++++++++++++- 2 files changed, 75 insertions(+), 16 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/README.md b/functions/kubernetes/charts/shuffle/README.md index 7dddb248..7c1315b9 100644 --- a/functions/kubernetes/charts/shuffle/README.md +++ b/functions/kubernetes/charts/shuffle/README.md @@ -127,12 +127,14 @@ When `worker.enableHelmDeployment` is set, env variables for app configuration a Configuration using env variables applies to ALL deployed apps. There is no way to assign different options (e.g. resources) to different apps, or scale apps individually. If you want full control, you can deploy apps using helm. This has the following advantages: + - full control over the deployment using helm values - granular control per app and version (e.g. have more replicas and resources for frequently used apps) - avoid problems with on-demand started apps (see https://github.com/Shuffle/Shuffle/issues/1739) To deploy apps using helm, set `apps.enabled=true`. By default, this deploys the `shuffle-tools`, `shuffle-subflow` and `http` apps. You can also deploy your own apps. See the following values file for an example. + ```yaml app: replicaCount: 1 # default to 1 replica per app @@ -142,26 +144,27 @@ app: apps: enabled: true # Deploy apps using helm. -# Configure default apps + # Configure default apps shuffleTools: enabled: true # default shuffleSubflow: enabled: true # default http: enabled: true # default -# optionally override defaults from app values: - replicaCount: 1 + # optionally override defaults from app values: + replicaCount: 1 resources: {} - -# Deploy additional apps (e.g. opensearch) + + # Deploy additional apps (e.g. opensearch) opensearch: enabled: true # required to actually deploy the app name: opensearch # required. The name and version must match the values of the `api.yaml` file of the app. version: 1.1.0 # required. -# optionally change app configuration: + # optionally change app configuration: replicaCount: 3 resources: {} ``` + The key of an app in the `apps` map does not matter, as long as it is unique. We are not using an array here, to allow overriding values in stage-specific value files or using the command line, e.g. `helm upgrade ... --set apps.shuffleTools.replicas=3`. @@ -173,6 +176,34 @@ It is possible to use a hybrid approach - deploy some apps using helm, while sti If you do not want Worker to manage app deployments, set `worker.manageAppDeployments=true`. This effectively removes the required permissions from the Shuffle Worker Kubernetes Service Account. You are required to deploy all apps that are in use by your Shuffle instance manually using Helm. +### Shuffle App Service Accounts + +By default a shared `shuffle-app` service account is used for all apps. +If you are deploying apps using helm, you can choose to have a dedicated service account per app. +To enable it, set `apps.MY_APP.serviceAccount.create=true` and provide a name using `apps.MY_APP.serviceAccount.name`. +You can also set `apps.MY_APP.serviceAccount.create=false` while still providing a name to use an existing service account. + +```yaml +apps: + myAppWithCustomServiceAccount: + enabled: true + name: my-custom-service-account + version: 1.0.0 + serviceAccount: + create: true + name: shuffle-app-myapp + + anotherAppWithExistingServiceAccount: + enabled: true + name: another-app + version: 1.0.0 + serviceAccount: + create: false + name: existing-service-account-name +``` + +All service accounts use the `shuffle-app` role by default. + ## OpenSearch Shuffle uses OpenSearch as its database. This helm chart installs a single-node OpenSearch cluster using [the Bitnami Helm Chart](https://github.com/bitnami/charts/blob/main/bitnami/opensearch/values.yaml). @@ -873,7 +904,6 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia ##### Other Parameters - | Name | Description | Value | | ----------------------------- | -------------------------------------------------- | ------- | | `apps.enabled` | Whether apps should be deployed using helm. | `false` | @@ -982,7 +1012,6 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia #### Other Parameters - | Name | Description | Value | | ----------------------------- | -------------------------------------------------- | ------- | | `apps.enabled` | Whether apps should be deployed using helm. | `false` | @@ -1090,10 +1119,3 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `vault.secrets` | A list of VaultSecrets to create | `[]` | ### Other Parameters - - - - - - - diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml index 975dd8f8..af04851f 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml @@ -12,6 +12,43 @@ {{- $_ := set $appValues.image "repository" ($appValues.image.repository | default (printf "%s/%s" $.Values.shuffle.appBaseImageName $appValues.name)) -}} {{/* use app version as default tag */}} {{- $_ := set $appValues.image "tag" ($appValues.image.tag | default $appValues.version) -}} + +{{/* Only create a service account if create is explicitly enabled on that specific app ($app not $appValues). Otherwise the shared shuffle-app service account is used. */}} +{{- $shouldCreateDedicatedServiceAccount := and $app.serviceAccount.create $app.serviceAccount.name -}} +{{- if $shouldCreateDedicatedServiceAccount }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $app.serviceAccount.name | trunc 63 | trimSuffix "-" }} + namespace: {{ include "common.names.namespace" $ | quote }} + labels: {{- include "shuffle.app.labels" ( dict "customLabels" $.Values.commonLabels "context" $ ) | nindent 4 }} + {{- if or $appValues.serviceAccount.annotations $.Values.commonAnnotations }} + {{- $annotations := include "common.tplvalues.merge" (dict "values" (list $appValues.serviceAccount.annotations $.Values.commonAnnotations) "context" $) }} + annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $ ) | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ $appValues.serviceAccount.automountServiceAccountToken }} +{{- include "shuffle.app.serviceAccount.imagePullSecrets" $ | nindent 0 }} +{{- if $appValues.rbac.create }} +--- +kind: RoleBinding +apiVersion: {{ include "common.capabilities.rbac.apiVersion" $ }} +metadata: + name: {{ $app.serviceAccount.name | trunc 63 | trimSuffix "-" }} + namespace: {{ include "common.names.namespace" $ | quote }} + labels: {{- include "shuffle.app.labels" ( dict "customLabels" $.Values.commonLabels "context" $ ) | nindent 4 }} + {{- if $.Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" $.Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +subjects: + - kind: ServiceAccount + name: {{ $app.serviceAccount.name | trunc 63 | trimSuffix "-" }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "shuffle.app.name" $ }} +{{- end }} +{{- end }} --- apiVersion: v1 kind: Service @@ -28,7 +65,7 @@ spec: ports: - name: http port: 80 - targetPort: {{ $.Values.app.exposedContainerPort }} + targetPort: {{ $appValues.exposedContainerPort }} protocol: TCP appProtocol: http {{- $podLabels := include "common.tplvalues.merge" (dict "values" (list $appValues.podLabels $.Values.commonLabels) "context" $) }} From 699e91d8db98ddf8d39acb32197a38b9c31d5ed6 Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Mon, 9 Feb 2026 13:15:19 +0530 Subject: [PATCH 38/41] new health opensearch endpoint --- backend/go-app/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 0f5983c8..451caa07 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5445,6 +5445,7 @@ func initHandlers() { r.HandleFunc("/api/v1/_ah/health", shuffle.HealthCheckHandler) r.HandleFunc("/api/v1/health", shuffle.RunOpsHealthCheck).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/health/stats", shuffle.GetOpsDashboardStats).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/health/opensearch-prefix", shuffle.HandleFixOpensearchPrefix).Methods("POST", "OPTIONS") // Make user related locations // Fix user changes with org From 874bdbcae55efb0ac910bbd2edaa3ca6381f3fde Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Mon, 9 Feb 2026 13:26:29 +0530 Subject: [PATCH 39/41] fix opensearch prefix button on health page --- frontend/src/components/HealthPage.jsx | 59 +++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 7 deletions(-) diff --git a/frontend/src/components/HealthPage.jsx b/frontend/src/components/HealthPage.jsx index 287ad9b1..b60a46be 100644 --- a/frontend/src/components/HealthPage.jsx +++ b/frontend/src/components/HealthPage.jsx @@ -24,6 +24,7 @@ const HealthPage = (props) => { const [liveExecutionsRange, setLiveExecutionsRange] = useState('1h'); // Default to 1h const [isHealthLoading, setIsHealthLoading] = useState(false); // Loading state for HealthBarChart const [isLiveExecutionsLoading, setIsLiveExecutionsLoading] = useState(false); // Loading state for LiveExecutionsChart + const [isFixingOpensearchPrefix, setIsFixingOpensearchPrefix] = useState(false); const isCloud = (window.location.host === "localhost:3002" || window.location.host === "shuffler.io") ? true : (import.meta.env.VITE_IS_SSR === "true"); @@ -364,19 +365,63 @@ const HealthPage = (props) => { } }; + const handleFixOpensearchPrefix = async () => { + if (isFixingOpensearchPrefix) { + return; + } + + setIsFixingOpensearchPrefix(true); + try { + const response = await fetch(`${globalUrl}/api/v1/health/opensearch-prefix`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }); + + const data = await response.json(); + if (!response.ok || !data.success) { + const reason = data && data.reason ? data.reason : "Failed to fix opensearch prefix"; + throw new Error(reason); + } + + const reindexed = data.reindexed ? data.reindexed.length : 0; + const aliasUpdates = data.alias_updates ? data.alias_updates.length : 0; + const deleted = data.deleted_indices ? data.deleted_indices.length : 0; + toast.success(`Fixed opensearch prefix (reindexed ${reindexed}, aliases ${aliasUpdates}, deleted ${deleted})`); + } catch (error) { + console.error("Error fixing opensearch prefix:", error); + toast.error(error.message || "Failed to fix opensearch prefix"); + } finally { + setIsFixingOpensearchPrefix(false); + } + }; + const healthBarData = updateChartData() return (
{/* Health Bar Chart Section */} - - - - - - - +
+ + + + + + + + +
{/* Loading Bar for HealthBarChart */} {isHealthLoading && ( From f2138932ac5efbd9a2c632cf1a6c1de288b93cea Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Mon, 9 Feb 2026 18:02:02 +0530 Subject: [PATCH 40/41] fix: main.go changes --- backend/go-app/main.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 8a1fe9b1..80e4f832 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -1263,7 +1263,7 @@ func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { // Should run calculations if len(org.SSOConfig.OpenIdAuthorization) > 0 { - baseSSOUrl = shuffle.GetOpenIdUrl(request, *org) + baseSSOUrl, err = shuffle.GetOpenIdUrl(request, *org, user, "") if err != nil { log.Printf("[ERROR] Failed getting OpenID URL for org %s: %s", org.Name, err) } @@ -5683,16 +5683,16 @@ func initHandlers() { r.HandleFunc("/api/v1/dashboards/{key}/widgets/{widget_id}", shuffle.HandleGetWidget).Methods("GET", "OPTIONS") // Need to add auth in pprof -// if strings.ToLower(os.Getenv("SHUFFLE_DEBUG_MEMORY")) == "true" || strings.ToLower(os.Getenv("DEBUG_MEMORY")) == "true" { -// log.Printf("[DEBUG] Memory debugging is enabled on /debug/pprof") -// r.HandleFunc("/debug/pprof/", pprof.Index) -// r.HandleFunc("/debug/pprof/heap", pprof.Handler("heap").ServeHTTP) -// r.HandleFunc("/debug/pprof/profile", pprof.Profile) -// r.HandleFunc("/debug/pprof/symbol", pprof.Symbol) -// r.HandleFunc("/debug/pprof/trace", pprof.Trace) -// } else { -// log.Printf("[DEBUG] Memory debugging is disabled. To enable, set SHUFFLE_DEBUG_MEMORY or DEBUG_MEMORY to true") -// } + // if strings.ToLower(os.Getenv("SHUFFLE_DEBUG_MEMORY")) == "true" || strings.ToLower(os.Getenv("DEBUG_MEMORY")) == "true" { + // log.Printf("[DEBUG] Memory debugging is enabled on /debug/pprof") + // r.HandleFunc("/debug/pprof/", pprof.Index) + // r.HandleFunc("/debug/pprof/heap", pprof.Handler("heap").ServeHTTP) + // r.HandleFunc("/debug/pprof/profile", pprof.Profile) + // r.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + // r.HandleFunc("/debug/pprof/trace", pprof.Trace) + // } else { + // log.Printf("[DEBUG] Memory debugging is disabled. To enable, set SHUFFLE_DEBUG_MEMORY or DEBUG_MEMORY to true") + // } r.Use(shuffle.RequestMiddleware) http.Handle("/", r) From 15c18ef5db58705661256d8a38c93de7e1cca513 Mon Sep 17 00:00:00 2001 From: Aditya <60684641+0x0elliot@users.noreply.github.com> Date: Mon, 9 Feb 2026 18:03:09 +0530 Subject: [PATCH 41/41] chore: updating dependencies - shuffle-shared and singul --- backend/go-app/go.mod | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 5de1a9e6..831f0894 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -26,8 +26,8 @@ require ( github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.9.79 - github.com/shuffle/singul v0.0.20 + github.com/shuffle/shuffle-shared v0.9.83 + github.com/shuffle/singul v0.0.25 golang.org/x/crypto v0.45.0 google.golang.org/api v0.236.0 google.golang.org/grpc v1.72.2 @@ -65,6 +65,7 @@ require ( github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect + github.com/coreos/go-oidc/v3 v3.17.0 // indirect github.com/cyphar/filepath-securejoin v0.4.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.6.0 // indirect @@ -75,15 +76,16 @@ require ( github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/frikky/schemaless v0.0.25 // indirect + github.com/frikky/schemaless v0.0.28 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect - github.com/go-jose/go-jose/v4 v4.0.5 // indirect + github.com/go-jose/go-jose/v4 v4.1.3 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.23.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/protobuf v1.5.4 // indirect