diff --git a/functions/kubernetes/charts/shuffle/templates/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/_helpers.tpl index b78602bb..c3fae7b2 100644 --- a/functions/kubernetes/charts/shuffle/templates/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/_helpers.tpl @@ -4,3 +4,46 @@ Return the proper image name (for the init container volume-permissions image) {{- define "shuffle.volumePermissions.image" -}} {{- include "common.images.image" ( dict "imageRoot" .Values.volumePermissions.image "global" .Values.global "chart" .Chart ) -}} {{- end -}} + +{{/* +Return a value for the GOMEMLIMIT env variable based on a given kubernetes resource memory limit. +Usage: +{{ include "shuffle.k8sMemoryLimitToGOMEMLIMIT" (dict "k8sMemoryLimit" $.Values.resources.limits.memory "context" $) }} +*/}} +{{- define "shuffle.k8sMemoryLimitToGOMEMLIMIT" -}} +{{- $memoryLimit := .k8sMemoryLimit | default "" -}} +{{- $result := "" -}} +{{- if and $memoryLimit (gt (len $memoryLimit) 0) -}} +{{- $bytes := 0 -}} +{{- if hasSuffix "Ki" $memoryLimit -}} +{{- $bytes = mul ($memoryLimit | trimSuffix "Ki" | int) 1024 -}} +{{- else if hasSuffix "Mi" $memoryLimit -}} +{{- $bytes = mul ($memoryLimit | trimSuffix "Mi" | int) 1048576 -}} +{{- else if hasSuffix "Gi" $memoryLimit -}} +{{- $bytes = mul ($memoryLimit | trimSuffix "Gi" | int) 1073741824 -}} +{{- else if hasSuffix "Ti" $memoryLimit -}} +{{- $bytes = mul ($memoryLimit | trimSuffix "Ti" | int) 1099511627776 -}} +{{- else if hasSuffix "Pi" $memoryLimit -}} +{{- $bytes = mul ($memoryLimit | trimSuffix "Pi" | int) 1125899906842624 -}} +{{- else if hasSuffix "Ei" $memoryLimit -}} +{{- $bytes = mul ($memoryLimit | trimSuffix "Ei" | int) 1152921504606846976 -}} +{{- else if hasSuffix "K" $memoryLimit -}} +{{- $bytes = mul ($memoryLimit | trimSuffix "K" | int) 1000 -}} +{{- else if hasSuffix "M" $memoryLimit -}} +{{- $bytes = mul ($memoryLimit | trimSuffix "M" | int) 1000000 -}} +{{- else if hasSuffix "G" $memoryLimit -}} +{{- $bytes = mul ($memoryLimit | trimSuffix "G" | int) 1000000000 -}} +{{- else if hasSuffix "T" $memoryLimit -}} +{{- $bytes = mul ($memoryLimit | trimSuffix "T" | int) 1000000000000 -}} +{{- else if hasSuffix "P" $memoryLimit -}} +{{- $bytes = mul ($memoryLimit | trimSuffix "P" | int) 1000000000000000 -}} +{{- else if hasSuffix "E" $memoryLimit -}} +{{- $bytes = mul ($memoryLimit | trimSuffix "E" | int) 1000000000000000000 -}} +{{- else -}} +{{- $bytes = $memoryLimit | int -}} +{{- end -}} +{{- $gomaxmem := div (mul $bytes 9) 10 -}} +{{- $result = printf "%d" $gomaxmem -}} +{{- end -}} +{{- $result -}} +{{- end -}} \ No newline at end of file diff --git a/functions/kubernetes/charts/shuffle/templates/backend/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/backend/_helpers.tpl index 1d61b12e..13cb0ce7 100644 --- a/functions/kubernetes/charts/shuffle/templates/backend/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/backend/_helpers.tpl @@ -116,4 +116,10 @@ REGISTRY_URL: "{{ .Values.shuffle.appRegistry }}" # Used by app builder {{- if .Values.backend.debug }} DEBUG: "true" {{- end }} +{{- if .Values.backend.autoGOMEMLIMIT }} +{{- $backendResources := (.Values.backend.resources | default (include "common.resources.preset" (dict "type" .Values.backend.resourcesPreset) | fromYaml)) -}} +{{- if and $backendResources $backendResources.limits $backendResources.limits.memory }} +GOMEMLIMIT: {{ include "shuffle.k8sMemoryLimitToGOMEMLIMIT" (dict "k8sMemoryLimit" $backendResources.limits.memory) | quote }} +{{- end }} +{{- end }} {{- end -}} \ No newline at end of file diff --git a/functions/kubernetes/charts/shuffle/templates/orborus/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/orborus/_helpers.tpl index c69c84f3..0af574a1 100644 --- a/functions/kubernetes/charts/shuffle/templates/orborus/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/orborus/_helpers.tpl @@ -92,6 +92,12 @@ SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY: {{ .Values.orborus.executionConcurrency | {{- if .Values.orborus.debug }} DEBUG: "true" {{- end }} +{{- if .Values.orborus.autoGOMEMLIMIT }} +{{- $orborusResources := (.Values.orborus.resources | default (include "common.resources.preset" (dict "type" .Values.orborus.resourcesPreset) | fromYaml)) -}} +{{- if and $orborusResources $orborusResources.limits $orborusResources.limits.memory }} +GOMEMLIMIT: {{ include "shuffle.k8sMemoryLimitToGOMEMLIMIT" (dict "k8sMemoryLimit" $orborusResources.limits.memory) | quote }} +{{- end }} +{{- end }} {{- if .Values.orborus.manageWorkerDeployments }} # Shuffle worker configuration diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl index bcce6348..fd6cb6c9 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl @@ -123,12 +123,13 @@ Return the environment variables of shuffle apps in the format KEY: VALUE Usage: {{- include "shuffle.appInstance.env" (dict "app" $appValues "context" $) -}} + +WARNING: Do NOT add environment variables here that would conflict with shuffle.worker.env or shuffle.orborus.env. +Worker also sets all env variables that are defined here, because they will be passed down to apps when worker.manageAppDeployments is set. +Instead, add them directly to the deployment template (shuffle-apps.yaml). */}} {{- define "shuffle.appInstance.env" -}} SHUFFLE_APP_SDK_TIMEOUT: {{ .app.sdkTimeout | quote }} SHUFFLE_APP_EXPOSED_PORT: {{ .app.exposedContainerPort | quote }} SHUFFLE_LOGS_DISABLED: {{ .app.disableLogs | quote }} -{{- if .app.debug }} -DEBUG: "true" -{{- end }} {{- end -}} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml index 6ee29a84..521e7fe8 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml @@ -168,6 +168,10 @@ spec: value: {{ include "shuffle.backend.baseUrl" $ | quote }} - name: SHUFFLE_SWARM_CONFIG value: run # Shuffle Worker requires this to be set even when using K8s instead of swarm + {{- if $appValues.debug }} + - name: DEBUG + value: "true" + {{- end }} {{- $env := include "shuffle.appInstance.env" (dict "app" $appValues "context" $) | fromYaml }} {{- range $key, $val := $env }} - name: {{ $key | quote }} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl index 2e57fe04..c0e75c72 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl @@ -131,17 +131,15 @@ http://shuffle-workers.{{ .Release.Namespace }}.svc.cluster.local {{/* Return the environment variables of shuffle-worker in the format KEY: VALUE + +WARNING: Do NOT add environment variables here that would conflict with shuffle.orborus.env. +Orborus also sets all env variables that are defined here, because they will be passed down to worker when orborus.manageWorkerDeployments is set. +Instead, add them directly to the deployment template (shuffle-worker-dpl.yaml). */}} {{- define "shuffle.workerInstance.env" -}} -IS_KUBERNETES: "true" -KUBERNETES_NAMESPACE: "{{ .Release.Namespace }}" SHUFFLE_SWARM_CONFIG: "run" # Shuffle Worker requires this to be set even when using K8s instead of swarm -BASE_URL: {{ include "shuffle.backend.baseUrl" . | quote }} SHUFFLE_APP_EXPOSED_PORT: {{ .Values.app.exposedContainerPort | quote }} WORKER_HOSTNAME: {{ include "shuffle.worker.hostname" . }} -{{- if .Values.worker.debug }} -DEBUG: "true" -{{- end }} {{- if .Values.worker.manageAppDeployments }} # Shuffle app images diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml index 1e724ab3..eb1c1682 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml @@ -85,6 +85,23 @@ spec: env: - name: CLEANUP value: "false" # Do not remove resources when restarting worker + - name: IS_KUBERNETES + value: "true" + - name: KUBERNETES_NAMESPACE + value: {{ .Release.Namespace | quote }} + - name: BASE_URL + value: {{ include "shuffle.backend.baseUrl" . | quote }} + {{- if .Values.worker.debug }} + - name: DEBUG + value: "true" + {{- end }} + {{- if .Values.worker.autoGOMEMLIMIT }} + {{- $workerResources := (.Values.worker.resources | default (include "common.resources.preset" (dict "type" .Values.worker.resourcesPreset) | fromYaml)) -}} + {{- if and $workerResources $workerResources.limits $workerResources.limits.memory }} + - name: GOMEMLIMIT + value: {{ include "shuffle.k8sMemoryLimitToGOMEMLIMIT" (dict "k8sMemoryLimit" $workerResources.limits.memory) | quote }} + {{- end }} + {{- end }} {{- $env := include "shuffle.workerInstance.env" . | fromYaml }} {{- range $key, $val := $env }} - name: {{ $key | quote }} diff --git a/functions/kubernetes/charts/shuffle/values.yaml b/functions/kubernetes/charts/shuffle/values.yaml index 27e04aa2..87fd3bb0 100644 --- a/functions/kubernetes/charts/shuffle/values.yaml +++ b/functions/kubernetes/charts/shuffle/values.yaml @@ -520,6 +520,12 @@ backend: ## debug: false + ## @param backend.autoGOMEMLIMIT Automatically set GOMEMLIMIT environment variable to 90% of the container memory limit. + ## This helps prevent Go runtime memory limits from exceeding container limits, which can cause OOM kills. + ## Only effective when backend.resources.limits.memory is set or a resourcesPreset is used that defines memory limits. + ## + autoGOMEMLIMIT: true + ## @param backend.cleanupSchedule The interval in seconds at which the cleanup job runs ## cleanupSchedule: 300 @@ -1337,6 +1343,13 @@ orborus: ## debug: false + ## @param orborus.autoGOMEMLIMIT Automatically set GOMEMLIMIT environment variable to 90% of the container memory limit. + ## This helps prevent Go runtime memory limits from exceeding container limits, which can cause OOM kills. + ## Only effective when orborus.resources.limits.memory is set or a resourcesPreset is used that defines memory limits. + ## Only effective when orborus is deployed via helm. + ## + autoGOMEMLIMIT: true + ## @param orborus.executionConcurrency The maximum amount of concurrent workflow executions per worker ## executionConcurrency: 25 @@ -1751,6 +1764,13 @@ worker: ## debug: false + ## @param worker.autoGOMEMLIMIT Automatically set GOMEMLIMIT environment variable to 90% of the container memory limit. + ## This helps prevent Go runtime memory limits from exceeding container limits, which can cause OOM kills. + ## Only effective when worker.resources.limits.memory is set or a resourcesPreset is used that defines memory limits. + ## Only effective when worker is deployed via helm (see worker.enableHelmDeployment). + ## + autoGOMEMLIMIT: true + ## @param worker.manageAppDeployments Whether apps are deployed and managed by worker. When disabled, every used app is expected to to be already deployed (see apps.enabled). ## This effectively removes required RBAC permissions from the shuffle-worker service account to create deployments and services. ## The worker might still attempt to create kubernetes objects, resulting in an error. There is currently no way to tell the worker, that it should not manage k8s resources.