automatically set GOMEMLIMIT env variable

Signed-off-by: Pascal Sthamer <pascal+github@sthamer.xyz>

functions/kubernetes/charts/shuffle/templates/_helpers.tpl

@@ -4,3 +4,46 @@ Return the proper image name (for the init container volume-permissions image)
 {{- define "shuffle.volumePermissions.image" -}}
 {{- include "common.images.image" ( dict "imageRoot" .Values.volumePermissions.image "global" .Values.global "chart" .Chart ) -}}
 {{- end -}}
+
+{{/*
+Return a value for the GOMEMLIMIT env variable based on a given kubernetes resource memory limit.
+Usage:
+{{ include "shuffle.k8sMemoryLimitToGOMEMLIMIT" (dict "k8sMemoryLimit" $.Values.resources.limits.memory "context" $) }}
+*/}}
+{{- define "shuffle.k8sMemoryLimitToGOMEMLIMIT" -}}
+{{- $memoryLimit := .k8sMemoryLimit | default "" -}}
+{{- $result := "" -}}
+{{- if and $memoryLimit (gt (len $memoryLimit) 0) -}}
+{{-   $bytes := 0 -}}
+{{-   if hasSuffix "Ki" $memoryLimit -}}
+{{-     $bytes = mul ($memoryLimit | trimSuffix "Ki" | int) 1024 -}}
+{{-   else if hasSuffix "Mi" $memoryLimit -}}
+{{-     $bytes = mul ($memoryLimit | trimSuffix "Mi" | int) 1048576 -}}
+{{-   else if hasSuffix "Gi" $memoryLimit -}}
+{{-     $bytes = mul ($memoryLimit | trimSuffix "Gi" | int) 1073741824 -}}
+{{-   else if hasSuffix "Ti" $memoryLimit -}}
+{{-     $bytes = mul ($memoryLimit | trimSuffix "Ti" | int) 1099511627776 -}}
+{{-   else if hasSuffix "Pi" $memoryLimit -}}
+{{-     $bytes = mul ($memoryLimit | trimSuffix "Pi" | int) 1125899906842624 -}}
+{{-   else if hasSuffix "Ei" $memoryLimit -}}
+{{-     $bytes = mul ($memoryLimit | trimSuffix "Ei" | int) 1152921504606846976 -}}
+{{-   else if hasSuffix "K" $memoryLimit -}}
+{{-     $bytes = mul ($memoryLimit | trimSuffix "K" | int) 1000 -}}
+{{-   else if hasSuffix "M" $memoryLimit -}}
+{{-     $bytes = mul ($memoryLimit | trimSuffix "M" | int) 1000000 -}}
+{{-   else if hasSuffix "G" $memoryLimit -}}
+{{-     $bytes = mul ($memoryLimit | trimSuffix "G" | int) 1000000000 -}}
+{{-   else if hasSuffix "T" $memoryLimit -}}
+{{-     $bytes = mul ($memoryLimit | trimSuffix "T" | int) 1000000000000 -}}
+{{-   else if hasSuffix "P" $memoryLimit -}}
+{{-     $bytes = mul ($memoryLimit | trimSuffix "P" | int) 1000000000000000 -}}
+{{-   else if hasSuffix "E" $memoryLimit -}}
+{{-     $bytes = mul ($memoryLimit | trimSuffix "E" | int) 1000000000000000000 -}}
+{{-   else -}}
+{{-     $bytes = $memoryLimit | int -}}
+{{-   end -}}
+{{-   $gomaxmem := div (mul $bytes 9) 10 -}}
+{{-   $result = printf "%d" $gomaxmem -}}
+{{- end -}}
+{{- $result -}}
+{{- end -}}

functions/kubernetes/charts/shuffle/templates/backend/_helpers.tpl

@@ -116,4 +116,10 @@ REGISTRY_URL: "{{ .Values.shuffle.appRegistry }}" # Used by app builder
 {{- if .Values.backend.debug }}
 DEBUG: "true"
 {{- end }}
+{{- if .Values.backend.autoGOMEMLIMIT }}
+{{- $backendResources := (.Values.backend.resources | default (include "common.resources.preset" (dict "type" .Values.backend.resourcesPreset) | fromYaml)) -}}
+{{- if and $backendResources $backendResources.limits $backendResources.limits.memory }}
+GOMEMLIMIT: {{ include "shuffle.k8sMemoryLimitToGOMEMLIMIT" (dict "k8sMemoryLimit" $backendResources.limits.memory) | quote }}
+{{- end }}
+{{- end }}
 {{- end -}}

functions/kubernetes/charts/shuffle/templates/orborus/_helpers.tpl

@@ -92,6 +92,12 @@ SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY: {{ .Values.orborus.executionConcurrency |
 {{- if .Values.orborus.debug }}
 DEBUG: "true"
 {{- end }}
+{{- if .Values.orborus.autoGOMEMLIMIT }}
+{{- $orborusResources := (.Values.orborus.resources | default (include "common.resources.preset" (dict "type" .Values.orborus.resourcesPreset) | fromYaml)) -}}
+{{- if and $orborusResources $orborusResources.limits $orborusResources.limits.memory }}
+GOMEMLIMIT: {{ include "shuffle.k8sMemoryLimitToGOMEMLIMIT" (dict "k8sMemoryLimit" $orborusResources.limits.memory) | quote }}
+{{- end }}
+{{- end }}
 
 {{- if .Values.orborus.manageWorkerDeployments }}
 # Shuffle worker configuration

functions/kubernetes/charts/shuffle/templates/shuffle-app/_helpers.tpl

@@ -123,12 +123,13 @@ Return the environment variables of shuffle apps in the format
 KEY: VALUE
 Usage:
 {{- include "shuffle.appInstance.env" (dict "app" $appValues "context" $) -}}
+
+WARNING: Do NOT add environment variables here that would conflict with shuffle.worker.env or shuffle.orborus.env.
+Worker also sets all env variables that are defined here, because they will be passed down to apps when worker.manageAppDeployments is set.
+Instead, add them directly to the deployment template (shuffle-apps.yaml).
 */}}
 {{- define "shuffle.appInstance.env" -}}
 SHUFFLE_APP_SDK_TIMEOUT: {{ .app.sdkTimeout | quote }}
 SHUFFLE_APP_EXPOSED_PORT: {{ .app.exposedContainerPort | quote }}
 SHUFFLE_LOGS_DISABLED: {{ .app.disableLogs | quote }}
-{{- if .app.debug }}
-DEBUG: "true"
-{{- end }}
 {{- end -}}

functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-apps.yaml

@@ -168,6 +168,10 @@ spec:
               value: {{ include "shuffle.backend.baseUrl" $ | quote }}
             - name: SHUFFLE_SWARM_CONFIG
               value: run # Shuffle Worker requires this to be set even when using K8s instead of swarm
+            {{- if $appValues.debug }}
+            - name: DEBUG
+              value: "true"
+            {{- end }}
             {{- $env := include "shuffle.appInstance.env" (dict "app" $appValues "context" $) | fromYaml }}
             {{- range $key, $val := $env }}
             - name: {{ $key | quote }}

functions/kubernetes/charts/shuffle/templates/shuffle-worker/_helpers.tpl

@@ -131,17 +131,15 @@ http://shuffle-workers.{{ .Release.Namespace }}.svc.cluster.local
 {{/*
 Return the environment variables of shuffle-worker in the format
 KEY: VALUE
+
+WARNING: Do NOT add environment variables here that would conflict with shuffle.orborus.env.
+Orborus also sets all env variables that are defined here, because they will be passed down to worker when orborus.manageWorkerDeployments is set.
+Instead, add them directly to the deployment template (shuffle-worker-dpl.yaml).
 */}}
 {{- define "shuffle.workerInstance.env" -}}
-IS_KUBERNETES: "true"
-KUBERNETES_NAMESPACE: "{{ .Release.Namespace }}"
 SHUFFLE_SWARM_CONFIG: "run" # Shuffle Worker requires this to be set even when using K8s instead of swarm
-BASE_URL: {{ include "shuffle.backend.baseUrl" . | quote }}
 SHUFFLE_APP_EXPOSED_PORT: {{ .Values.app.exposedContainerPort | quote }}
 WORKER_HOSTNAME: {{ include "shuffle.worker.hostname" . }}
-{{- if .Values.worker.debug }}
-DEBUG: "true"
-{{- end }}
 
 {{- if .Values.worker.manageAppDeployments }}
 # Shuffle app images

functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml

@@ -85,6 +85,23 @@ spec:
           env:
             - name: CLEANUP
               value: "false" # Do not remove resources when restarting worker
+            - name: IS_KUBERNETES
+              value: "true"
+            - name: KUBERNETES_NAMESPACE
+              value: {{ .Release.Namespace | quote }}
+            - name: BASE_URL
+              value: {{ include "shuffle.backend.baseUrl" . | quote }}
+            {{- if .Values.worker.debug }}
+            - name: DEBUG
+              value: "true"
+            {{- end }}
+            {{- if .Values.worker.autoGOMEMLIMIT }}
+            {{- $workerResources := (.Values.worker.resources | default (include "common.resources.preset" (dict "type" .Values.worker.resourcesPreset) | fromYaml)) -}}
+            {{- if and $workerResources $workerResources.limits $workerResources.limits.memory }}
+            - name: GOMEMLIMIT
+              value: {{ include "shuffle.k8sMemoryLimitToGOMEMLIMIT" (dict "k8sMemoryLimit" $workerResources.limits.memory) | quote }}
+            {{- end }}
+            {{- end }}
             {{- $env := include "shuffle.workerInstance.env" . | fromYaml }}
             {{- range $key, $val := $env }}
             - name: {{ $key | quote }}

functions/kubernetes/charts/shuffle/values.yaml

@@ -520,6 +520,12 @@ backend:
   ##
   debug: false
 
+  ## @param backend.autoGOMEMLIMIT Automatically set GOMEMLIMIT environment variable to 90% of the container memory limit.
+  ## This helps prevent Go runtime memory limits from exceeding container limits, which can cause OOM kills.
+  ## Only effective when backend.resources.limits.memory is set or a resourcesPreset is used that defines memory limits.
+  ##
+  autoGOMEMLIMIT: true
+
   ## @param backend.cleanupSchedule The interval in seconds at which the cleanup job runs
   ##
   cleanupSchedule: 300
@@ -1337,6 +1343,13 @@ orborus:
   ##
   debug: false
 
+  ## @param orborus.autoGOMEMLIMIT Automatically set GOMEMLIMIT environment variable to 90% of the container memory limit.
+  ## This helps prevent Go runtime memory limits from exceeding container limits, which can cause OOM kills.
+  ## Only effective when orborus.resources.limits.memory is set or a resourcesPreset is used that defines memory limits.
+  ## Only effective when orborus is deployed via helm.
+  ##
+  autoGOMEMLIMIT: true
+
   ## @param orborus.executionConcurrency The maximum amount of concurrent workflow executions per worker
   ##
   executionConcurrency: 25
@@ -1751,6 +1764,13 @@ worker:
   ##
   debug: false
 
+  ## @param worker.autoGOMEMLIMIT Automatically set GOMEMLIMIT environment variable to 90% of the container memory limit.
+  ## This helps prevent Go runtime memory limits from exceeding container limits, which can cause OOM kills.
+  ## Only effective when worker.resources.limits.memory is set or a resourcesPreset is used that defines memory limits.
+  ## Only effective when worker is deployed via helm (see worker.enableHelmDeployment).
+  ##
+  autoGOMEMLIMIT: true
+
   ## @param worker.manageAppDeployments Whether apps are deployed and managed by worker. When disabled, every used app is expected to to be already deployed (see apps.enabled).
   ## This effectively removes required RBAC permissions from the shuffle-worker service account to create deployments and services.
   ## The worker might still attempt to create kubernetes objects, resulting in an error. There is currently no way to tell the worker, that it should not manage k8s resources.