start working on helm-deployed workers and apps

Signed-off-by: Pascal Sthamer <pascal+github@sthamer.xyz>
2025-12-01 10:52:18 +01:00
parent 2c9c453094
commit 5923c68657
8 changed files with 280 additions and 15 deletions
@@ -60,19 +60,31 @@ app.kubernetes.io/component: orborus
 {{- end -}}

 {{/*
-Return the common labels for worker components
+Return the common labels for worker components deployed via helm
 */}}
 {{- define "shuffle.worker.labels" -}}
-{{- include "common.labels.standard" . }}
-app.kubernetes.io/component: worker
+app.kubernetes.io/name: shuffle-worker
+helm.sh/chart: {{ include "common.names.chart" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+app.kubernetes.io/part-of: shuffle
+{{- with .Chart.AppVersion }}
+app.kubernetes.io/version: {{ . | replace "+" "_" | quote }}
+{{- end -}}
 {{- end -}}

 {{/*
-Return the common labels for app components
+Return the common labels for app components deployed via helm
 */}}
 {{- define "shuffle.app.labels" -}}
-{{- include "common.labels.standard" . }}
-app.kubernetes.io/component: app
+app.kubernetes.io/name: shuffle-app
+helm.sh/chart: {{ include "common.names.chart" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+app.kubernetes.io/part-of: shuffle
+{{- with .Chart.AppVersion }}
+app.kubernetes.io/version: {{ . | replace "+" "_" | quote }}
+{{- end -}}
 {{- end -}}

 {{/*
@@ -100,16 +112,18 @@ app.kubernetes.io/component: orborus
 {{- end -}}

 {{/*
-Return the match labels for worker components
-NOTE: This does not match the labels from shuffle.worker.labels, but the labels set by the orborus GoLang app.
+Return the match labels for worker components.
+These must match the labels of helm-deployed worker components (shuffle.worker.labels),
+as well as orborus-deployed worker components (deployk8sworker).
 */}}
 {{- define "shuffle.worker.matchLabels" -}}
 app.kubernetes.io/name: shuffle-worker
 {{- end -}}

 {{/*
-Return the match labels for app components
-NOTE: This does not match the labels from shuffle.worker.labels, but the labels set by the orborus GoLang app.
+Return the match labels for app components.
+These must match the labels of helm-deployed app components (shuffle.app.labels),
+as well as worker-deployed app deployments (deployK8sApp).
 */}}
 {{- define "shuffle.app.matchLabels" -}}
 app.kubernetes.io/name: shuffle-app
@@ -375,3 +389,7 @@ imagePullSecrets:
 {{- end }}
 {{- end }}
 {{- end -}}
+
+{{- define "shuffle.backend.baseUrl -}}
+http://{{ include "shuffle.backend.name" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.backend.containerPorts.http }}
+{{- end - }}
@@ -11,10 +11,10 @@ data:
  ENVIRONMENT_NAME: "{{ .Values.shuffle.org }}"
  ORG_ID: "{{ .Values.shuffle.org }}"
  TZ: "{{ .Values.shuffle.timezone }}"
-  BASE_URL: "http://{{ include "shuffle.backend.name" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.backend.containerPorts.http }}"
-  KUBERNETES_NAMESPACE: "{{ .Release.Namespace }}"
+  BASE_URL: {{ import "shuffle.backend.baseUrl" . | quote }}
  REGISTRY_URL: "{{ .Values.shuffle.appRegistry }}"

+  {{- if .Values.orborus.manageWorkerDeployments }}
  # Shuffle worker configuration
  SHUFFLE_WORKER_IMAGE: {{ include "shuffle.worker.image" . | quote }}
  SHUFFLE_WORKER_SERVICE_ACCOUNT_NAME: {{ include "shuffle.worker.serviceAccount.name" . | quote  }}
@@ -46,6 +46,7 @@ data:
  SHUFFLE_WORKER_EPHEMERAL_STORAGE_LIMIT: {{ (index $workerResources.limits "ephemeral-storage") | quote }}
  {{- end }}

+  {{- if .Values.worker.manageAppDeployments }}
  # Shuffle app configuration
  SHUFFLE_APP_EXPOSED_PORT: {{ .Values.app.exposedContainerPort | quote }}
  SHUFFLE_APP_SERVICE_ACCOUNT_NAME: {{ include "shuffle.app.serviceAccount.name" . | quote }}
@@ -76,3 +77,5 @@ data:
  {{- if and $appResources.limits (index $appResources.limits "ephemeral-storage") }}
  SHUFFLE_APP_EPHEMERAL_STORAGE_LIMIT: {{ (index $appResources.limits "ephemeral-storage") | quote }}
  {{- end }}
+  {{- end }}
+  {{- end }}
@@ -9,6 +9,7 @@ metadata:
  annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
  {{- end }}
 rules:
+  {{- if .Values.orborus.manageWorkerDeployments }}
  - verbs:
      - list
      - create
@@ -26,4 +27,19 @@ rules:
      - apps
    resources:
      - deployments
+  {{- else }}
+  - verbs:
+      - list
+    apiGroups:
+      - ''
+    resources:
+      - pods
+      - services
+  - verbs:
+      - list
+    apiGroups:
+      - apps
+    resources:
+      - deployments
+  {{- end }}
 {{- end }}
@@ -0,0 +1,17 @@
+TODO:
+  - Deploy each app (configurable in values) here.
+  - Have defaults for all apps
+  - Allow overriding per app (serviceAccount, securityContext, volumes , env, resources, replicas, ...)
+
+{{- range $index, $app := .Values.app.deployViaHelm }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+spec:
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+spec:
+{{- end }}
@@ -0,0 +1,161 @@
+{{- if .Values.worker.enableHelmDeployment }}
+apiVersion: {{ include "common.capabilities.deployment.apiVersion" . }}
+kind: Deployment
+metadata:
+  name: {{ template "shuffle.worker.name" . }}
+  namespace: {{ include "common.names.namespace" . | quote }}
+  labels: {{- include "shuffle.worker.labels" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
+  {{- if or .Values.worker.deploymentAnnotations .Values.commonAnnotations }}
+  {{- $annotations := include "common.tplvalues.merge" (dict "values" (list .Values.worker.deploymentAnnotations .Values.commonAnnotations) "context" .) }}
+  annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $ ) | nindent 4 }}
+  {{- end }}
+spec:
+  {{- if not .Values.worker.autoscaling.hpa.enabled }}
+  replicas: {{ .Values.worker.replicaCount }}
+  {{- end }}
+  {{- if .Values.worker.updateStrategy }}
+  strategy: {{- toYaml .Values.worker.updateStrategy | nindent 4 }}
+  {{- end }}
+  {{- $podLabels := include "common.tplvalues.merge" (dict "values" (list .Values.worker.podLabels .Values.commonLabels) "context" .) }}
+  selector:
+    matchLabels: {{- include "shuffle.worker.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 6 }}
+  template:
+    metadata:
+      {{- if .Values.worker.podAnnotations }}
+      annotations: {{- include "common.tplvalues.render" (dict "value" .Values.worker.podAnnotations "context" $) | nindent 8 }}
+      {{- end }}
+      labels: {{- include "shuffle.worker.labels" ( dict "customLabels" $podLabels "context" $ ) | nindent 8 }}
+    spec:
+      {{- include "shuffle.worker.imagePullSecrets" . | nindent 6 }}
+      serviceAccountName: {{ template "shuffle.worker.serviceAccount.name" . }}
+      automountServiceAccountToken: {{ .Values.worker.automountServiceAccountToken }}
+      {{- if .Values.worker.hostAliases }}
+      hostAliases: {{- include "common.tplvalues.render" (dict "value" .Values.worker.hostAliases "context" $) | nindent 8 }}
+      {{- end }}
+      {{- if .Values.worker.affinity }}
+      affinity: {{- include "common.tplvalues.render" ( dict "value" .Values.worker.affinity "context" $) | nindent 8 }}
+      {{- else }}
+      affinity:
+        podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.worker.podAffinityPreset "component" "orborus" "customLabels" $podLabels "context" $) | nindent 10 }}
+        podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.worker.podAntiAffinityPreset "component" "orborus" "customLabels" $podLabels "context" $) | nindent 10 }}
+        nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.worker.nodeAffinityPreset.type "key" .Values.worker.nodeAffinityPreset.key "values" .Values.worker.nodeAffinityPreset.values) | nindent 10 }}
+      {{- end }}
+      {{- if .Values.worker.nodeSelector }}
+      nodeSelector: {{- include "common.tplvalues.render" ( dict "value" .Values.worker.nodeSelector "context" $) | nindent 8 }}
+      {{- end }}
+      {{- if .Values.worker.tolerations }}
+      tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.worker.tolerations "context" .) | nindent 8 }}
+      {{- end }}
+      {{- if .Values.worker.priorityClassName }}
+      priorityClassName: {{ .Values.worker.priorityClassName | quote }}
+      {{- end }}
+      {{- if .Values.worker.schedulerName }}
+      schedulerName: {{ .Values.worker.schedulerName | quote }}
+      {{- end }}
+      {{- if .Values.worker.topologySpreadConstraints }}
+      topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.worker.topologySpreadConstraints "context" .) | nindent 8 }}
+      {{- end }}
+      {{- if .Values.worker.podSecurityContext.enabled }}
+      securityContext: {{- omit .Values.worker.podSecurityContext "enabled" | toYaml | nindent 8 }}
+      {{- end }}
+      {{- if .Values.worker.terminationGracePeriodSeconds }}
+      terminationGracePeriodSeconds: {{ .Values.worker.terminationGracePeriodSeconds }}
+      {{- end }}
+      initContainers:
+        {{- if .Values.worker.initContainers }}
+          {{- include "common.tplvalues.render" (dict "value" .Values.worker.initContainers "context" $) | nindent 8 }}
+        {{- end }}
+      containers:
+        - name: orborus
+          image: {{ template "shuffle.worker.image" . }}
+          imagePullPolicy: {{ .Values.worker.image.pullPolicy }}
+          {{- if .Values.worker.containerSecurityContext.enabled }}
+          securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.worker.containerSecurityContext "context" $) | nindent 12 }}
+          {{- end }}
+          {{- if .Values.diagnosticMode.enabled }}
+          command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }}
+          {{- else if .Values.worker.command }}
+          command: {{- include "common.tplvalues.render" (dict "value" .Values.worker.command "context" $) | nindent 12 }}
+          {{- end }}
+          {{- if .Values.diagnosticMode.enabled }}
+          args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 12 }}
+          {{- else if .Values.worker.args }}
+          args: {{- include "common.tplvalues.render" (dict "value" .Values.worker.args "context" $) | nindent 12 }}
+          {{- end }}
+          env:
+            - name: IS_KUBERNETES
+              value: "true"
+            - name: KUBERNETES_NAMESPACE
+              value: "{{ .Release.Namespace }}"
+            - name: SHUFFLE_LOGS_DISABLED
+              value: "false" # TODO: Allow to configure. Share with orborus / apps?
+            - name: BASE_URL
+              value: {{ import "shuffle.backend.baseUrl" . | quote }}
+            - name: REGISTRY_URL # TODO: REGISTRY_URL vs. SHUFFLE_BASE_IMAGE_REGISTRY
+              value: {{ .Values.shuffle.appRegistry }}
+            - name: SHUFFLE_BASE_IMAGE_REGISTRY
+              value: {{ .Values.shuffle.appRegistry }}
+            - name: SHUFFLE_BASE_IMAGE_NAME
+              value: shuffle # TODO: Allow to configure? Use same as for orborus (shared value)
+            - name: SHUFFLE_APP_EXPOSED_PORT
+              value: {{ .Values.app.exposedContainerPort | quote }}
+            - name: WORKER_HOSTNAME
+              value: {{ include "shuffle.worker.name" }}.{{ .Release.Namespace }}.svc.cluster.local
+            {{- if .Values.worker.extraEnvVars }}
+            {{- include "common.tplvalues.render" (dict "value" .Values.worker.extraEnvVars "context" $) | nindent 12 }}
+            {{- end }}
+          envFrom:
+            {{- if .Values.worker.extraEnvVarsCM }}
+            - configMapRef:
+                name: {{ include "common.tplvalues.render" (dict "value" .Values.worker.extraEnvVarsCM "context" $) }}
+            {{- end }}
+            {{- if .Values.worker.extraEnvVarsSecret }}
+            - secretRef:
+                name: {{ include "common.tplvalues.render" (dict "value" .Values.worker.extraEnvVarsSecret "context" $) }}
+            {{- end }}
+          {{- if .Values.worker.resources }}
+          resources: {{- toYaml .Values.worker.resources | nindent 12 }}
+          {{- else if ne .Values.worker.resourcesPreset "none" }}
+          resources: {{- include "common.resources.preset" (dict "type" .Values.worker.resourcesPreset) | nindent 12 }}
+          {{- end }}
+          ports:
+            {{- if .Values.worker.extraContainerPorts }}
+            {{- include "common.tplvalues.render" (dict "value" .Values.worker.extraContainerPorts "context" $) | nindent 12 }}
+            {{- end }}
+          {{- if not .Values.diagnosticMode.enabled }}
+          {{- if .Values.worker.customLivenessProbe }}
+          livenessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.worker.customLivenessProbe "context" $) | nindent 12 }}
+          {{- else if .Values.worker.livenessProbe.enabled }}
+          livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.worker.livenessProbe "enabled") "context" $) | nindent 12 }}
+          {{- end }}
+          {{- if .Values.worker.customReadinessProbe }}
+          readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.worker.customReadinessProbe "context" $) | nindent 12 }}
+          {{- else if .Values.worker.readinessProbe.enabled }}
+          readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.worker.readinessProbe "enabled") "context" $) | nindent 12 }}
+          {{- end }}
+          {{- if .Values.worker.customStartupProbe }}
+          startupProbe: {{- include "common.tplvalues.render" (dict "value" .Values.worker.customStartupProbe "context" $) | nindent 12 }}
+          {{- else if .Values.worker.startupProbe.enabled }}
+          startupProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.worker.startupProbe "enabled") "context" $) | nindent 12 }}
+          {{- end }}
+          {{- end }}
+          {{- if .Values.worker.lifecycleHooks }}
+          lifecycle: {{- include "common.tplvalues.render" (dict "value" .Values.worker.lifecycleHooks "context" $) | nindent 12 }}
+          {{- end }}
+          volumeMounts:
+            - name: empty-dir
+              mountPath: /tmp
+              subPath: tmp-dir
+          {{- if .Values.worker.extraVolumeMounts }}
+          {{- include "common.tplvalues.render" (dict "value" .Values.worker.extraVolumeMounts "context" $) | nindent 12 }}
+          {{- end }}
+        {{- if .Values.worker.sidecars }}
+        {{- include "common.tplvalues.render" ( dict "value" .Values.worker.sidecars "context" $) | nindent 8 }}
+        {{- end }}
+      volumes:
+        - name: empty-dir
+          emptyDir: {}
+        {{- if .Values.worker.extraVolumes }}
+        {{- include "common.tplvalues.render" (dict "value" .Values.worker.extraVolumes "context" $) | nindent 8 }}
+        {{- end }}
+{{- end }}
@@ -9,6 +9,7 @@ metadata:
  annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
  {{- end }}
 rules:
+  {{- if .Values.worker.manageAppDeployments }}
  - apiGroups: [""]
    resources: ["pods"]
    verbs: ["list", "delete"]
@@ -18,4 +19,11 @@ rules:
  - apiGroups: ["apps"]
    resources: ["deployments"]
    verbs: ["list", "create"]
+  {{- else }}
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["list"]
+  - apiGroups: ["apps"]
+    resources: ["deployments"]
+    verbs: ["list"]
 {{- end }}