start working on helm-deployed workers and apps

Signed-off-by: Pascal Sthamer <pascal+github@sthamer.xyz>

functions/kubernetes/charts/shuffle/templates/_helpers.tpl

@@ -60,19 +60,31 @@ app.kubernetes.io/component: orborus
 {{- end -}}
 
 {{/*
-Return the common labels for worker components
+Return the common labels for worker components deployed via helm
 */}}
 {{- define "shuffle.worker.labels" -}}
-{{- include "common.labels.standard" . }}
-app.kubernetes.io/component: worker
+app.kubernetes.io/name: shuffle-worker
+helm.sh/chart: {{ include "common.names.chart" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+app.kubernetes.io/part-of: shuffle
+{{- with .Chart.AppVersion }}
+app.kubernetes.io/version: {{ . | replace "+" "_" | quote }}
+{{- end -}}
 {{- end -}}
 
 {{/*
-Return the common labels for app components
+Return the common labels for app components deployed via helm
 */}}
 {{- define "shuffle.app.labels" -}}
-{{- include "common.labels.standard" . }}
-app.kubernetes.io/component: app
+app.kubernetes.io/name: shuffle-app
+helm.sh/chart: {{ include "common.names.chart" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+app.kubernetes.io/part-of: shuffle
+{{- with .Chart.AppVersion }}
+app.kubernetes.io/version: {{ . | replace "+" "_" | quote }}
+{{- end -}}
 {{- end -}}
 
 {{/*
@@ -100,16 +112,18 @@ app.kubernetes.io/component: orborus
 {{- end -}}
 
 {{/*
-Return the match labels for worker components
-NOTE: This does not match the labels from shuffle.worker.labels, but the labels set by the orborus GoLang app.
+Return the match labels for worker components.
+These must match the labels of helm-deployed worker components (shuffle.worker.labels),
+as well as orborus-deployed worker components (deployk8sworker).
 */}}
 {{- define "shuffle.worker.matchLabels" -}}
 app.kubernetes.io/name: shuffle-worker
 {{- end -}}
 
 {{/*
-Return the match labels for app components
-NOTE: This does not match the labels from shuffle.worker.labels, but the labels set by the orborus GoLang app.
+Return the match labels for app components.
+These must match the labels of helm-deployed app components (shuffle.app.labels),
+as well as worker-deployed app deployments (deployK8sApp).
 */}}
 {{- define "shuffle.app.matchLabels" -}}
 app.kubernetes.io/name: shuffle-app
@@ -375,3 +389,7 @@ imagePullSecrets:
 {{- end }}
 {{- end }}
 {{- end -}}
+
+{{- define "shuffle.backend.baseUrl -}}
+http://{{ include "shuffle.backend.name" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.backend.containerPorts.http }}
+{{- end - }}

functions/kubernetes/charts/shuffle/templates/orborus/orborus-cm-env.yaml

@@ -11,10 +11,10 @@ data:
   ENVIRONMENT_NAME: "{{ .Values.shuffle.org }}"
   ORG_ID: "{{ .Values.shuffle.org }}"
   TZ: "{{ .Values.shuffle.timezone }}"
-  BASE_URL: "http://{{ include "shuffle.backend.name" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.backend.containerPorts.http }}"
-  KUBERNETES_NAMESPACE: "{{ .Release.Namespace }}"
+  BASE_URL: {{ import "shuffle.backend.baseUrl" . | quote }}
   REGISTRY_URL: "{{ .Values.shuffle.appRegistry }}"
 
+  {{- if .Values.orborus.manageWorkerDeployments }}
   # Shuffle worker configuration
   SHUFFLE_WORKER_IMAGE: {{ include "shuffle.worker.image" . | quote }}
   SHUFFLE_WORKER_SERVICE_ACCOUNT_NAME: {{ include "shuffle.worker.serviceAccount.name" . | quote  }}
@@ -46,6 +46,7 @@ data:
   SHUFFLE_WORKER_EPHEMERAL_STORAGE_LIMIT: {{ (index $workerResources.limits "ephemeral-storage") | quote }}
   {{- end }}
 
+  {{- if .Values.worker.manageAppDeployments }}
   # Shuffle app configuration
   SHUFFLE_APP_EXPOSED_PORT: {{ .Values.app.exposedContainerPort | quote }}
   SHUFFLE_APP_SERVICE_ACCOUNT_NAME: {{ include "shuffle.app.serviceAccount.name" . | quote }}
@@ -76,3 +77,5 @@ data:
   {{- if and $appResources.limits (index $appResources.limits "ephemeral-storage") }}
   SHUFFLE_APP_EPHEMERAL_STORAGE_LIMIT: {{ (index $appResources.limits "ephemeral-storage") | quote }}
   {{- end }}
+  {{- end }}
+  {{- end }}

functions/kubernetes/charts/shuffle/templates/orborus/orborus-role.yaml

@@ -9,6 +9,7 @@ metadata:
   annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
   {{- end }}
 rules:
+  {{- if .Values.orborus.manageWorkerDeployments }}
   - verbs:
       - list
       - create
@@ -26,4 +27,19 @@ rules:
       - apps
     resources:
       - deployments
+  {{- else }}
+  - verbs:
+      - list
+    apiGroups:
+      - ''
+    resources:
+      - pods
+      - services
+  - verbs:
+      - list
+    apiGroups:
+      - apps
+    resources:
+      - deployments
+  {{- end }}
 {{- end }}

functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-deployments.yaml

@@ -0,0 +1,17 @@
+TODO:
+  - Deploy each app (configurable in values) here.
+  - Have defaults for all apps
+  - Allow overriding per app (serviceAccount, securityContext, volumes , env, resources, replicas, ...)
+
+{{- range $index, $app := .Values.app.deployViaHelm }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+spec:
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+spec:
+{{- end }}

functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml

@@ -0,0 +1,161 @@
+{{- if .Values.worker.enableHelmDeployment }}
+apiVersion: {{ include "common.capabilities.deployment.apiVersion" . }}
+kind: Deployment
+metadata:
+  name: {{ template "shuffle.worker.name" . }}
+  namespace: {{ include "common.names.namespace" . | quote }}
+  labels: {{- include "shuffle.worker.labels" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
+  {{- if or .Values.worker.deploymentAnnotations .Values.commonAnnotations }}
+  {{- $annotations := include "common.tplvalues.merge" (dict "values" (list .Values.worker.deploymentAnnotations .Values.commonAnnotations) "context" .) }}
+  annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $ ) | nindent 4 }}
+  {{- end }}
+spec:
+  {{- if not .Values.worker.autoscaling.hpa.enabled }}
+  replicas: {{ .Values.worker.replicaCount }}
+  {{- end }}
+  {{- if .Values.worker.updateStrategy }}
+  strategy: {{- toYaml .Values.worker.updateStrategy | nindent 4 }}
+  {{- end }}
+  {{- $podLabels := include "common.tplvalues.merge" (dict "values" (list .Values.worker.podLabels .Values.commonLabels) "context" .) }}
+  selector:
+    matchLabels: {{- include "shuffle.worker.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 6 }}
+  template:
+    metadata:
+      {{- if .Values.worker.podAnnotations }}
+      annotations: {{- include "common.tplvalues.render" (dict "value" .Values.worker.podAnnotations "context" $) | nindent 8 }}
+      {{- end }}
+      labels: {{- include "shuffle.worker.labels" ( dict "customLabels" $podLabels "context" $ ) | nindent 8 }}
+    spec:
+      {{- include "shuffle.worker.imagePullSecrets" . | nindent 6 }}
+      serviceAccountName: {{ template "shuffle.worker.serviceAccount.name" . }}
+      automountServiceAccountToken: {{ .Values.worker.automountServiceAccountToken }}
+      {{- if .Values.worker.hostAliases }}
+      hostAliases: {{- include "common.tplvalues.render" (dict "value" .Values.worker.hostAliases "context" $) | nindent 8 }}
+      {{- end }}
+      {{- if .Values.worker.affinity }}
+      affinity: {{- include "common.tplvalues.render" ( dict "value" .Values.worker.affinity "context" $) | nindent 8 }}
+      {{- else }}
+      affinity:
+        podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.worker.podAffinityPreset "component" "orborus" "customLabels" $podLabels "context" $) | nindent 10 }}
+        podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.worker.podAntiAffinityPreset "component" "orborus" "customLabels" $podLabels "context" $) | nindent 10 }}
+        nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.worker.nodeAffinityPreset.type "key" .Values.worker.nodeAffinityPreset.key "values" .Values.worker.nodeAffinityPreset.values) | nindent 10 }}
+      {{- end }}
+      {{- if .Values.worker.nodeSelector }}
+      nodeSelector: {{- include "common.tplvalues.render" ( dict "value" .Values.worker.nodeSelector "context" $) | nindent 8 }}
+      {{- end }}
+      {{- if .Values.worker.tolerations }}
+      tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.worker.tolerations "context" .) | nindent 8 }}
+      {{- end }}
+      {{- if .Values.worker.priorityClassName }}
+      priorityClassName: {{ .Values.worker.priorityClassName | quote }}
+      {{- end }}
+      {{- if .Values.worker.schedulerName }}
+      schedulerName: {{ .Values.worker.schedulerName | quote }}
+      {{- end }}
+      {{- if .Values.worker.topologySpreadConstraints }}
+      topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.worker.topologySpreadConstraints "context" .) | nindent 8 }}
+      {{- end }}
+      {{- if .Values.worker.podSecurityContext.enabled }}
+      securityContext: {{- omit .Values.worker.podSecurityContext "enabled" | toYaml | nindent 8 }}
+      {{- end }}
+      {{- if .Values.worker.terminationGracePeriodSeconds }}
+      terminationGracePeriodSeconds: {{ .Values.worker.terminationGracePeriodSeconds }}
+      {{- end }}
+      initContainers:
+        {{- if .Values.worker.initContainers }}
+          {{- include "common.tplvalues.render" (dict "value" .Values.worker.initContainers "context" $) | nindent 8 }}
+        {{- end }}
+      containers:
+        - name: orborus
+          image: {{ template "shuffle.worker.image" . }}
+          imagePullPolicy: {{ .Values.worker.image.pullPolicy }}
+          {{- if .Values.worker.containerSecurityContext.enabled }}
+          securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.worker.containerSecurityContext "context" $) | nindent 12 }}
+          {{- end }}
+          {{- if .Values.diagnosticMode.enabled }}
+          command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }}
+          {{- else if .Values.worker.command }}
+          command: {{- include "common.tplvalues.render" (dict "value" .Values.worker.command "context" $) | nindent 12 }}
+          {{- end }}
+          {{- if .Values.diagnosticMode.enabled }}
+          args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 12 }}
+          {{- else if .Values.worker.args }}
+          args: {{- include "common.tplvalues.render" (dict "value" .Values.worker.args "context" $) | nindent 12 }}
+          {{- end }}
+          env:
+            - name: IS_KUBERNETES
+              value: "true"
+            - name: KUBERNETES_NAMESPACE
+              value: "{{ .Release.Namespace }}"
+            - name: SHUFFLE_LOGS_DISABLED
+              value: "false" # TODO: Allow to configure. Share with orborus / apps?
+            - name: BASE_URL
+              value: {{ import "shuffle.backend.baseUrl" . | quote }}
+            - name: REGISTRY_URL # TODO: REGISTRY_URL vs. SHUFFLE_BASE_IMAGE_REGISTRY
+              value: {{ .Values.shuffle.appRegistry }}
+            - name: SHUFFLE_BASE_IMAGE_REGISTRY
+              value: {{ .Values.shuffle.appRegistry }}
+            - name: SHUFFLE_BASE_IMAGE_NAME
+              value: shuffle # TODO: Allow to configure? Use same as for orborus (shared value)
+            - name: SHUFFLE_APP_EXPOSED_PORT
+              value: {{ .Values.app.exposedContainerPort | quote }}
+            - name: WORKER_HOSTNAME
+              value: {{ include "shuffle.worker.name" }}.{{ .Release.Namespace }}.svc.cluster.local
+            {{- if .Values.worker.extraEnvVars }}
+            {{- include "common.tplvalues.render" (dict "value" .Values.worker.extraEnvVars "context" $) | nindent 12 }}
+            {{- end }}
+          envFrom:
+            {{- if .Values.worker.extraEnvVarsCM }}
+            - configMapRef:
+                name: {{ include "common.tplvalues.render" (dict "value" .Values.worker.extraEnvVarsCM "context" $) }}
+            {{- end }}
+            {{- if .Values.worker.extraEnvVarsSecret }}
+            - secretRef:
+                name: {{ include "common.tplvalues.render" (dict "value" .Values.worker.extraEnvVarsSecret "context" $) }}
+            {{- end }}
+          {{- if .Values.worker.resources }}
+          resources: {{- toYaml .Values.worker.resources | nindent 12 }}
+          {{- else if ne .Values.worker.resourcesPreset "none" }}
+          resources: {{- include "common.resources.preset" (dict "type" .Values.worker.resourcesPreset) | nindent 12 }}
+          {{- end }}
+          ports:
+            {{- if .Values.worker.extraContainerPorts }}
+            {{- include "common.tplvalues.render" (dict "value" .Values.worker.extraContainerPorts "context" $) | nindent 12 }}
+            {{- end }}
+          {{- if not .Values.diagnosticMode.enabled }}
+          {{- if .Values.worker.customLivenessProbe }}
+          livenessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.worker.customLivenessProbe "context" $) | nindent 12 }}
+          {{- else if .Values.worker.livenessProbe.enabled }}
+          livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.worker.livenessProbe "enabled") "context" $) | nindent 12 }}
+          {{- end }}
+          {{- if .Values.worker.customReadinessProbe }}
+          readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.worker.customReadinessProbe "context" $) | nindent 12 }}
+          {{- else if .Values.worker.readinessProbe.enabled }}
+          readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.worker.readinessProbe "enabled") "context" $) | nindent 12 }}
+          {{- end }}
+          {{- if .Values.worker.customStartupProbe }}
+          startupProbe: {{- include "common.tplvalues.render" (dict "value" .Values.worker.customStartupProbe "context" $) | nindent 12 }}
+          {{- else if .Values.worker.startupProbe.enabled }}
+          startupProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.worker.startupProbe "enabled") "context" $) | nindent 12 }}
+          {{- end }}
+          {{- end }}
+          {{- if .Values.worker.lifecycleHooks }}
+          lifecycle: {{- include "common.tplvalues.render" (dict "value" .Values.worker.lifecycleHooks "context" $) | nindent 12 }}
+          {{- end }}
+          volumeMounts:
+            - name: empty-dir
+              mountPath: /tmp
+              subPath: tmp-dir
+          {{- if .Values.worker.extraVolumeMounts }}
+          {{- include "common.tplvalues.render" (dict "value" .Values.worker.extraVolumeMounts "context" $) | nindent 12 }}
+          {{- end }}
+        {{- if .Values.worker.sidecars }}
+        {{- include "common.tplvalues.render" ( dict "value" .Values.worker.sidecars "context" $) | nindent 8 }}
+        {{- end }}
+      volumes:
+        - name: empty-dir
+          emptyDir: {}
+        {{- if .Values.worker.extraVolumes }}
+        {{- include "common.tplvalues.render" (dict "value" .Values.worker.extraVolumes "context" $) | nindent 8 }}
+        {{- end }}
+{{- end }}

functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-role.yaml

@@ -9,6 +9,7 @@ metadata:
   annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
   {{- end }}
 rules:
+  {{- if .Values.worker.manageAppDeployments }}
   - apiGroups: [""]
     resources: ["pods"]
     verbs: ["list", "delete"]
@@ -18,4 +19,11 @@ rules:
   - apiGroups: ["apps"]
     resources: ["deployments"]
     verbs: ["list", "create"]
+  {{- else }}
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["list"]
+  - apiGroups: ["apps"]
+    resources: ["deployments"]
+    verbs: ["list"]
 {{- end }}

functions/kubernetes/charts/shuffle/values.yaml

@@ -84,7 +84,7 @@ shuffle:
 
   ## @param shuffle.appRegistry The registry from / to which shuffle apps are pulled / pushed
   ##
-  appRegistry: ""
+  appRegistry: "" # TODO: Default?
 
   ## @param shuffle.timezone The timezone used by Shuffle
   ##
@@ -1209,7 +1209,7 @@ orborus:
     - name: KUBERNETES_NAMESPACE
       value: "shuffle"
     - name: SHUFFLE_BASE_IMAGE_NAME
-      value: "frikky/shuffle"
+      value: "frikky/shuffle" # TODO: Allow to configure, share with worker
   ## @param orborus.extraEnvVarsCM Name of existing ConfigMap containing extra env vars for orborus containers
   ##
   extraEnvVarsCM: ""
@@ -1346,10 +1346,20 @@ orborus:
     ##           istio: pilot
     ##
     extraEgress: []
+  
+    ## @param orborus.manageWorkerDeployments Whether workers are deployed and managed by orborus. When disabled, every worker is expected to be already deployed (see worker.enableHelmDeployment).
+    ## This effectively removes required RBAC permissions from the shuffle-orborus service account to create deployments and services.
+    ## Orborus might still attempt to create kubernetes objects, resulting in an error. There is currently no way to tell orborus, that it should not manage k8s resources.
+    ## You likely want to enable worker.enableHelmDeployment when enabling this.
+    manageWorkerDeployments: true
 
 ## @section worker Parameters
 ##
-worker:
+worker: # TODO: Add values that other helm deployed components have, mark helm-only params
+  ## @param worker.enableHelmDeployment Deploy worker via helm. By default, workers are deployed by Orborus.
+  ## You might want to disable orborus.manageWorkerDeployments when enabling this.
+  enableHelmDeployment: false
+
   ## worker image
   ## @param worker.image.registry worker image registry
   ## @param worker.image.repository worker image repository
@@ -1480,6 +1490,11 @@ worker:
     ##           istio: pilot
     ##
     extraEgress: []
+  
+  ## @param worker.manageAppDeployments Whether apps are deployed and managed by worker. When disabled, every used app is expected to to be already deployed (see app.deployViaHelm).
+  ## This effectively removes required RBAC permissions from the shuffle-worker service account to create deployments and services.
+  ## The worker might still attempt to create kubernetes objects, resulting in an error. There is currently no way to tell the worker, that it should not manage k8s resources.
+  manageAppDeployments: true
 
 ## @section app Parameters
 ##
@@ -1598,6 +1613,32 @@ app:
   ##
   exposedContainerPort: 80
 
+  ## @param app.deployViaHelm A list of apps that should be deployed using helm.
+  ## By default, workers create deployments and services for apps when they are first needed (or during startup for some selected apps).
+  ## Deploying apps via workers has some drawbacks, such as:
+  ## - A workflow fails when the app is not deployed when the workflow gets executed (see https://github.com/Shuffle/Shuffle/issues/1739)
+  ## - There is no way to set different service accounts, security contexts, resources, env variables, volume mounts, or replicas for different apps
+  ## - Worker needs elevated permissions in Kubernetes
+  ## Note that you can deploy some apps via helm, while keeping the flexibility of letting workers deploy apps if they are not already deployed.
+  ## If you deploy all needed apps via helm and dont want workers to create additional deployments, set worker.manageAppDeployments to false.
+  ##
+  ## Sample value:
+  ## - app: my-app # required
+  ##   version: 1.0.0 # required
+  ##   replicas: 1 # optional, default to 1
+  ##   image: "" # optional, default to appregistry/baseimage/appname:appversion
+  ##   imagePullSecrets: null # optional, defaults to global.imagePullSecrets
+  ##   resources: null # optional, defaults to app.resources (or app.resourcePreset)
+  ##   extraVolumes: [] # optional
+  ##   extraVolumeMounts: [] # optional
+  ##   extraEnv: [] # optional
+  ##   podSecurityContext: null # optional, defaults to app.podSecurityContext
+  ##   containerSecurityContext: null # optional, defaults to app.containerSecurityContext
+  ##   
+  ##   TODO: allow to override resources, serviceAccount, mounts, env, security Contexts. Fall back to app defaults as defined above.
+  ##
+  deployViaHelm: []
+
 ## @section Traffic Exposure Parameters
 ##
 

functions/onprem/orborus/orborus.go

@@ -1160,6 +1160,7 @@ func fixk8sRoles() {
 	}
 }
 
+// TODO: Check if deployment or service already exist by labels and only create if not already exists
 func deployK8sWorker(image string, identifier string, env []string) error {
 	env = append(env, fmt.Sprintf("IS_KUBERNETES=true"))
 	env = append(env, fmt.Sprintf("KUBERNETES_NAMESPACE=%s", os.Getenv("KUBERNETES_NAMESPACE")))