diff --git a/functions/kubernetes/charts/shuffle/templates/_helpers.tpl b/functions/kubernetes/charts/shuffle/templates/_helpers.tpl index dc93dfc2..67ca45cf 100644 --- a/functions/kubernetes/charts/shuffle/templates/_helpers.tpl +++ b/functions/kubernetes/charts/shuffle/templates/_helpers.tpl @@ -60,19 +60,31 @@ app.kubernetes.io/component: orborus {{- end -}} {{/* -Return the common labels for worker components +Return the common labels for worker components deployed via helm */}} {{- define "shuffle.worker.labels" -}} -{{- include "common.labels.standard" . }} -app.kubernetes.io/component: worker +app.kubernetes.io/name: shuffle-worker +helm.sh/chart: {{ include "common.names.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/part-of: shuffle +{{- with .Chart.AppVersion }} +app.kubernetes.io/version: {{ . | replace "+" "_" | quote }} +{{- end -}} {{- end -}} {{/* -Return the common labels for app components +Return the common labels for app components deployed via helm */}} {{- define "shuffle.app.labels" -}} -{{- include "common.labels.standard" . }} -app.kubernetes.io/component: app +app.kubernetes.io/name: shuffle-app +helm.sh/chart: {{ include "common.names.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/part-of: shuffle +{{- with .Chart.AppVersion }} +app.kubernetes.io/version: {{ . | replace "+" "_" | quote }} +{{- end -}} {{- end -}} {{/* @@ -100,16 +112,18 @@ app.kubernetes.io/component: orborus {{- end -}} {{/* -Return the match labels for worker components -NOTE: This does not match the labels from shuffle.worker.labels, but the labels set by the orborus GoLang app. +Return the match labels for worker components. +These must match the labels of helm-deployed worker components (shuffle.worker.labels), +as well as orborus-deployed worker components (deployk8sworker). */}} {{- define "shuffle.worker.matchLabels" -}} app.kubernetes.io/name: shuffle-worker {{- end -}} {{/* -Return the match labels for app components -NOTE: This does not match the labels from shuffle.worker.labels, but the labels set by the orborus GoLang app. +Return the match labels for app components. +These must match the labels of helm-deployed app components (shuffle.app.labels), +as well as worker-deployed app deployments (deployK8sApp). */}} {{- define "shuffle.app.matchLabels" -}} app.kubernetes.io/name: shuffle-app @@ -375,3 +389,7 @@ imagePullSecrets: {{- end }} {{- end }} {{- end -}} + +{{- define "shuffle.backend.baseUrl -}} +http://{{ include "shuffle.backend.name" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.backend.containerPorts.http }} +{{- end - }} \ No newline at end of file diff --git a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-cm-env.yaml b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-cm-env.yaml index 70209c21..28f2aa7e 100644 --- a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-cm-env.yaml +++ b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-cm-env.yaml @@ -11,10 +11,10 @@ data: ENVIRONMENT_NAME: "{{ .Values.shuffle.org }}" ORG_ID: "{{ .Values.shuffle.org }}" TZ: "{{ .Values.shuffle.timezone }}" - BASE_URL: "http://{{ include "shuffle.backend.name" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.backend.containerPorts.http }}" - KUBERNETES_NAMESPACE: "{{ .Release.Namespace }}" + BASE_URL: {{ import "shuffle.backend.baseUrl" . | quote }} REGISTRY_URL: "{{ .Values.shuffle.appRegistry }}" + {{- if .Values.orborus.manageWorkerDeployments }} # Shuffle worker configuration SHUFFLE_WORKER_IMAGE: {{ include "shuffle.worker.image" . | quote }} SHUFFLE_WORKER_SERVICE_ACCOUNT_NAME: {{ include "shuffle.worker.serviceAccount.name" . | quote }} @@ -46,6 +46,7 @@ data: SHUFFLE_WORKER_EPHEMERAL_STORAGE_LIMIT: {{ (index $workerResources.limits "ephemeral-storage") | quote }} {{- end }} + {{- if .Values.worker.manageAppDeployments }} # Shuffle app configuration SHUFFLE_APP_EXPOSED_PORT: {{ .Values.app.exposedContainerPort | quote }} SHUFFLE_APP_SERVICE_ACCOUNT_NAME: {{ include "shuffle.app.serviceAccount.name" . | quote }} @@ -76,3 +77,5 @@ data: {{- if and $appResources.limits (index $appResources.limits "ephemeral-storage") }} SHUFFLE_APP_EPHEMERAL_STORAGE_LIMIT: {{ (index $appResources.limits "ephemeral-storage") | quote }} {{- end }} + {{- end }} + {{- end }} diff --git a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-role.yaml b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-role.yaml index 90041d89..b3f0ca13 100644 --- a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-role.yaml +++ b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-role.yaml @@ -9,6 +9,7 @@ metadata: annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} {{- end }} rules: + {{- if .Values.orborus.manageWorkerDeployments }} - verbs: - list - create @@ -26,4 +27,19 @@ rules: - apps resources: - deployments + {{- else }} + - verbs: + - list + apiGroups: + - '' + resources: + - pods + - services + - verbs: + - list + apiGroups: + - apps + resources: + - deployments + {{- end }} {{- end }} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-deployments.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-deployments.yaml new file mode 100644 index 00000000..a1679e40 --- /dev/null +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-deployments.yaml @@ -0,0 +1,17 @@ +TODO: + - Deploy each app (configurable in values) here. + - Have defaults for all apps + - Allow overriding per app (serviceAccount, securityContext, volumes , env, resources, replicas, ...) + +{{- range $index, $app := .Values.app.deployViaHelm }} +--- +apiVersion: v1 +kind: Service +metadata: +spec: +--- +apiVersion: apps/v1 +kind: Deployment +metadata: +spec: +{{- end }} \ No newline at end of file diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml new file mode 100644 index 00000000..18641999 --- /dev/null +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-dpl.yaml @@ -0,0 +1,161 @@ +{{- if .Values.worker.enableHelmDeployment }} +apiVersion: {{ include "common.capabilities.deployment.apiVersion" . }} +kind: Deployment +metadata: + name: {{ template "shuffle.worker.name" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "shuffle.worker.labels" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if or .Values.worker.deploymentAnnotations .Values.commonAnnotations }} + {{- $annotations := include "common.tplvalues.merge" (dict "values" (list .Values.worker.deploymentAnnotations .Values.commonAnnotations) "context" .) }} + annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + {{- if not .Values.worker.autoscaling.hpa.enabled }} + replicas: {{ .Values.worker.replicaCount }} + {{- end }} + {{- if .Values.worker.updateStrategy }} + strategy: {{- toYaml .Values.worker.updateStrategy | nindent 4 }} + {{- end }} + {{- $podLabels := include "common.tplvalues.merge" (dict "values" (list .Values.worker.podLabels .Values.commonLabels) "context" .) }} + selector: + matchLabels: {{- include "shuffle.worker.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 6 }} + template: + metadata: + {{- if .Values.worker.podAnnotations }} + annotations: {{- include "common.tplvalues.render" (dict "value" .Values.worker.podAnnotations "context" $) | nindent 8 }} + {{- end }} + labels: {{- include "shuffle.worker.labels" ( dict "customLabels" $podLabels "context" $ ) | nindent 8 }} + spec: + {{- include "shuffle.worker.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ template "shuffle.worker.serviceAccount.name" . }} + automountServiceAccountToken: {{ .Values.worker.automountServiceAccountToken }} + {{- if .Values.worker.hostAliases }} + hostAliases: {{- include "common.tplvalues.render" (dict "value" .Values.worker.hostAliases "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.worker.affinity }} + affinity: {{- include "common.tplvalues.render" ( dict "value" .Values.worker.affinity "context" $) | nindent 8 }} + {{- else }} + affinity: + podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.worker.podAffinityPreset "component" "orborus" "customLabels" $podLabels "context" $) | nindent 10 }} + podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.worker.podAntiAffinityPreset "component" "orborus" "customLabels" $podLabels "context" $) | nindent 10 }} + nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.worker.nodeAffinityPreset.type "key" .Values.worker.nodeAffinityPreset.key "values" .Values.worker.nodeAffinityPreset.values) | nindent 10 }} + {{- end }} + {{- if .Values.worker.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" ( dict "value" .Values.worker.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.worker.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.worker.tolerations "context" .) | nindent 8 }} + {{- end }} + {{- if .Values.worker.priorityClassName }} + priorityClassName: {{ .Values.worker.priorityClassName | quote }} + {{- end }} + {{- if .Values.worker.schedulerName }} + schedulerName: {{ .Values.worker.schedulerName | quote }} + {{- end }} + {{- if .Values.worker.topologySpreadConstraints }} + topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.worker.topologySpreadConstraints "context" .) | nindent 8 }} + {{- end }} + {{- if .Values.worker.podSecurityContext.enabled }} + securityContext: {{- omit .Values.worker.podSecurityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + {{- if .Values.worker.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ .Values.worker.terminationGracePeriodSeconds }} + {{- end }} + initContainers: + {{- if .Values.worker.initContainers }} + {{- include "common.tplvalues.render" (dict "value" .Values.worker.initContainers "context" $) | nindent 8 }} + {{- end }} + containers: + - name: orborus + image: {{ template "shuffle.worker.image" . }} + imagePullPolicy: {{ .Values.worker.image.pullPolicy }} + {{- if .Values.worker.containerSecurityContext.enabled }} + securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.worker.containerSecurityContext "context" $) | nindent 12 }} + {{- end }} + {{- if .Values.diagnosticMode.enabled }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }} + {{- else if .Values.worker.command }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.worker.command "context" $) | nindent 12 }} + {{- end }} + {{- if .Values.diagnosticMode.enabled }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 12 }} + {{- else if .Values.worker.args }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.worker.args "context" $) | nindent 12 }} + {{- end }} + env: + - name: IS_KUBERNETES + value: "true" + - name: KUBERNETES_NAMESPACE + value: "{{ .Release.Namespace }}" + - name: SHUFFLE_LOGS_DISABLED + value: "false" # TODO: Allow to configure. Share with orborus / apps? + - name: BASE_URL + value: {{ import "shuffle.backend.baseUrl" . | quote }} + - name: REGISTRY_URL # TODO: REGISTRY_URL vs. SHUFFLE_BASE_IMAGE_REGISTRY + value: {{ .Values.shuffle.appRegistry }} + - name: SHUFFLE_BASE_IMAGE_REGISTRY + value: {{ .Values.shuffle.appRegistry }} + - name: SHUFFLE_BASE_IMAGE_NAME + value: shuffle # TODO: Allow to configure? Use same as for orborus (shared value) + - name: SHUFFLE_APP_EXPOSED_PORT + value: {{ .Values.app.exposedContainerPort | quote }} + - name: WORKER_HOSTNAME + value: {{ include "shuffle.worker.name" }}.{{ .Release.Namespace }}.svc.cluster.local + {{- if .Values.worker.extraEnvVars }} + {{- include "common.tplvalues.render" (dict "value" .Values.worker.extraEnvVars "context" $) | nindent 12 }} + {{- end }} + envFrom: + {{- if .Values.worker.extraEnvVarsCM }} + - configMapRef: + name: {{ include "common.tplvalues.render" (dict "value" .Values.worker.extraEnvVarsCM "context" $) }} + {{- end }} + {{- if .Values.worker.extraEnvVarsSecret }} + - secretRef: + name: {{ include "common.tplvalues.render" (dict "value" .Values.worker.extraEnvVarsSecret "context" $) }} + {{- end }} + {{- if .Values.worker.resources }} + resources: {{- toYaml .Values.worker.resources | nindent 12 }} + {{- else if ne .Values.worker.resourcesPreset "none" }} + resources: {{- include "common.resources.preset" (dict "type" .Values.worker.resourcesPreset) | nindent 12 }} + {{- end }} + ports: + {{- if .Values.worker.extraContainerPorts }} + {{- include "common.tplvalues.render" (dict "value" .Values.worker.extraContainerPorts "context" $) | nindent 12 }} + {{- end }} + {{- if not .Values.diagnosticMode.enabled }} + {{- if .Values.worker.customLivenessProbe }} + livenessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.worker.customLivenessProbe "context" $) | nindent 12 }} + {{- else if .Values.worker.livenessProbe.enabled }} + livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.worker.livenessProbe "enabled") "context" $) | nindent 12 }} + {{- end }} + {{- if .Values.worker.customReadinessProbe }} + readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.worker.customReadinessProbe "context" $) | nindent 12 }} + {{- else if .Values.worker.readinessProbe.enabled }} + readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.worker.readinessProbe "enabled") "context" $) | nindent 12 }} + {{- end }} + {{- if .Values.worker.customStartupProbe }} + startupProbe: {{- include "common.tplvalues.render" (dict "value" .Values.worker.customStartupProbe "context" $) | nindent 12 }} + {{- else if .Values.worker.startupProbe.enabled }} + startupProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.worker.startupProbe "enabled") "context" $) | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.worker.lifecycleHooks }} + lifecycle: {{- include "common.tplvalues.render" (dict "value" .Values.worker.lifecycleHooks "context" $) | nindent 12 }} + {{- end }} + volumeMounts: + - name: empty-dir + mountPath: /tmp + subPath: tmp-dir + {{- if .Values.worker.extraVolumeMounts }} + {{- include "common.tplvalues.render" (dict "value" .Values.worker.extraVolumeMounts "context" $) | nindent 12 }} + {{- end }} + {{- if .Values.worker.sidecars }} + {{- include "common.tplvalues.render" ( dict "value" .Values.worker.sidecars "context" $) | nindent 8 }} + {{- end }} + volumes: + - name: empty-dir + emptyDir: {} + {{- if .Values.worker.extraVolumes }} + {{- include "common.tplvalues.render" (dict "value" .Values.worker.extraVolumes "context" $) | nindent 8 }} + {{- end }} +{{- end }} diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-role.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-role.yaml index fdef5855..fffa4d4b 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-role.yaml +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-role.yaml @@ -9,6 +9,7 @@ metadata: annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} {{- end }} rules: + {{- if .Values.worker.manageAppDeployments }} - apiGroups: [""] resources: ["pods"] verbs: ["list", "delete"] @@ -18,4 +19,11 @@ rules: - apiGroups: ["apps"] resources: ["deployments"] verbs: ["list", "create"] + {{- else }} + - apiGroups: [""] + resources: ["pods"] + verbs: ["list"] + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["list"] {{- end }} diff --git a/functions/kubernetes/charts/shuffle/values.yaml b/functions/kubernetes/charts/shuffle/values.yaml index 6e1b6b8f..8ee8ee06 100644 --- a/functions/kubernetes/charts/shuffle/values.yaml +++ b/functions/kubernetes/charts/shuffle/values.yaml @@ -84,7 +84,7 @@ shuffle: ## @param shuffle.appRegistry The registry from / to which shuffle apps are pulled / pushed ## - appRegistry: "" + appRegistry: "" # TODO: Default? ## @param shuffle.timezone The timezone used by Shuffle ## @@ -1209,7 +1209,7 @@ orborus: - name: KUBERNETES_NAMESPACE value: "shuffle" - name: SHUFFLE_BASE_IMAGE_NAME - value: "frikky/shuffle" + value: "frikky/shuffle" # TODO: Allow to configure, share with worker ## @param orborus.extraEnvVarsCM Name of existing ConfigMap containing extra env vars for orborus containers ## extraEnvVarsCM: "" @@ -1346,10 +1346,20 @@ orborus: ## istio: pilot ## extraEgress: [] + + ## @param orborus.manageWorkerDeployments Whether workers are deployed and managed by orborus. When disabled, every worker is expected to be already deployed (see worker.enableHelmDeployment). + ## This effectively removes required RBAC permissions from the shuffle-orborus service account to create deployments and services. + ## Orborus might still attempt to create kubernetes objects, resulting in an error. There is currently no way to tell orborus, that it should not manage k8s resources. + ## You likely want to enable worker.enableHelmDeployment when enabling this. + manageWorkerDeployments: true ## @section worker Parameters ## -worker: +worker: # TODO: Add values that other helm deployed components have, mark helm-only params + ## @param worker.enableHelmDeployment Deploy worker via helm. By default, workers are deployed by Orborus. + ## You might want to disable orborus.manageWorkerDeployments when enabling this. + enableHelmDeployment: false + ## worker image ## @param worker.image.registry worker image registry ## @param worker.image.repository worker image repository @@ -1480,6 +1490,11 @@ worker: ## istio: pilot ## extraEgress: [] + + ## @param worker.manageAppDeployments Whether apps are deployed and managed by worker. When disabled, every used app is expected to to be already deployed (see app.deployViaHelm). + ## This effectively removes required RBAC permissions from the shuffle-worker service account to create deployments and services. + ## The worker might still attempt to create kubernetes objects, resulting in an error. There is currently no way to tell the worker, that it should not manage k8s resources. + manageAppDeployments: true ## @section app Parameters ## @@ -1598,6 +1613,32 @@ app: ## exposedContainerPort: 80 + ## @param app.deployViaHelm A list of apps that should be deployed using helm. + ## By default, workers create deployments and services for apps when they are first needed (or during startup for some selected apps). + ## Deploying apps via workers has some drawbacks, such as: + ## - A workflow fails when the app is not deployed when the workflow gets executed (see https://github.com/Shuffle/Shuffle/issues/1739) + ## - There is no way to set different service accounts, security contexts, resources, env variables, volume mounts, or replicas for different apps + ## - Worker needs elevated permissions in Kubernetes + ## Note that you can deploy some apps via helm, while keeping the flexibility of letting workers deploy apps if they are not already deployed. + ## If you deploy all needed apps via helm and dont want workers to create additional deployments, set worker.manageAppDeployments to false. + ## + ## Sample value: + ## - app: my-app # required + ## version: 1.0.0 # required + ## replicas: 1 # optional, default to 1 + ## image: "" # optional, default to appregistry/baseimage/appname:appversion + ## imagePullSecrets: null # optional, defaults to global.imagePullSecrets + ## resources: null # optional, defaults to app.resources (or app.resourcePreset) + ## extraVolumes: [] # optional + ## extraVolumeMounts: [] # optional + ## extraEnv: [] # optional + ## podSecurityContext: null # optional, defaults to app.podSecurityContext + ## containerSecurityContext: null # optional, defaults to app.containerSecurityContext + ## + ## TODO: allow to override resources, serviceAccount, mounts, env, security Contexts. Fall back to app defaults as defined above. + ## + deployViaHelm: [] + ## @section Traffic Exposure Parameters ## diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index a1cda9a2..b9e7607f 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -1160,6 +1160,7 @@ func fixk8sRoles() { } } +// TODO: Check if deployment or service already exist by labels and only create if not already exists func deployK8sWorker(image string, identifier string, env []string) error { env = append(env, fmt.Sprintf("IS_KUBERNETES=true")) env = append(env, fmt.Sprintf("KUBERNETES_NAMESPACE=%s", os.Getenv("KUBERNETES_NAMESPACE")))