From 9fa02c15b139b8d339d47b50dfababffcc69f799 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Thu, 17 Apr 2025 09:59:37 +0200 Subject: [PATCH 1/5] feat(k8s): allow to set security contexts for worker and apps Signed-off-by: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> --- functions/kubernetes/charts/shuffle/README.md | 92 ++++++--- .../templates/orborus/orborus-dpl.yaml | 16 ++ .../charts/shuffle/values.schema.json | 194 ++++++++++++++++++ .../kubernetes/charts/shuffle/values.yaml | 84 ++++++++ functions/onprem/orborus/orborus.go | 45 +++- functions/onprem/worker/worker.go | 66 ++++-- 6 files changed, 442 insertions(+), 55 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/README.md b/functions/kubernetes/charts/shuffle/README.md index ff3885d1..5c70ab50 100644 --- a/functions/kubernetes/charts/shuffle/README.md +++ b/functions/kubernetes/charts/shuffle/README.md @@ -477,39 +477,69 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia ### worker Parameters -| Name | Description | Value | -| ---------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------ | -| `worker.image.registry` | worker image registry | `ghcr.io` | -| `worker.image.repository` | worker image repository | `shuffle/shuffle-worker` | -| `worker.image.tag` | worker image tag (immutable tags are recommended, defaults to appVersion) | `""` | -| `worker.image.digest` | worker image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended) | `""` | -| `worker.serviceAccount.create` | Specifies whether a ServiceAccount should be created | `true` | -| `worker.serviceAccount.name` | The name of the ServiceAccount to use. | `""` | -| `worker.serviceAccount.annotations` | Additional Service Account annotations (evaluated as a template) | `{}` | -| `worker.serviceAccount.automountServiceAccountToken` | Automount service account token for the worker service account | `true` | -| `worker.serviceAccount.imagePullSecrets` | Add image pull secrets to the worker service account | `[]` | -| `worker.rbac.create` | Specifies whether RBAC resources should be created | `true` | -| `worker.networkPolicy.enabled` | Specifies whether a NetworkPolicy should be created | `true` | -| `worker.networkPolicy.allowExternal` | Don't require server label for connections | `true` | -| `worker.networkPolicy.allowExternalEgress` | Allow the pod to access any range of port and all destinations. | `true` | -| `worker.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` | -| `worker.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | +| Name | Description | Value | +| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------ | +| `worker.image.registry` | worker image registry | `ghcr.io` | +| `worker.image.repository` | worker image repository | `shuffle/shuffle-worker` | +| `worker.image.tag` | worker image tag (immutable tags are recommended, defaults to appVersion) | `""` | +| `worker.image.digest` | worker image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag image tag (immutable tags are recommended) | `""` | +| `worker.podSecurityContext.enabled` | Enable worker pods' Security Context | `true` | +| `worker.podSecurityContext.fsGroupChangePolicy` | Set filesystem group change policy for worker pods | `Always` | +| `worker.podSecurityContext.sysctls` | Set kernel settings using the sysctl interface for worker pods | `[]` | +| `worker.podSecurityContext.supplementalGroups` | Set filesystem extra groups for worker pods | `[]` | +| `worker.podSecurityContext.fsGroup` | Set fsGroup in worker pods' Security Context | `1001` | +| `worker.containerSecurityContext.enabled` | Enabled worker container' Security Context | `true` | +| `worker.containerSecurityContext.seLinuxOptions` | Set SELinux options in worker container | `{}` | +| `worker.containerSecurityContext.runAsUser` | Set runAsUser in worker container' Security Context | `1001` | +| `worker.containerSecurityContext.runAsGroup` | Set runAsGroup in worker container' Security Context | `1001` | +| `worker.containerSecurityContext.runAsNonRoot` | Set runAsNonRoot in worker container' Security Context | `true` | +| `worker.containerSecurityContext.readOnlyRootFilesystem` | Set readOnlyRootFilesystem in worker container' Security Context | `true` | +| `worker.containerSecurityContext.privileged` | Set privileged in worker container' Security Context | `false` | +| `worker.containerSecurityContext.allowPrivilegeEscalation` | Set allowPrivilegeEscalation in worker container' Security Context | `false` | +| `worker.containerSecurityContext.capabilities.drop` | List of capabilities to be dropped in worker container | `["ALL"]` | +| `worker.containerSecurityContext.seccompProfile.type` | Set seccomp profile in worker container | `RuntimeDefault` | +| `worker.serviceAccount.create` | Specifies whether a ServiceAccount should be created | `true` | +| `worker.serviceAccount.name` | The name of the ServiceAccount to use. | `""` | +| `worker.serviceAccount.annotations` | Additional Service Account annotations (evaluated as a template) | `{}` | +| `worker.serviceAccount.automountServiceAccountToken` | Automount service account token for the worker service account | `true` | +| `worker.serviceAccount.imagePullSecrets` | Add image pull secrets to the worker service account | `[]` | +| `worker.rbac.create` | Specifies whether RBAC resources should be created | `true` | +| `worker.networkPolicy.enabled` | Specifies whether a NetworkPolicy should be created | `true` | +| `worker.networkPolicy.allowExternal` | Don't require server label for connections | `true` | +| `worker.networkPolicy.allowExternalEgress` | Allow the pod to access any range of port and all destinations. | `true` | +| `worker.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` | +| `worker.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | ### app Parameters -| Name | Description | Value | -| ------------------------------------------------- | ---------------------------------------------------------------------------------- | ------ | -| `app.serviceAccount.create` | Specifies whether a ServiceAccount should be created | `true` | -| `app.serviceAccount.name` | The name of the ServiceAccount to use. | `""` | -| `app.serviceAccount.annotations` | Additional Service Account annotations (evaluated as a template) | `{}` | -| `app.serviceAccount.automountServiceAccountToken` | Automount service account token for the app service account | `true` | -| `app.serviceAccount.imagePullSecrets` | Add image pull secrets to the app service account | `[]` | -| `app.rbac.create` | Specifies whether RBAC resources should be created | `true` | -| `app.networkPolicy.enabled` | Specifies whether a NetworkPolicy should be created | `true` | -| `app.networkPolicy.allowExternal` | Don't require server label for connections | `true` | -| `app.networkPolicy.allowExternalEgress` | Allow the pod to access any range of port and all destinations. | `true` | -| `app.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` | -| `app.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | +| Name | Description | Value | +| ------------------------------------------------------- | ---------------------------------------------------------------------------------- | ---------------- | +| `app.podSecurityContext.enabled` | Enable app pods' Security Context | `true` | +| `app.podSecurityContext.fsGroupChangePolicy` | Set filesystem group change policy for app pods | `Always` | +| `app.podSecurityContext.sysctls` | Set kernel settings using the sysctl interface for app pods | `[]` | +| `app.podSecurityContext.supplementalGroups` | Set filesystem extra groups for app pods | `[]` | +| `app.podSecurityContext.fsGroup` | Set fsGroup in app pods' Security Context | `1001` | +| `app.containerSecurityContext.enabled` | Enabled app container' Security Context | `true` | +| `app.containerSecurityContext.seLinuxOptions` | Set SELinux options in app container | `{}` | +| `app.containerSecurityContext.runAsUser` | Set runAsUser in app container' Security Context | `1001` | +| `app.containerSecurityContext.runAsGroup` | Set runAsGroup in app container' Security Context | `1001` | +| `app.containerSecurityContext.runAsNonRoot` | Set runAsNonRoot in app container' Security Context | `true` | +| `app.containerSecurityContext.readOnlyRootFilesystem` | Set readOnlyRootFilesystem in app container' Security Context | `true` | +| `app.containerSecurityContext.privileged` | Set privileged in app container' Security Context | `false` | +| `app.containerSecurityContext.allowPrivilegeEscalation` | Set allowPrivilegeEscalation in app container' Security Context | `false` | +| `app.containerSecurityContext.capabilities.drop` | List of capabilities to be dropped in app container | `["ALL"]` | +| `app.containerSecurityContext.seccompProfile.type` | Set seccomp profile in app container | `RuntimeDefault` | +| `app.serviceAccount.create` | Specifies whether a ServiceAccount should be created | `true` | +| `app.serviceAccount.name` | The name of the ServiceAccount to use. | `""` | +| `app.serviceAccount.annotations` | Additional Service Account annotations (evaluated as a template) | `{}` | +| `app.serviceAccount.automountServiceAccountToken` | Automount service account token for the app service account | `true` | +| `app.serviceAccount.imagePullSecrets` | Add image pull secrets to the app service account | `[]` | +| `app.rbac.create` | Specifies whether RBAC resources should be created | `true` | +| `app.networkPolicy.enabled` | Specifies whether a NetworkPolicy should be created | `true` | +| `app.networkPolicy.allowExternal` | Don't require server label for connections | `true` | +| `app.networkPolicy.allowExternalEgress` | Allow the pod to access any range of port and all destinations. | `true` | +| `app.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` | +| `app.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]` | ### Traffic Exposure Parameters @@ -607,3 +637,5 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia ### Other Parameters + + diff --git a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml index a2d9c278..5911eb19 100644 --- a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml +++ b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml @@ -88,8 +88,24 @@ spec: value: "true" - name: SHUFFLE_WORKER_SERVICE_ACCOUNT_NAME value: {{ include "shuffle.worker.serviceAccount.name" . }} + {{- if .Values.worker.podSecurityContext.enabled }} + - name: SHUFFLE_WORKER_POD_SECURITY_CONTEXT + value: {{ omit .Values.worker.podSecurityContext "enabled" | mustToJson | quote }} + {{- end }} + {{- if .Values.worker.containerSecurityContext.enabled }} + - name: SHUFFLE_WORKER_CONTAINER_SECURITY_CONTEXT + value: {{ include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.worker.containerSecurityContext "context" $) | fromYaml | mustToJson | quote }} + {{- end }} - name: SHUFFLE_APP_SERVICE_ACCOUNT_NAME value: {{ include "shuffle.app.serviceAccount.name" . }} + {{- if .Values.app.podSecurityContext.enabled }} + - name: SHUFFLE_APP_POD_SECURITY_CONTEXT + value: {{ omit .Values.app.podSecurityContext "enabled" | mustToJson | quote }} + {{- end }} + {{- if .Values.app.containerSecurityContext.enabled }} + - name: SHUFFLE_APP_CONTAINER_SECURITY_CONTEXT + value: {{ include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.app.containerSecurityContext "context" $) | fromYaml | mustToJson | quote }} + {{- end }} {{- if .Values.orborus.extraEnvVars }} {{- include "common.tplvalues.render" (dict "value" .Values.orborus.extraEnvVars "context" $) | nindent 12 }} {{- end }} diff --git a/functions/kubernetes/charts/shuffle/values.schema.json b/functions/kubernetes/charts/shuffle/values.schema.json index b785b76e..c2848aaa 100644 --- a/functions/kubernetes/charts/shuffle/values.schema.json +++ b/functions/kubernetes/charts/shuffle/values.schema.json @@ -2074,6 +2074,103 @@ } } }, + "podSecurityContext": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable worker pods' Security Context", + "default": true + }, + "fsGroupChangePolicy": { + "type": "string", + "description": "Set filesystem group change policy for worker pods", + "default": "Always" + }, + "sysctls": { + "type": "array", + "description": "Set kernel settings using the sysctl interface for worker pods", + "default": [], + "items": {} + }, + "supplementalGroups": { + "type": "array", + "description": "Set filesystem extra groups for worker pods", + "default": [], + "items": {} + }, + "fsGroup": { + "type": "number", + "description": "Set fsGroup in worker pods' Security Context", + "default": 1001 + } + } + }, + "containerSecurityContext": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enabled worker container' Security Context", + "default": true + }, + "runAsUser": { + "type": "number", + "description": "Set runAsUser in worker container' Security Context", + "default": 1001 + }, + "runAsGroup": { + "type": "number", + "description": "Set runAsGroup in worker container' Security Context", + "default": 1001 + }, + "runAsNonRoot": { + "type": "boolean", + "description": "Set runAsNonRoot in worker container' Security Context", + "default": true + }, + "readOnlyRootFilesystem": { + "type": "boolean", + "description": "Set readOnlyRootFilesystem in worker container' Security Context", + "default": true + }, + "privileged": { + "type": "boolean", + "description": "Set privileged in worker container' Security Context", + "default": false + }, + "allowPrivilegeEscalation": { + "type": "boolean", + "description": "Set allowPrivilegeEscalation in worker container' Security Context", + "default": false + }, + "capabilities": { + "type": "object", + "properties": { + "drop": { + "type": "array", + "description": "List of capabilities to be dropped in worker container", + "default": [ + "ALL" + ], + "items": { + "type": "string" + } + } + } + }, + "seccompProfile": { + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "Set seccomp profile in worker container", + "default": "RuntimeDefault" + } + } + } + } + }, "serviceAccount": { "type": "object", "properties": { @@ -2152,6 +2249,103 @@ "app": { "type": "object", "properties": { + "podSecurityContext": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable app pods' Security Context", + "default": true + }, + "fsGroupChangePolicy": { + "type": "string", + "description": "Set filesystem group change policy for app pods", + "default": "Always" + }, + "sysctls": { + "type": "array", + "description": "Set kernel settings using the sysctl interface for app pods", + "default": [], + "items": {} + }, + "supplementalGroups": { + "type": "array", + "description": "Set filesystem extra groups for app pods", + "default": [], + "items": {} + }, + "fsGroup": { + "type": "number", + "description": "Set fsGroup in app pods' Security Context", + "default": 1001 + } + } + }, + "containerSecurityContext": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enabled app container' Security Context", + "default": true + }, + "runAsUser": { + "type": "number", + "description": "Set runAsUser in app container' Security Context", + "default": 1001 + }, + "runAsGroup": { + "type": "number", + "description": "Set runAsGroup in app container' Security Context", + "default": 1001 + }, + "runAsNonRoot": { + "type": "boolean", + "description": "Set runAsNonRoot in app container' Security Context", + "default": true + }, + "readOnlyRootFilesystem": { + "type": "boolean", + "description": "Set readOnlyRootFilesystem in app container' Security Context", + "default": true + }, + "privileged": { + "type": "boolean", + "description": "Set privileged in app container' Security Context", + "default": false + }, + "allowPrivilegeEscalation": { + "type": "boolean", + "description": "Set allowPrivilegeEscalation in app container' Security Context", + "default": false + }, + "capabilities": { + "type": "object", + "properties": { + "drop": { + "type": "array", + "description": "List of capabilities to be dropped in app container", + "default": [ + "ALL" + ], + "items": { + "type": "string" + } + } + } + }, + "seccompProfile": { + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "Set seccomp profile in app container", + "default": "RuntimeDefault" + } + } + } + } + }, "serviceAccount": { "type": "object", "properties": { diff --git a/functions/kubernetes/charts/shuffle/values.yaml b/functions/kubernetes/charts/shuffle/values.yaml index 507c2468..db11c0e6 100644 --- a/functions/kubernetes/charts/shuffle/values.yaml +++ b/functions/kubernetes/charts/shuffle/values.yaml @@ -1328,6 +1328,48 @@ worker: tag: "" digest: "" + ## Configure Pods Security Context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod + ## @param worker.podSecurityContext.enabled Enable worker pods' Security Context + ## @param worker.podSecurityContext.fsGroupChangePolicy Set filesystem group change policy for worker pods + ## @param worker.podSecurityContext.sysctls Set kernel settings using the sysctl interface for worker pods + ## @param worker.podSecurityContext.supplementalGroups Set filesystem extra groups for worker pods + ## @param worker.podSecurityContext.fsGroup Set fsGroup in worker pods' Security Context + ## + podSecurityContext: + enabled: true + fsGroupChangePolicy: Always + sysctls: [] + supplementalGroups: [] + fsGroup: 1001 + + ## Configure Container Security Context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + ## @param worker.containerSecurityContext.enabled Enabled worker container' Security Context + ## @param worker.containerSecurityContext.seLinuxOptions [object,nullable] Set SELinux options in worker container + ## @param worker.containerSecurityContext.runAsUser Set runAsUser in worker container' Security Context + ## @param worker.containerSecurityContext.runAsGroup Set runAsGroup in worker container' Security Context + ## @param worker.containerSecurityContext.runAsNonRoot Set runAsNonRoot in worker container' Security Context + ## @param worker.containerSecurityContext.readOnlyRootFilesystem Set readOnlyRootFilesystem in worker container' Security Context + ## @param worker.containerSecurityContext.privileged Set privileged in worker container' Security Context + ## @param worker.containerSecurityContext.allowPrivilegeEscalation Set allowPrivilegeEscalation in worker container' Security Context + ## @param worker.containerSecurityContext.capabilities.drop List of capabilities to be dropped in worker container + ## @param worker.containerSecurityContext.seccompProfile.type Set seccomp profile in worker container + ## + containerSecurityContext: + enabled: true + seLinuxOptions: {} + runAsUser: 1001 + runAsGroup: 1001 + runAsNonRoot: true + readOnlyRootFilesystem: true + privileged: false + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: "RuntimeDefault" + ## ServiceAccount configuration ## serviceAccount: @@ -1390,6 +1432,48 @@ worker: ## @section app Parameters ## app: + ## Configure Pods Security Context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod + ## @param app.podSecurityContext.enabled Enable app pods' Security Context + ## @param app.podSecurityContext.fsGroupChangePolicy Set filesystem group change policy for app pods + ## @param app.podSecurityContext.sysctls Set kernel settings using the sysctl interface for app pods + ## @param app.podSecurityContext.supplementalGroups Set filesystem extra groups for app pods + ## @param app.podSecurityContext.fsGroup Set fsGroup in app pods' Security Context + ## + podSecurityContext: + enabled: true + fsGroupChangePolicy: Always + sysctls: [] + supplementalGroups: [] + fsGroup: 1001 + + ## Configure Container Security Context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + ## @param app.containerSecurityContext.enabled Enabled app container' Security Context + ## @param app.containerSecurityContext.seLinuxOptions [object,nullable] Set SELinux options in app container + ## @param app.containerSecurityContext.runAsUser Set runAsUser in app container' Security Context + ## @param app.containerSecurityContext.runAsGroup Set runAsGroup in app container' Security Context + ## @param app.containerSecurityContext.runAsNonRoot Set runAsNonRoot in app container' Security Context + ## @param app.containerSecurityContext.readOnlyRootFilesystem Set readOnlyRootFilesystem in app container' Security Context + ## @param app.containerSecurityContext.privileged Set privileged in app container' Security Context + ## @param app.containerSecurityContext.allowPrivilegeEscalation Set allowPrivilegeEscalation in app container' Security Context + ## @param app.containerSecurityContext.capabilities.drop List of capabilities to be dropped in app container + ## @param app.containerSecurityContext.seccompProfile.type Set seccomp profile in app container + ## + containerSecurityContext: + enabled: true + seLinuxOptions: {} + runAsUser: 1001 + runAsGroup: 1001 + runAsNonRoot: true + readOnlyRootFilesystem: true + privileged: false + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: "RuntimeDefault" + ## ServiceAccount configuration ## serviceAccount: diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 1f37da85..b44ca0c6 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -76,7 +76,11 @@ var maxCPUPercent = 90 var isKubernetes = os.Getenv("IS_KUBERNETES") var kubernetesNamespace = os.Getenv("KUBERNETES_NAMESPACE") var workerServiceAccountName = os.Getenv("SHUFFLE_WORKER_SERVICE_ACCOUNT_NAME") +var workerPodSecurityContext = os.Getenv("SHUFFLE_WORKER_POD_SECURITY_CONTEXT") +var workerContainerSecurityContext = os.Getenv("SHUFFLE_WORKER_CONTAINER_SECURITY_CONTEXT") var appServiceAccountName = os.Getenv("SHUFFLE_APP_SERVICE_ACCOUNT_NAME") +var appPodSecurityContext = os.Getenv("SHUFFLE_APP_POD_SECURITY_CONTEXT") +var appContainerSecurityContext = os.Getenv("SHUFFLE_APP_CONTAINER_SECURITY_CONTEXT") // var baseimagename = "docker.pkg.github.com/shuffle/shuffle" // var baseimagename = "ghcr.io/frikky" @@ -747,7 +751,7 @@ func handleBackendImageDownload(ctx context.Context, images string) error { //log.Printf("[DEBUG] Removing existing image (s): %s", images) newImages := []string{} - successful := []string{} + successful := []string{} for _, curimage := range strings.Split(images, ",") { curimage = strings.TrimSpace(curimage) if shuffle.ArrayContains(handled, curimage) { @@ -1000,6 +1004,14 @@ func deployK8sWorker(image string, identifier string, env []string) error { env = append(env, fmt.Sprintf("SHUFFLE_APP_SERVICE_ACCOUNT_NAME=%s", appServiceAccountName)) } + if len(appPodSecurityContext) > 0 { + env = append(env, fmt.Sprintf("SHUFFLE_APP_POD_SECURITY_CONTEXT=%s", appPodSecurityContext)) + } + + if len(appContainerSecurityContext) > 0 { + env = append(env, fmt.Sprintf("SHUFFLE_APP_CONTAINER_SECURITY_CONTEXT=%s", appContainerSecurityContext)) + } + clientset, _, err := shuffle.GetKubernetesClient() if err != nil { log.Printf("[ERROR] Error getting kubernetes client:", err) @@ -1081,10 +1093,33 @@ func deployK8sWorker(image string, identifier string, env []string) error { "app.kubernetes.io/instance": identifier, } + // Parse security contexts from env + var podSecurityContext *corev1.PodSecurityContext + var containerSecurityContext *corev1.SecurityContext + + if len(workerPodSecurityContext) > 0 { + podSecurityContext = &corev1.PodSecurityContext{} + err = json.Unmarshal([]byte(workerPodSecurityContext), podSecurityContext) + if err != nil { + log.Printf("[ERROR] Failed to unmarshal worker pod security context: %v", err) + return fmt.Errorf("failed to unmarshal worker pod security context: %v", err) + } + } + + if len(workerContainerSecurityContext) > 0 { + containerSecurityContext = &corev1.SecurityContext{} + err = json.Unmarshal([]byte(workerContainerSecurityContext), containerSecurityContext) + if err != nil { + log.Printf("[ERROR] Failed to unmarshal worker container security context: %v", err) + return fmt.Errorf("failed to unmarshal worker container security context: %v", err) + } + } + containerAttachment := corev1.Container{ - Name: identifier, - Image: kubernetesImage, - Env: buildEnvVars(envMap), + Name: identifier, + Image: kubernetesImage, + Env: buildEnvVars(envMap), + SecurityContext: containerSecurityContext, //ImagePullPolicy: "Never", ImagePullPolicy: corev1.PullIfNotPresent, @@ -1201,6 +1236,7 @@ func deployK8sWorker(image string, identifier string, env []string) error { }, DNSPolicy: corev1.DNSClusterFirst, ServiceAccountName: workerServiceAccountName, + SecurityContext: podSecurityContext, }, }, }, @@ -1271,7 +1307,6 @@ func deployWorker(image string, identifier string, env []string, executionReques Resources: container.Resources{}, } - // This is just to test the mounting locally so // I can control from what source I'm mounting // the certs to. Default behaviour is: diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index a8f1c204..4cdacbb7 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -57,9 +57,13 @@ var logsDisabled = os.Getenv("SHUFFLE_LOGS_DISABLED") var cleanupEnv = strings.ToLower(os.Getenv("CLEANUP")) var swarmNetworkName = os.Getenv("SHUFFLE_SWARM_NETWORK_NAME") var dockerApiVersion = strings.ToLower(os.Getenv("DOCKER_API_VERSION")) -var appServiceAccountName = os.Getenv("SHUFFLE_APP_SERVICE_ACCOUNT_NAME") +// Kubernetes settings +var appServiceAccountName = os.Getenv("SHUFFLE_APP_SERVICE_ACCOUNT_NAME") +var appPodSecurityContext = os.Getenv("SHUFFLE_APP_POD_SECURITY_CONTEXT") +var appContainerSecurityContext = os.Getenv("SHUFFLE_APP_CONTAINER_SECURITY_CONTEXT") var kubernetesNamespace = os.Getenv("KUBERNETES_NAMESPACE") + var executionCount int64 var baseimagename = os.Getenv("SHUFFLE_BASE_IMAGE_NAME") @@ -503,6 +507,28 @@ func deployk8sApp(image string, identifier string, env []string) error { "app.kubernetes.io/instance": name, } + // Parse security contexts from env + var podSecurityContext *corev1.PodSecurityContext + var containerSecurityContext *corev1.SecurityContext + + if len(appPodSecurityContext) > 0 { + podSecurityContext = &corev1.PodSecurityContext{} + err = json.Unmarshal([]byte(appPodSecurityContext), podSecurityContext) + if err != nil { + log.Printf("[ERROR] Failed to unmarshal app pod security context: %v", err) + return fmt.Errorf("failed to unmarshal app pod security context: %v", err) + } + } + + if len(appContainerSecurityContext) > 0 { + containerSecurityContext = &corev1.SecurityContext{} + err = json.Unmarshal([]byte(appContainerSecurityContext), containerSecurityContext) + if err != nil { + log.Printf("[ERROR] Failed to unmarshal app container security context: %v", err) + return fmt.Errorf("failed to unmarshal app container security context: %v", err) + } + } + // pod := &corev1.Pod{ // ObjectMeta: metav1.ObjectMeta{ // Name: podName, @@ -596,13 +622,15 @@ func deployk8sApp(image string, identifier string, env []string) error { Spec: corev1.PodSpec{ Containers: []corev1.Container{ { - Name: value, - Image: image, - Env: buildEnvVars(envMap), + Name: value, + Image: image, + Env: buildEnvVars(envMap), + SecurityContext: containerSecurityContext, }, }, DNSPolicy: corev1.DNSClusterFirst, ServiceAccountName: appServiceAccountName, + SecurityContext: podSecurityContext, }, }, }, @@ -917,7 +945,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] // Add more volume binds if possible if len(volumeBinds) > 0 { - // Only use mounts, not direct binds + // Only use mounts, not direct binds hostConfig.Binds = []string{} hostConfig.Mounts = []mount.Mount{} for _, bind := range volumeBinds { @@ -931,7 +959,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] sourceFolder := bindSplit[0] destinationFolder := bindSplit[1] - readOnly := false + readOnly := false if len(bindSplit) > 2 { mode := bindSplit[2] if mode == "ro" { @@ -940,9 +968,9 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] } builtMount := mount.Mount{ - Type: mount.TypeBind, - Source: sourceFolder, - Target: destinationFolder, + Type: mount.TypeBind, + Source: sourceFolder, + Target: destinationFolder, ReadOnly: readOnly, } @@ -1853,18 +1881,18 @@ func executionInit(workflowExecution shuffle.WorkflowExecution) error { } } - // Validates RERUN of single actions - // Identified by: + // Validates RERUN of single actions + // Identified by: // 1. Predefined result from previous exec // 2. Only ONE action // 3. Every predefined result having result.Action.Category == "rerun" /* - if len(workflowExecution.Workflow.Actions) == 1 && len(workflowExecution.Results) > 0 { - finished := shuffle.ValidateFinished(ctx, extra, workflowExecution) - if finished { - return nil + if len(workflowExecution.Workflow.Actions) == 1 && len(workflowExecution.Results) > 0 { + finished := shuffle.ValidateFinished(ctx, extra, workflowExecution) + if finished { + return nil + } } - } */ nextActions = append(nextActions, startAction) @@ -1954,7 +1982,6 @@ func executionInit(workflowExecution shuffle.WorkflowExecution) error { //log.Printf("Successfully downloaded and built %s", image) } - visited := []string{} executed := []string{} environments := []string{} @@ -3777,7 +3804,7 @@ func checkStandaloneRun() { if !strings.Contains(backendUrl, "http") { log.Printf("[ERROR] Backend URL should start with http:// or https://") return - + } // Format: @@ -3851,7 +3878,7 @@ func checkStandaloneRun() { continue } - // This is to handle reruns of SINGLE actions + // This is to handle reruns of SINGLE actions if result.Action.Category == "rerun" { newResults = append(newResults, result) continue @@ -3905,7 +3932,6 @@ func checkStandaloneRun() { log.Printf("\n\n\n[DEBUG] Finished resetting execution %s. Body: %s. Starting execution.\n\n\n", newresp.Status, string(body)) - } // Initial loop etc From ff2fc39c379224c70135ad79eb3564d5621008e6 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Fri, 25 Apr 2025 08:26:37 +0200 Subject: [PATCH 2/5] default backed update strategy to recreate Signed-off-by: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> --- functions/kubernetes/charts/shuffle/README.md | 2 +- functions/kubernetes/charts/shuffle/values.schema.json | 2 +- functions/kubernetes/charts/shuffle/values.yaml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/README.md b/functions/kubernetes/charts/shuffle/README.md index ff3885d1..45988a4c 100644 --- a/functions/kubernetes/charts/shuffle/README.md +++ b/functions/kubernetes/charts/shuffle/README.md @@ -217,7 +217,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `backend.affinity` | Affinity for backend pods assignment | `{}` | | `backend.nodeSelector` | Node labels for backend pods assignment | `{}` | | `backend.tolerations` | Tolerations for backend pods assignment | `[]` | -| `backend.updateStrategy.type` | backend deployment strategy type | `RollingUpdate` | +| `backend.updateStrategy.type` | backend deployment strategy type | `Recreate` | | `backend.priorityClassName` | backend pods' priorityClassName | `""` | | `backend.topologySpreadConstraints` | Topology Spread Constraints for backend pod assignment spread across your cluster among failure-domains | `[]` | | `backend.schedulerName` | Name of the k8s scheduler (other than default) for backend pods | `""` | diff --git a/functions/kubernetes/charts/shuffle/values.schema.json b/functions/kubernetes/charts/shuffle/values.schema.json index b785b76e..529f484d 100644 --- a/functions/kubernetes/charts/shuffle/values.schema.json +++ b/functions/kubernetes/charts/shuffle/values.schema.json @@ -517,7 +517,7 @@ "type": { "type": "string", "description": "backend deployment strategy type", - "default": "RollingUpdate" + "default": "Recreate" } } }, diff --git a/functions/kubernetes/charts/shuffle/values.yaml b/functions/kubernetes/charts/shuffle/values.yaml index 507c2468..9723e96c 100644 --- a/functions/kubernetes/charts/shuffle/values.yaml +++ b/functions/kubernetes/charts/shuffle/values.yaml @@ -310,14 +310,14 @@ backend: ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ ## tolerations: [] - ## ONLY FOR DEPLOYMENTS: ## @param backend.updateStrategy.type backend deployment strategy type ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy ## updateStrategy: ## Can be set to RollingUpdate or Recreate + ## Backend uses ReadWriteOnce volumes by default, which is incompatible with RollingUpdate ## - type: RollingUpdate + type: Recreate ## @param backend.priorityClassName backend pods' priorityClassName ## priorityClassName: "" From 57287cc54bf2cffd95c5ade3cc9f0d5e58b09864 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Fri, 25 Apr 2025 08:27:12 +0200 Subject: [PATCH 3/5] allow to configure service labels for backend and frontend Signed-off-by: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> --- functions/kubernetes/charts/shuffle/README.md | 2 ++ .../templates/backend/backend-svc.yaml | 3 ++- .../templates/frontend/frontend-svc.yaml | 3 ++- .../charts/shuffle/values.schema.json | 20 +++++++++++++++++++ .../kubernetes/charts/shuffle/values.yaml | 16 +++++++++++++++ 5 files changed, 42 insertions(+), 2 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/README.md b/functions/kubernetes/charts/shuffle/README.md index 45988a4c..f0c9df60 100644 --- a/functions/kubernetes/charts/shuffle/README.md +++ b/functions/kubernetes/charts/shuffle/README.md @@ -244,6 +244,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `backend.autoscaling.hpa.maxReplicas` | Maximum number of replicas | `""` | | `backend.autoscaling.hpa.targetCPU` | Target CPU utilization percentage | `""` | | `backend.autoscaling.hpa.targetMemory` | Target Memory utilization percentage | `""` | +| `backend.service.labels` | Extra labels for backend service | `{}` | | `backend.serviceAccount.create` | Specifies whether a ServiceAccount should be created | `true` | | `backend.serviceAccount.name` | The name of the ServiceAccount to use. | `""` | | `backend.serviceAccount.annotations` | Additional Service Account annotations (evaluated as a template) | `{}` | @@ -359,6 +360,7 @@ The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env varia | `frontend.autoscaling.hpa.maxReplicas` | Maximum number of replicas | `""` | | `frontend.autoscaling.hpa.targetCPU` | Target CPU utilization percentage | `""` | | `frontend.autoscaling.hpa.targetMemory` | Target Memory utilization percentage | `""` | +| `frontend.service.labels` | Extra labels for frontend service | `{}` | | `frontend.serviceAccount.create` | Specifies whether a ServiceAccount should be created | `true` | | `frontend.serviceAccount.name` | The name of the ServiceAccount to use. | `""` | | `frontend.serviceAccount.annotations` | Additional Service Account annotations (evaluated as a template) | `{}` | diff --git a/functions/kubernetes/charts/shuffle/templates/backend/backend-svc.yaml b/functions/kubernetes/charts/shuffle/templates/backend/backend-svc.yaml index 18328899..d85382e3 100644 --- a/functions/kubernetes/charts/shuffle/templates/backend/backend-svc.yaml +++ b/functions/kubernetes/charts/shuffle/templates/backend/backend-svc.yaml @@ -3,7 +3,8 @@ kind: Service metadata: name: {{ template "shuffle.backend.name" . }} namespace: {{ include "common.names.namespace" . | quote }} - labels: {{- include "shuffle.backend.labels" (dict "customLabels" .Values.commonLabels "context" $) | nindent 4 }} + {{- $serviceLabels := include "common.tplvalues.merge" (dict "values" (list .Values.backend.service.labels .Values.commonLabels) "context" .) }} + labels: {{- include "shuffle.backend.labels" (dict "customLabels" $serviceLabels "context" $) | nindent 4 }} {{- if .Values.commonAnnotations }} annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} {{- end }} diff --git a/functions/kubernetes/charts/shuffle/templates/frontend/frontend-svc.yaml b/functions/kubernetes/charts/shuffle/templates/frontend/frontend-svc.yaml index 76851c0a..37b8140d 100644 --- a/functions/kubernetes/charts/shuffle/templates/frontend/frontend-svc.yaml +++ b/functions/kubernetes/charts/shuffle/templates/frontend/frontend-svc.yaml @@ -3,7 +3,8 @@ kind: Service metadata: name: {{ template "shuffle.frontend.name" . }} namespace: {{ include "common.names.namespace" . | quote }} - labels: {{- include "shuffle.frontend.labels" (dict "customLabels" .Values.commonLabels "context" $) | nindent 4 }} + {{- $serviceLabels := include "common.tplvalues.merge" (dict "values" (list .Values.frontend.service.labels .Values.commonLabels) "context" .) }} + labels: {{- include "shuffle.frontend.labels" (dict "customLabels" $serviceLabels "context" $) | nindent 4 }} {{- if .Values.commonAnnotations }} annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} {{- end }} diff --git a/functions/kubernetes/charts/shuffle/values.schema.json b/functions/kubernetes/charts/shuffle/values.schema.json index 529f484d..a46c274d 100644 --- a/functions/kubernetes/charts/shuffle/values.schema.json +++ b/functions/kubernetes/charts/shuffle/values.schema.json @@ -683,6 +683,16 @@ } } }, + "service": { + "type": "object", + "properties": { + "labels": { + "type": "object", + "description": "Extra labels for backend service", + "default": {} + } + } + }, "serviceAccount": { "type": "object", "properties": { @@ -1362,6 +1372,16 @@ } } }, + "service": { + "type": "object", + "properties": { + "labels": { + "type": "object", + "description": "Extra labels for frontend service", + "default": {} + } + } + }, "serviceAccount": { "type": "object", "properties": { diff --git a/functions/kubernetes/charts/shuffle/values.yaml b/functions/kubernetes/charts/shuffle/values.yaml index 9723e96c..6dfdcd97 100644 --- a/functions/kubernetes/charts/shuffle/values.yaml +++ b/functions/kubernetes/charts/shuffle/values.yaml @@ -421,6 +421,14 @@ backend: targetCPU: "" targetMemory: "" + ## Service configuration + ## + service: + ## @param backend.service.labels Extra labels for backend service + ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + ## + labels: {} + ## ServiceAccount configuration ## serviceAccount: @@ -870,6 +878,14 @@ frontend: targetCPU: "" targetMemory: "" + ## Service configuration + ## + service: + ## @param frontend.service.labels Extra labels for frontend service + ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + ## + labels: {} + ## ServiceAccount configuration ## serviceAccount: From 39cfdb84cec8a380f4ca3a071fc32a7e56a2567b Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Fri, 25 Apr 2025 08:27:18 +0200 Subject: [PATCH 4/5] add appProtocol Signed-off-by: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> --- .../charts/shuffle/templates/backend/backend-svc.yaml | 1 + .../charts/shuffle/templates/frontend/frontend-svc.yaml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/functions/kubernetes/charts/shuffle/templates/backend/backend-svc.yaml b/functions/kubernetes/charts/shuffle/templates/backend/backend-svc.yaml index d85382e3..990f9410 100644 --- a/functions/kubernetes/charts/shuffle/templates/backend/backend-svc.yaml +++ b/functions/kubernetes/charts/shuffle/templates/backend/backend-svc.yaml @@ -15,5 +15,6 @@ spec: port: {{ .Values.backend.containerPorts.http }} targetPort: http protocol: TCP + appProtocol: http {{- $podLabels := include "common.tplvalues.merge" (dict "values" (list .Values.backend.podLabels .Values.commonLabels) "context" .) }} selector: {{- include "shuffle.backend.matchLabels" (dict "customLabels" $podLabels "context" $) | nindent 4 }} diff --git a/functions/kubernetes/charts/shuffle/templates/frontend/frontend-svc.yaml b/functions/kubernetes/charts/shuffle/templates/frontend/frontend-svc.yaml index 37b8140d..de2db9fb 100644 --- a/functions/kubernetes/charts/shuffle/templates/frontend/frontend-svc.yaml +++ b/functions/kubernetes/charts/shuffle/templates/frontend/frontend-svc.yaml @@ -15,11 +15,13 @@ spec: port: {{ .Values.frontend.containerPorts.http }} targetPort: http protocol: TCP + appProtocol: http {{- if .Values.frontend.containerPorts.https }} - name: https port: {{ .Values.frontend.containerPorts.https }} targetPort: https protocol: TCP + appProtocol: https {{- end }} {{- $podLabels := include "common.tplvalues.merge" (dict "values" (list .Values.frontend.podLabels .Values.commonLabels) "context" .) }} selector: {{- include "shuffle.frontend.matchLabels" (dict "customLabels" $podLabels "context" $) | nindent 4 }} From e9d934a6d97f56f69d7e182f5ee7efa435bd3442 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Wed, 14 May 2025 15:10:11 +0200 Subject: [PATCH 5/5] helm: set backend_url for workflow executions to cluster-internal address Signed-off-by: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> --- .../shuffle/templates/backend/backend-cm-env.yaml | 6 ++++-- .../shuffle/templates/orborus/orborus-cm-env.yaml | 2 +- .../shuffle-app/shuffle-app-network-policy.yaml | 10 ++++++++++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/functions/kubernetes/charts/shuffle/templates/backend/backend-cm-env.yaml b/functions/kubernetes/charts/shuffle/templates/backend/backend-cm-env.yaml index e7138dc0..dbbdff94 100644 --- a/functions/kubernetes/charts/shuffle/templates/backend/backend-cm-env.yaml +++ b/functions/kubernetes/charts/shuffle/templates/backend/backend-cm-env.yaml @@ -8,18 +8,20 @@ metadata: annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} {{- end }} data: - BACKEND_PORT: "5001" + BACKEND_PORT: "{{ .Values.backend.containerPorts.http }}" {{- if .Values.shuffle.baseUrl }} BASE_URL: "{{ .Values.shuffle.baseUrl }}" SSO_REDIRECT_URL: "{{ .Values.shuffle.baseUrl }}" {{- else }} - BASE_URL: "http://{{ include "shuffle.backend.name" . }}:5001" + BASE_URL: "http://{{ include "shuffle.backend.name" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.backend.containerPorts.http }}" {{- end }} ORG_ID: "{{ .Values.shuffle.org }}" SHUFFLE_APP_DOWNLOAD_LOCATION: "{{ .Values.backend.apps.downloadLocation }}" SHUFFLE_DOWNLOAD_AUTH_BRANCH: "{{ .Values.backend.apps.downloadBranch }}" SHUFFLE_APP_FORCE_UPDATE: "{{ .Values.backend.apps.forceUpdate }}" SHUFFLE_CHAT_DISABLED: "true" + # Sets backend_url parameter for workflow execution to the cluster-internal shuffle-backend address + SHUFFLE_CLOUDRUN_URL: "http://{{ include "shuffle.backend.name" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.backend.containerPorts.http }}" SHUFFLE_OPENSEARCH_URL: {{ include "common.tplvalues.render" (dict "value" .Values.backend.openSearch.url "context" $) }} SHUFFLE_OPENSEARCH_USERNAME: "{{ .Values.backend.openSearch.username }}" SHUFFLE_OPENSEARCH_CERTIFICATE_FILE: "{{ .Values.backend.openSearch.certificateFile }}" diff --git a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-cm-env.yaml b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-cm-env.yaml index 57b020a7..504b674d 100644 --- a/functions/kubernetes/charts/shuffle/templates/orborus/orborus-cm-env.yaml +++ b/functions/kubernetes/charts/shuffle/templates/orborus/orborus-cm-env.yaml @@ -11,7 +11,7 @@ data: ENVIRONMENT_NAME: "{{ .Values.shuffle.org }}" ORG_ID: "{{ .Values.shuffle.org }}" TZ: "{{ .Values.shuffle.timezone }}" - BASE_URL: "http://{{ include "shuffle.backend.name" . }}.{{ .Release.Namespace }}.svc.cluster.local:5001" + BASE_URL: "http://{{ include "shuffle.backend.name" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.backend.containerPorts.http }}" KUBERNETES_NAMESPACE: "{{ .Release.Namespace }}" KUBERNETES_SERVICE_ACCOUNT: {{ include "shuffle.orborus.serviceAccount.name" . }} SHUFFLE_WORKER_IMAGE: "{{ include "shuffle.worker.image" . }}" diff --git a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-network-policy.yaml b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-network-policy.yaml index d4a24fe6..5303123d 100644 --- a/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-network-policy.yaml +++ b/functions/kubernetes/charts/shuffle/templates/shuffle-app/shuffle-app-network-policy.yaml @@ -29,6 +29,16 @@ spec: - namespaceSelector: matchLabels: kubernetes.io/metadata.name: kube-system + # Allow access to backend + - ports: + - port: {{ .Values.backend.containerPorts.http }} + protocol: TCP + to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Release.Namespace }} + podSelector: + matchLabels: {{ include "shuffle.backend.matchLabels" . | nindent 14 }} # Allow access to workers - ports: - port: 33333