From e554943c8d518b87391ac4308629de5247ac4fe5 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Tue, 18 Feb 2025 16:22:48 +0100 Subject: [PATCH 1/4] add common kubernetes labels Signed-off-by: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> --- functions/onprem/orborus/orborus.go | 25 +++++++++++++++------ functions/onprem/worker/worker.go | 35 ++++++++++++++++++----------- 2 files changed, 40 insertions(+), 20 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 093faf79..894f6594 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -1078,8 +1078,17 @@ func deployK8sWorker(image string, identifier string, env []string) error { } } - containerLabels := map[string]string{ - "container": "shuffle-worker", + labels := map[string]string{ + "app.kubernetes.io/name": "shuffle-worker", + "app.kubernetes.io/instance": identifier, + // "app.kubernetes.io/version": "", + "app.kuvernetes.io/part-of": "shuffle", + "app.kubernetes.io/managed-by": "shuffle-orborus", + } + + matchLabels := map[string]string{ + "app.kubernetes.io/name": "shuffle-worker", + "app.kubernetes.io/instance": identifier, } containerAttachment := corev1.Container{ @@ -1184,16 +1193,17 @@ func deployK8sWorker(image string, identifier string, env []string) error { deployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: identifier, + Name: identifier, + Labels: labels, }, Spec: appsv1.DeploymentSpec{ Replicas: int32Ptr(replicaNumberInt32), Selector: &metav1.LabelSelector{ - MatchLabels: containerLabels, + MatchLabels: matchLabels, }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: containerLabels, + Labels: labels, }, Spec: corev1.PodSpec{ Containers: []corev1.Container{ @@ -1214,10 +1224,11 @@ func deployK8sWorker(image string, identifier string, env []string) error { // kubectl expose deployment shuffle-workers --type=NodePort --port=33333 --target-port=33333 service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: identifier, + Name: identifier, + Labels: labels, }, Spec: corev1.ServiceSpec{ - Selector: containerLabels, + Selector: matchLabels, Ports: []corev1.ServicePort{ { Protocol: "TCP", diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 996764fd..064301b9 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -487,9 +487,22 @@ func deployk8sApp(image string, identifier string, env []string) error { //fix naming convention // podUuid := uuid.NewV4().String() - // podName := fmt.Sprintf("%s-%s", value, podUuid) + // name := fmt.Sprintf("%s-%s", value, podUuid) // replace identifier "_" with "-" - podName := strings.ReplaceAll(identifier, "_", "-") + name := strings.ReplaceAll(identifier, "_", "-") + + labels := map[string]string{ + "app.kubernetes.io/name": "shuffle-app", + "app.kubernetes.io/instance": name, + // "app.kubernetes.io/version": "", + "app.kuvernetes.io/part-of": "shuffle", + "app.kubernetes.io/managed-by": "shuffle-worker", + } + + matchLabels := map[string]string{ + "app.kubernetes.io/name": "shuffle-app", + "app.kubernetes.io/instance": name, + } // pod := &corev1.Pod{ // ObjectMeta: metav1.ObjectMeta{ @@ -569,20 +582,17 @@ func deployk8sApp(image string, identifier string, env []string) error { deployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: podName, + Name: name, + Labels: labels, }, Spec: appsv1.DeploymentSpec{ Replicas: int32Ptr(replicaNumberInt32), Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": podName, - }, + MatchLabels: matchLabels, }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": podName, - }, + Labels: labels, }, Spec: corev1.PodSpec{ Containers: []corev1.Container{ @@ -606,12 +616,11 @@ func deployk8sApp(image string, identifier string, env []string) error { // kubectl expose deployment {podName} --type=NodePort --port=80 --target-port=80 service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: podName, + Name: name, + Labels: labels, }, Spec: corev1.ServiceSpec{ - Selector: map[string]string{ - "app": podName, - }, + Selector: matchLabels, Ports: []corev1.ServicePort{ { Protocol: "TCP", From ead70439be8464356e78cff8c8da1cf2513e59b1 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Tue, 18 Feb 2025 16:32:36 +0100 Subject: [PATCH 2/4] fix typo, format file Signed-off-by: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> --- functions/onprem/orborus/orborus.go | 30 ++++++++++++++--------------- functions/onprem/worker/worker.go | 14 +++++++------- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 894f6594..bd551a99 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -1082,7 +1082,7 @@ func deployK8sWorker(image string, identifier string, env []string) error { "app.kubernetes.io/name": "shuffle-worker", "app.kubernetes.io/instance": identifier, // "app.kubernetes.io/version": "", - "app.kuvernetes.io/part-of": "shuffle", + "app.kubernetes.io/part-of": "shuffle", "app.kubernetes.io/managed-by": "shuffle-orborus", } @@ -2237,7 +2237,7 @@ func main() { if incRequest.Type == "PIPELINE_CREATE" || incRequest.Type == "PIPELINE_START" || incRequest.Type == "PIPELINE_STOP" || incRequest.Type == "PIPELINE_DELETE" { os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") - tenzirDisabled = false + tenzirDisabled = false // Running NEW or editing pipelines err := handlePipeline(incRequest) @@ -2262,7 +2262,7 @@ func main() { } else if incRequest.Type == "CATEGORY_UPDATE" { os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") - tenzirDisabled = false + tenzirDisabled = false err = handleFileCategoryChange() if err != nil { @@ -2307,7 +2307,7 @@ func main() { // Manual command = overrides to allow starting of Tenzir from the frontend anyway. os.Setenv("SHUFFLE_SKIP_PIPELINES", "false") - tenzirDisabled = false + tenzirDisabled = false // Removed either way toBeRemoved.Data = append(toBeRemoved.Data, incRequest) @@ -2317,7 +2317,7 @@ func main() { if strings.Contains(fmt.Sprintf("%s", err), "node available") { // Disabling until UI is updated os.Setenv("SHUFFLE_SKIP_PIPELINES", "true") - tenzirDisabled = true + tenzirDisabled = true log.Printf("[ERROR] Failed to start tenzir, reason: %s", err) err = shuffle.CreateOrgNotification( @@ -2636,7 +2636,7 @@ func deployTenzirNode() error { // return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES") //log.Printf("[INFO] Pipelines are enabled by user") } else { - return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES") + return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES") } if isKubernetes == "true" { @@ -2836,12 +2836,12 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri }, } - // FIXME: Is this necessary? Seems to screw up networking: + // FIXME: Is this necessary? Seems to screw up networking: // conflicting options: hostname and the network mode /* - if isKubernetes != "true" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" { - hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) - } + if isKubernetes != "true" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" { + hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) + } */ resp, err := dockercli.ContainerCreate(ctx, config, hostConfig, networkingConfig, nil, containerName) @@ -3049,7 +3049,6 @@ func createPipeline(command, identifier string) (string, error) { return "", err } - body, err := ioutil.ReadAll(resp.Body) if err != nil { log.Printf("[ERROR] Failed reading response body: %s", err) @@ -3067,8 +3066,8 @@ func createPipeline(command, identifier string) (string, error) { } type PipelineResponse struct { - ID string `json:"id"` - Message string `json:"message"` + ID string `json:"id"` + Message string `json:"message"` Severity string `json:"severity"` } @@ -3291,7 +3290,7 @@ func handleFileCategoryChange() error { tenzirStorageFolder = "/tmp/" } - tenzirStorageFolder = strings.TrimRight(tenzirStorageFolder, "/") + tenzirStorageFolder = strings.TrimRight(tenzirStorageFolder, "/") sigmaPath := fmt.Sprintf("%s/sigma_rules", tenzirStorageFolder) err = extractZIP("files.zip", sigmaPath) if err != nil { @@ -3299,7 +3298,6 @@ func handleFileCategoryChange() error { return err } - log.Printf("[DEBUG] Detection files copied to '%s' successfully.", sigmaPath) return nil @@ -3383,7 +3381,7 @@ func removeFileCategory() error { tenzirStorageFolder = "/tmp/" } - tenzirStorageFolder = strings.TrimRight(tenzirStorageFolder, "/") + tenzirStorageFolder = strings.TrimRight(tenzirStorageFolder, "/") //sigmaPath := "/var/lib/tenzir/sigma_rules/*" sigmaPath := fmt.Sprintf("%s/sigma_rules", tenzirStorageFolder) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 064301b9..6a580229 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -23,8 +23,8 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/filters" - "github.com/docker/docker/api/types/mount" "github.com/docker/docker/api/types/image" + "github.com/docker/docker/api/types/mount" dockerclient "github.com/docker/docker/client" // This is for automatic removal of certain code :) @@ -105,11 +105,11 @@ var window = shuffle.NewTimeWindow(10 * time.Second) // Images to be autodeployed in the latest version of Shuffle. var autoDeploy = map[string]string{ - "http:1.4.0": "frikky/shuffle:http_1.4.0", - "http:1.3.0": "frikky/shuffle:http_1.3.0", - "shuffle-tools:1.2.0": "frikky/shuffle:shuffle-tools_1.2.0", - "shuffle-subflow:1.0.0": "frikky/shuffle:shuffle-subflow_1.0.0", - "shuffle-subflow:1.1.0": "frikky/shuffle:shuffle-subflow_1.1.0", + "http:1.4.0": "frikky/shuffle:http_1.4.0", + "http:1.3.0": "frikky/shuffle:http_1.3.0", + "shuffle-tools:1.2.0": "frikky/shuffle:shuffle-tools_1.2.0", + "shuffle-subflow:1.0.0": "frikky/shuffle:shuffle-subflow_1.0.0", + "shuffle-subflow:1.1.0": "frikky/shuffle:shuffle-subflow_1.1.0", // "shuffle-tools-fork:1.0.0": "frikky/shuffle:shuffle-tools-fork_1.0.0", } @@ -495,7 +495,7 @@ func deployk8sApp(image string, identifier string, env []string) error { "app.kubernetes.io/name": "shuffle-app", "app.kubernetes.io/instance": name, // "app.kubernetes.io/version": "", - "app.kuvernetes.io/part-of": "shuffle", + "app.kubernetes.io/part-of": "shuffle", "app.kubernetes.io/managed-by": "shuffle-worker", } From 80aa1a3fd2a1b30b60a62fab9b80088736be2d91 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Wed, 19 Feb 2025 11:30:40 +0100 Subject: [PATCH 3/4] update cleanup and label selectors Signed-off-by: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> --- functions/onprem/orborus/orborus.go | 44 ++++++----------------------- functions/onprem/worker/worker.go | 3 +- 2 files changed, 11 insertions(+), 36 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index bd551a99..15baf03c 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -216,8 +216,7 @@ func skipCheckInCleanup(name string) bool { func cleanupExistingNodes(ctx context.Context) error { if isKubernetes == "true" { - // of course, this doesn't clean up "nodes" but - // rather pods, services, roles etc. + // Cleanup all workers created by orborus and all apps created by workers. if kubernetesNamespace == "" { kubernetesNamespace = "default" @@ -229,62 +228,38 @@ func cleanupExistingNodes(ctx context.Context) error { return err } - // Delete all pods - pods, err := clientset.CoreV1().Pods(kubernetesNamespace).List(context.Background(), metav1.ListOptions{}) - if err != nil { - log.Printf("[ERROR] Failed listing pods: %s", err) - return err - } - - for _, pod := range pods.Items { - // check if pod.Name starts with: - // "backend-", "frontend-", "orborus-", "opensearch-" or "memcached-" - if skipCheckInCleanup(pod.Name) { - continue - } - - err := clientset.CoreV1().Pods(kubernetesNamespace).Delete(context.Background(), pod.Name, metav1.DeleteOptions{}) - if err != nil { - log.Printf("[ERROR] Failed deleting pod %s: %s", pod.Name, err) - } - } - // Delete all services - services, err := clientset.CoreV1().Services(kubernetesNamespace).List(context.Background(), metav1.ListOptions{}) + services, err := clientset.CoreV1().Services(kubernetesNamespace).List(context.Background(), metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name in (shuffle-worker, shuffle-app),app.kubernetes.io/managed-by in (shuffle-orborus, shuffle-worker)", + }) if err != nil { log.Printf("[ERROR] Failed listing services: %s", err) return err } for _, service := range services.Items { - if skipCheckInCleanup(service.Name) { - continue - } - err := clientset.CoreV1().Services(kubernetesNamespace).Delete(context.Background(), service.Name, metav1.DeleteOptions{}) if err != nil { log.Printf("[ERROR] Failed deleting service %s: %s", service.Name, err) } } - deployments, err := clientset.AppsV1().Deployments(kubernetesNamespace).List(context.Background(), metav1.ListOptions{}) + deployments, err := clientset.AppsV1().Deployments(kubernetesNamespace).List(context.Background(), metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name in (shuffle-worker, shuffle-app),app.kubernetes.io/managed-by in (shuffle-orborus, shuffle-worker)", + }) if err != nil { log.Printf("[ERROR] Failed listing deployments: %s", err) return err } for _, deployment := range deployments.Items { - if skipCheckInCleanup(deployment.Name) { - continue - } - err := clientset.AppsV1().Deployments(kubernetesNamespace).Delete(context.Background(), deployment.Name, metav1.DeleteOptions{}) if err != nil { log.Printf("[ERROR] Failed deleting deployment %s: %s", deployment.Name, err) } } - log.Printf("[INFO] Cleaned up all pods and services in namespace %s. Waiting 10 seconds for cleanup to reflect", kubernetesNamespace) + log.Printf("[INFO] Cleaned up all services and deployments in namespace %s. Waiting 10 seconds for cleanup to reflect", kubernetesNamespace) time.Sleep(10 * time.Second) @@ -3546,9 +3521,8 @@ func getRunningWorkers(ctx context.Context, workerTimeout int) int { return 0 } - labelSelector := "app=shuffle-worker" pods, podErr := clientset.CoreV1().Pods(kubernetesNamespace).List(ctx, metav1.ListOptions{ - LabelSelector: labelSelector, + LabelSelector: "app.kubernetes.io/name=shuffle-worker", }) if podErr != nil { log.Printf("[ERROR] Failed getting running workers: %s", podErr) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 6a580229..9e319796 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -966,7 +966,8 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] func cleanupKubernetesExecution(clientset *kubernetes.Clientset, workflowExecution shuffle.WorkflowExecution, namespace string) error { // workerName := fmt.Sprintf("worker-%s", workflowExecution.ExecutionId) - labelSelector := fmt.Sprintf("app=shuffle-app,executionId=%s", workflowExecution.ExecutionId) + // FIXME: The executionId label is currently not set + labelSelector := fmt.Sprintf("app.kubernetes.io/name=shuffle-app,executionId=%s", workflowExecution.ExecutionId) podList, err := clientset.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{ LabelSelector: labelSelector, From e769418ecf73d77681cfb1a548e03158c9d4da51 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Wed, 26 Feb 2025 09:04:53 +0100 Subject: [PATCH 4/4] add legacy labels Signed-off-by: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> --- functions/onprem/orborus/orborus.go | 2 ++ functions/onprem/worker/worker.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 15baf03c..3f07f60e 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -1059,6 +1059,8 @@ func deployK8sWorker(image string, identifier string, env []string) error { // "app.kubernetes.io/version": "", "app.kubernetes.io/part-of": "shuffle", "app.kubernetes.io/managed-by": "shuffle-orborus", + // Keep legacy labels for backward compatibility + "container": "shuffle-worker", } matchLabels := map[string]string{ diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 9e319796..ed376797 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -497,6 +497,8 @@ func deployk8sApp(image string, identifier string, env []string) error { // "app.kubernetes.io/version": "", "app.kubernetes.io/part-of": "shuffle", "app.kubernetes.io/managed-by": "shuffle-worker", + // Keep legacy labels for backward compatibility + "app": name, } matchLabels := map[string]string{