Merge pull request #1846 from PROCYDE/app-tmp-volume

feat(k8s): allow to mount tmp volume to apps

functions/onprem/orborus/orborus.go

@@ -1146,6 +1146,10 @@ func deployK8sWorker(image string, identifier string, env []string) error {
 		env = append(env, fmt.Sprintf("SHUFFLE_APP_CONTAINER_SECURITY_CONTEXT=%s", appContainerSecurityContext))
 	}
 
+	if len(os.Getenv("SHUFFLE_APP_MOUNT_TMP_VOLUME")) > 0 {
+		env = append(env, fmt.Sprintf("SHUFFLE_APP_MOUNT_TMP_VOLUME=%s", os.Getenv("SHUFFLE_APP_MOUNT_TMP_VOLUME")))
+	}
+
 	if len(os.Getenv("SHUFFLE_LOGS_DISABLED")) > 0 {
 		env = append(env, fmt.Sprintf("SHUFFLE_LOGS_DISABLED=%s", os.Getenv("SHUFFLE_LOGS_DISABLED")))
 	}
@@ -3049,8 +3053,8 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri
 		Healthcheck: healthconfig,
 		ExposedPorts: nat.PortSet{
 			"5160/tcp": struct{}{},
-			"1514/udp":  struct{}{},
+			"1514/udp": struct{}{},
-			"1514/tcp":  struct{}{},
+			"1514/tcp": struct{}{},
 		},
 		Entrypoint: []string{containerName},
 		Env:        []string{},
@@ -3096,8 +3100,8 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri
 
 	hostConfig := &container.HostConfig{
 		PortBindings: nat.PortMap{
-			"1514/tcp":  []nat.PortBinding{{HostPort: "1514"}},
+			"1514/tcp": []nat.PortBinding{{HostPort: "1514"}},
-			"1514/udp":  []nat.PortBinding{{HostPort: "1514"}},
+			"1514/udp": []nat.PortBinding{{HostPort: "1514"}},
 			"5160/tcp": []nat.PortBinding{{HostPort: "5160"}},
 		},
 		Mounts: []mount.Mount{
@@ -3107,16 +3111,16 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri
 				Target: "/tmp",
 			},
 			/*
-			{
+				{
-				Type:   "bind",
+					Type:   "bind",
-				Source: tenzirStorageFolder,
+					Source: tenzirStorageFolder,
-				Target: "/var/log/tenzir/",
+					Target: "/var/log/tenzir/",
-			},
+				},
-			{
+				{
-				Type:   "bind",
+					Type:   "bind",
-				Source: tenzirStorageFolder,
+					Source: tenzirStorageFolder,
-				Target: "/var/cache/tenzir/",
+					Target: "/var/cache/tenzir/",
-			},
+				},
 			*/
 		},
 		VolumeDriver: "local",
@@ -3334,7 +3338,7 @@ func createPipeline(command, identifier string) (string, error) {
 		"name":        identifier,
 		"hidden":      false,
 		"retry_delay": "500.0ms",
-		"unstoppable":  true,
+		"unstoppable": true,
 	}
 
 	requestBodyJSON, err := json.Marshal(requestBody)
@@ -3406,20 +3410,20 @@ func updatePipelineState(command, pipelineId, action string) (string, error) {
 	url := fmt.Sprintf("%s/api/v0/pipeline/update", pipelineUrl)
 	forwardMethod := "POST"
 	requestBody := map[string]interface{}{
-		"id":         pipelineId,
+		"id":     pipelineId,
-		"action":     action,
+		"action": action,
 
 		/*
-		"autostart": map[string]bool{
+			"autostart": map[string]bool{
-			"created":   true,
+				"created":   true,
-			"completed": false,
+				"completed": false,
-			"failed":    false,
+				"failed":    false,
-		},
+			},
-		"autodelete": map[string]bool{
+			"autodelete": map[string]bool{
-			"completed": false,
+				"completed": false,
-			"failed":    false,
+				"failed":    false,
-			"stopped":   false,
+				"stopped":   false,
-		},
+			},
 		*/
 	}
 

functions/onprem/worker/worker.go

@@ -2,7 +2,7 @@ package main
 
 import (
 	"github.com/shuffle/shuffle-shared"
-	"github.com/shuffle/singul/pkg"
+	singul "github.com/shuffle/singul/pkg"
 
 	"bytes"
 	"context"
@@ -664,6 +664,27 @@ func deployk8sApp(image string, identifier string, env []string) error {
 		},
 	}
 
+	if os.Getenv("SHUFFLE_APP_MOUNT_TMP_VOLUME") == "true" {
+		deployment.Spec.Template.Spec.Volumes = append(
+			deployment.Spec.Template.Spec.Volumes,
+			corev1.Volume{
+				Name: "tmp",
+				VolumeSource: corev1.VolumeSource{
+					EmptyDir: &corev1.EmptyDirVolumeSource{},
+				},
+			},
+		)
+
+		deployment.Spec.Template.Spec.Containers[0].VolumeMounts = append(
+			deployment.Spec.Template.Spec.Containers[0].VolumeMounts,
+			corev1.VolumeMount{
+				Name:      "tmp",
+				ReadOnly:  false,
+				MountPath: "/tmp",
+			},
+		)
+	}
+
 	if len(os.Getenv("REGISTRY_URL")) > 0 && len(os.Getenv("SHUFFLE_BASE_IMAGE_NAME")) > 0 {
 		log.Printf("[INFO] Setting image pull policy to Always as private registry is used.")
 		//containerAttachment.ImagePullPolicy = corev1.PullAlways
@@ -3233,8 +3254,6 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl
 	log.Printf("[DEBUG] Deploying service for %s to swarm on port %d", name, deployport)
 	//containerName := fmt.Sprintf("shuffle-worker-%s", parsedUuid)
 
-
-
 	// Check if the image exists or not - just in case
 	_, _, err := dockercli.ImageInspectWithRaw(context.Background(), image)
 	if err != nil {
@@ -3257,7 +3276,6 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl
 		}
 	}
 
-
 	if len(baseimagename) == 0 || baseimagename == "/" {
 		baseimagename = "frikky/shuffle"
 		//var baseimagename = "frikky/shuffle"
@@ -3310,7 +3328,6 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl
 		nodeCount = 1
 	}
 
-
 	replicatedJobs := uint64(replicas * nodeCount)
 	log.Printf("[DEBUG] Deploying app with name %s with image %s", name, image)
 
@@ -3472,7 +3489,6 @@ func findAppInfo(image, name string, redeploy bool) (int, error) {
 	// chance of being successful
 	time.Sleep(time.Duration(rand.Intn(1500)) * time.Millisecond)
 
-
 	highest := baseport
 	exposedPort := -1
 
@@ -3695,7 +3711,6 @@ func initSwarmNetwork() error {
 	mtu := 1500
 	options["com.docker.network.driver.mtu"] = fmt.Sprintf("%d", mtu)
 
-
 	ingressOptions := network.CreateOptions{
 		Driver:     "overlay",
 		Attachable: false,
@@ -3782,8 +3797,6 @@ func initSwarmNetwork() error {
 	return nil
 }
 
-
-
 /*** ENDREMOVE ***/
 
 func sendAppRequest(ctx context.Context, incomingUrl, appName string, port int, action *shuffle.Action, workflowExecution *shuffle.WorkflowExecution, image string, attempts int64) error {
@@ -4835,7 +4848,7 @@ func runWebserver(listener net.Listener) {
 	if strings.ToLower(os.Getenv("SHUFFLE_SWARM_CONFIG")) == "run" || strings.ToLower(os.Getenv("SHUFFLE_APP_REPLICAS")) == "" {
 		// go AutoScaleApps(ctx, dockercli, maxExecutionsPerMinute)
 	}
-	if (strings.ToLower(os.Getenv("SHUFFLE_DEBUG_MEMORY")) == "true" || strings.ToLower(os.Getenv("DEBUG_MEMORY")) == "true") {
+	if strings.ToLower(os.Getenv("SHUFFLE_DEBUG_MEMORY")) == "true" || strings.ToLower(os.Getenv("DEBUG_MEMORY")) == "true" {
 		r.HandleFunc("/debug/pprof/", pprof.Index)
 		r.HandleFunc("/debug/pprof/heap", pprof.Handler("heap").ServeHTTP)
 		r.HandleFunc("/debug/pprof/profile", pprof.Profile)