Fixed issue with Orborus and Worker network

functions/onprem/orborus/orborus.go

@@ -64,6 +64,7 @@ var cleanupEnv = strings.ToLower(os.Getenv("CLEANUP"))
 var timezone = os.Getenv("TZ")
 var containerName = os.Getenv("ORBORUS_CONTAINER_NAME")
 var swarmConfig = os.Getenv("SHUFFLE_SWARM_CONFIG")
+var swarmNetworkName = os.Getenv("SHUFFLE_SWARM_NETWORK_NAME")
 var executionIds = []string{}
 
 var dockercli *dockerclient.Client
@@ -141,6 +142,9 @@ func deployServiceWorkers(image string) {
 	if swarmConfig == "run" {
 		// frikky@debian:~/git/shuffle/functions/onprem/worker$ docker service create --replicas 5 --name shuffle-workers --env SHUFFLE_SWARM_CONFIG=run --publish published=33333,target=33333 ghcr.io/frikky/shuffle-worker:nightly
 		networkName := "shuffle-executions"
+		if len(swarmNetworkName) > 0 {
+			networkName = swarmNetworkName
+		}
 		ctx := context.Background()
 
 		//docker network create --driver=overlay workers
@@ -374,7 +378,7 @@ func deployWorker(image string, identifier string, env []string, executionReques
 			log.Printf("[ERROR] Failed to start container in environment %s: %s", environment, err)
 			return err
 		} else {
-			log.Printf("[INFO] Container %s was created under environment %s", cont.ID, environment)
+			log.Printf("[INFO] Container %s was created under environment %s for execution %s", cont.ID, environment, executionRequest.ExecutionId)
 		}
 
 		//stats, err := cli.ContainerInspect(context.Background(), containerName)
@@ -469,7 +473,7 @@ func initializeImages() {
 
 	pullOptions := types.ImagePullOptions{}
 	for _, image := range images {
-		log.Printf("[INFO] Pulling image %s", image)
+		log.Printf("[DEBUG] Pulling image %s", image)
 		reader, err := dockercli.ImagePull(ctx, image, pullOptions)
 		if err != nil {
 			log.Printf("[ERROR] Failed getting image %s: %s", image, err)
@@ -477,7 +481,7 @@ func initializeImages() {
 		}
 
 		io.Copy(os.Stdout, reader)
-		log.Printf("[INFO] Successfully downloaded and built %s", image)
+		log.Printf("[DEBUG] Successfully downloaded and built %s", image)
 	}
 }
 
@@ -772,7 +776,7 @@ func main() {
 			err = deployWorker(workerImage, containerName, env, execution)
 			zombiecounter += 1
 			if err == nil {
-				log.Printf("[INFO] ExecutionID %s was deployed and to be removed from queue.", execution.ExecutionId)
+				//log.Printf("[DEBUG] ExecutionID %s was deployed and to be removed from queue.", execution.ExecutionId)
 				toBeRemoved.Data = append(toBeRemoved.Data, execution)
 				executionIds = append(executionIds, execution.ExecutionId)
 			} else {

functions/onprem/worker/worker.go

@@ -45,6 +45,7 @@ var baseUrl = os.Getenv("BASE_URL")
 var appCallbackUrl = os.Getenv("BASE_URL")
 var cleanupEnv = strings.ToLower(os.Getenv("CLEANUP"))
 var dockerApiVersion = strings.ToLower(os.Getenv("DOCKER_API_VERSION"))
+var swarmNetworkName = os.Getenv("SHUFFLE_SWARM_NETWORK_NAME")
 var timezone = os.Getenv("TZ")
 var baseimagename = "frikky/shuffle"
 var registryName = "registry.hub.docker.com"
@@ -69,6 +70,7 @@ var startAction string
 var results []shuffle.ActionResult
 var allLogs map[string]string
 var containerIds []string
+var downloadedImages []string
 
 var executionRunning bool
 
@@ -228,6 +230,12 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env []
 		appName = strings.ToLower(appName)
 		log.Printf("[INFO] New appname: %s, image: %s", appName, image)
 
+		if !shuffle.ArrayContains(downloadedImages, image) {
+			log.Printf("[DEBUG] Downloading image %s from backend as it's first iteration", image)
+			downloadDockerImageBackend(&http.Client{}, image)
+			downloadedImages = append(downloadedImages, image)
+		}
+
 		exposedPort, err := findAppInfo(image, appName)
 		if err != nil {
 			log.Printf("[ERROR] Failed finding and creating port for %s: %s", appName, err)
@@ -338,9 +346,40 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env []
 
 	err = cli.ContainerStart(ctx, cont.ID, types.ContainerStartOptions{})
 	if err != nil {
-		log.Printf("[ERROR] Failed to start container in environment %s: %s", environment, err)
+		if strings.Contains(fmt.Sprintf("%s", err), "cannot join network") {
-		//shutdown(workflowExecution, workflowExecution.Workflow.ID, true)
+			parsedUuid := uuid.NewV4()
-		return err
+			identifier = fmt.Sprintf("%s-%s-nonetwork", identifier, parsedUuid)
+			hostConfig = &container.HostConfig{
+				LogConfig: container.LogConfig{
+					Type:   "json-file",
+					Config: map[string]string{},
+				},
+				Resources: container.Resources{},
+			}
+
+			cont, err = cli.ContainerCreate(
+				context.Background(),
+				config,
+				hostConfig,
+				nil,
+				nil,
+				identifier,
+			)
+
+			if err != nil {
+				log.Printf("[ERROR] Container create error (3): %s", err)
+				return err
+			}
+
+			log.Printf("[DEBUG] Running secondary check without network with worker")
+			err = cli.ContainerStart(ctx, cont.ID, types.ContainerStartOptions{})
+		}
+
+		if err != nil {
+			log.Printf("[ERROR] Failed to start container in environment %s: %s", environment, err)
+			//shutdown(workflowExecution, workflowExecution.Workflow.ID, true)
+			return err
+		}
 	}
 
 	log.Printf("[INFO] Container %s was created for %s", cont.ID, identifier)
@@ -375,9 +414,12 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env []
 				// FIXME: Re-add log tracking which can be sent to backend
 				//allLogs[actionId] = logs
 
-				if stats.ContainerJSONBase.State.Status == "exited" && !strings.Contains(logs, "Normal execution.") {
+				if stats.ContainerJSONBase.State.Status == "exited" && (!strings.Contains(logs, "Normal execution.") && !strings.Contains(logs, "indicates microservices")) {
-					log.Printf("[WARNING] BAD Execution Logs for %s: %s", action.ID, logs)
+
-					exit = true
+					if len(logs) > 10 {
+						log.Printf("[ERROR] BAD Execution Logs for %s: %s", action.ID, logs)
+						exit = true
+					}
 				}
 			}
 
@@ -2175,6 +2217,11 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl
 	}
 
 	//image := fmt.Sprintf("%s:%s", baseimagename, name)
+	networkName := "shuffle-executions"
+	if len(swarmNetworkName) > 0 {
+		networkName = swarmNetworkName
+	}
+
 	log.Printf("[DEBUG] Deploying app with name %s with image %s", name, image)
 	containerName := fmt.Sprintf(strings.Replace(name, ".", "-", -1))
 	serviceSpec := swarm.ServiceSpec{
@@ -2184,7 +2231,7 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl
 		},
 		Networks: []swarm.NetworkAttachmentConfig{
 			swarm.NetworkAttachmentConfig{
-				Target: "shuffle-executions",
+				Target: networkName,
 			},
 		},
 		EndpointSpec: &swarm.EndpointSpec{
@@ -2505,6 +2552,8 @@ func main() {
 	log.Printf("[INFO] Running with timezone %s and swarm config %#v", timezone, os.Getenv("SHUFFLE_SWARM_CONFIG"))
 	if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" {
 		workflowExecution := shuffle.WorkflowExecution{}
+
+		// Forcing download just in case on the first iteration.
 		listener := webserverSetup(workflowExecution)
 		//err := executionInit(workflowExecution)
 		//if err != nil {
@@ -2523,7 +2572,6 @@ func main() {
 	}
 
 	//imageName := fmt.Sprintf("%s/%s:shuffle_openapi_1.0.0", registryName, baseimagename)
-	//downloadDockerImageBackend(client, imageName)
 
 	// WORKER_TESTING_WORKFLOW should be a workflow ID
 	authorization := ""