diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 7f04a360..e5ac9a6f 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -64,6 +64,7 @@ var cleanupEnv = strings.ToLower(os.Getenv("CLEANUP")) var timezone = os.Getenv("TZ") var containerName = os.Getenv("ORBORUS_CONTAINER_NAME") var swarmConfig = os.Getenv("SHUFFLE_SWARM_CONFIG") +var swarmNetworkName = os.Getenv("SHUFFLE_SWARM_NETWORK_NAME") var executionIds = []string{} var dockercli *dockerclient.Client @@ -141,6 +142,9 @@ func deployServiceWorkers(image string) { if swarmConfig == "run" { // frikky@debian:~/git/shuffle/functions/onprem/worker$ docker service create --replicas 5 --name shuffle-workers --env SHUFFLE_SWARM_CONFIG=run --publish published=33333,target=33333 ghcr.io/frikky/shuffle-worker:nightly networkName := "shuffle-executions" + if len(swarmNetworkName) > 0 { + networkName = swarmNetworkName + } ctx := context.Background() //docker network create --driver=overlay workers @@ -374,7 +378,7 @@ func deployWorker(image string, identifier string, env []string, executionReques log.Printf("[ERROR] Failed to start container in environment %s: %s", environment, err) return err } else { - log.Printf("[INFO] Container %s was created under environment %s", cont.ID, environment) + log.Printf("[INFO] Container %s was created under environment %s for execution %s", cont.ID, environment, executionRequest.ExecutionId) } //stats, err := cli.ContainerInspect(context.Background(), containerName) @@ -469,7 +473,7 @@ func initializeImages() { pullOptions := types.ImagePullOptions{} for _, image := range images { - log.Printf("[INFO] Pulling image %s", image) + log.Printf("[DEBUG] Pulling image %s", image) reader, err := dockercli.ImagePull(ctx, image, pullOptions) if err != nil { log.Printf("[ERROR] Failed getting image %s: %s", image, err) @@ -477,7 +481,7 @@ func initializeImages() { } io.Copy(os.Stdout, reader) - log.Printf("[INFO] Successfully downloaded and built %s", image) + log.Printf("[DEBUG] Successfully downloaded and built %s", image) } } @@ -772,7 +776,7 @@ func main() { err = deployWorker(workerImage, containerName, env, execution) zombiecounter += 1 if err == nil { - log.Printf("[INFO] ExecutionID %s was deployed and to be removed from queue.", execution.ExecutionId) + //log.Printf("[DEBUG] ExecutionID %s was deployed and to be removed from queue.", execution.ExecutionId) toBeRemoved.Data = append(toBeRemoved.Data, execution) executionIds = append(executionIds, execution.ExecutionId) } else { diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 2dd3c7ac..f430a357 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -45,6 +45,7 @@ var baseUrl = os.Getenv("BASE_URL") var appCallbackUrl = os.Getenv("BASE_URL") var cleanupEnv = strings.ToLower(os.Getenv("CLEANUP")) var dockerApiVersion = strings.ToLower(os.Getenv("DOCKER_API_VERSION")) +var swarmNetworkName = os.Getenv("SHUFFLE_SWARM_NETWORK_NAME") var timezone = os.Getenv("TZ") var baseimagename = "frikky/shuffle" var registryName = "registry.hub.docker.com" @@ -69,6 +70,7 @@ var startAction string var results []shuffle.ActionResult var allLogs map[string]string var containerIds []string +var downloadedImages []string var executionRunning bool @@ -228,6 +230,12 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] appName = strings.ToLower(appName) log.Printf("[INFO] New appname: %s, image: %s", appName, image) + if !shuffle.ArrayContains(downloadedImages, image) { + log.Printf("[DEBUG] Downloading image %s from backend as it's first iteration", image) + downloadDockerImageBackend(&http.Client{}, image) + downloadedImages = append(downloadedImages, image) + } + exposedPort, err := findAppInfo(image, appName) if err != nil { log.Printf("[ERROR] Failed finding and creating port for %s: %s", appName, err) @@ -338,9 +346,40 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] err = cli.ContainerStart(ctx, cont.ID, types.ContainerStartOptions{}) if err != nil { - log.Printf("[ERROR] Failed to start container in environment %s: %s", environment, err) - //shutdown(workflowExecution, workflowExecution.Workflow.ID, true) - return err + if strings.Contains(fmt.Sprintf("%s", err), "cannot join network") { + parsedUuid := uuid.NewV4() + identifier = fmt.Sprintf("%s-%s-nonetwork", identifier, parsedUuid) + hostConfig = &container.HostConfig{ + LogConfig: container.LogConfig{ + Type: "json-file", + Config: map[string]string{}, + }, + Resources: container.Resources{}, + } + + cont, err = cli.ContainerCreate( + context.Background(), + config, + hostConfig, + nil, + nil, + identifier, + ) + + if err != nil { + log.Printf("[ERROR] Container create error (3): %s", err) + return err + } + + log.Printf("[DEBUG] Running secondary check without network with worker") + err = cli.ContainerStart(ctx, cont.ID, types.ContainerStartOptions{}) + } + + if err != nil { + log.Printf("[ERROR] Failed to start container in environment %s: %s", environment, err) + //shutdown(workflowExecution, workflowExecution.Workflow.ID, true) + return err + } } log.Printf("[INFO] Container %s was created for %s", cont.ID, identifier) @@ -375,9 +414,12 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] // FIXME: Re-add log tracking which can be sent to backend //allLogs[actionId] = logs - if stats.ContainerJSONBase.State.Status == "exited" && !strings.Contains(logs, "Normal execution.") { - log.Printf("[WARNING] BAD Execution Logs for %s: %s", action.ID, logs) - exit = true + if stats.ContainerJSONBase.State.Status == "exited" && (!strings.Contains(logs, "Normal execution.") && !strings.Contains(logs, "indicates microservices")) { + + if len(logs) > 10 { + log.Printf("[ERROR] BAD Execution Logs for %s: %s", action.ID, logs) + exit = true + } } } @@ -2175,6 +2217,11 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl } //image := fmt.Sprintf("%s:%s", baseimagename, name) + networkName := "shuffle-executions" + if len(swarmNetworkName) > 0 { + networkName = swarmNetworkName + } + log.Printf("[DEBUG] Deploying app with name %s with image %s", name, image) containerName := fmt.Sprintf(strings.Replace(name, ".", "-", -1)) serviceSpec := swarm.ServiceSpec{ @@ -2184,7 +2231,7 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl }, Networks: []swarm.NetworkAttachmentConfig{ swarm.NetworkAttachmentConfig{ - Target: "shuffle-executions", + Target: networkName, }, }, EndpointSpec: &swarm.EndpointSpec{ @@ -2505,6 +2552,8 @@ func main() { log.Printf("[INFO] Running with timezone %s and swarm config %#v", timezone, os.Getenv("SHUFFLE_SWARM_CONFIG")) if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" { workflowExecution := shuffle.WorkflowExecution{} + + // Forcing download just in case on the first iteration. listener := webserverSetup(workflowExecution) //err := executionInit(workflowExecution) //if err != nil { @@ -2523,7 +2572,6 @@ func main() { } //imageName := fmt.Sprintf("%s/%s:shuffle_openapi_1.0.0", registryName, baseimagename) - //downloadDockerImageBackend(client, imageName) // WORKER_TESTING_WORKFLOW should be a workflow ID authorization := ""