From 2633ff4b4d74d1caa75b78577a16f2d00d6d1e29 Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 5 Sep 2020 08:56:29 +0200 Subject: [PATCH] #135: Started building app exist validation --- backend/app_sdk/build.sh | 4 +- backend/go-app/docker.go | 43 ++++++++++++++ backend/go-app/walkoff.go | 52 +++++++---------- docker-compose.yml | 10 ++-- frontend/src/views/AppCreator.jsx | 2 +- functions/onprem/orborus/build.sh | 2 +- functions/onprem/orborus/orborus.go | 89 ++++++++++++++++------------- functions/onprem/worker/worker.go | 58 +++++++++---------- 8 files changed, 152 insertions(+), 108 deletions(-) diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index a8fe1e93..1ded486e 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -3,11 +3,11 @@ NAME=app_sdk VERSION=0.6.0 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force -docker build . -t frikky/shuffle:$NAME -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t gchr.io/frikky/app_sdk:0.6.0 +docker build . -t frikky/shuffle:$NAME -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/app_sdk:0.6.0 #docker push frikky/shuffle:$NAME #docker push frikky/$NAME:$VERSION #docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION #docker push ghcr.io/frikky/$NAME:$VERSION -docker push gchr.io/frikky/app_sdk +docker push ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index 717f04b7..1303b8e1 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -642,6 +642,49 @@ func handleStartHookDocker(resp http.ResponseWriter, request *http.Request) { return } +// Checks if an image exists +func imageCheckBuilder(images []string) error { + log.Printf("ImageNames: %#v", images) + ctx := context.Background() + client, err := client.NewEnvClient() + if err != nil { + log.Printf("Unable to create docker client: %s", err) + return err + } + + allImages, err := client.ImageList(ctx, types.ImageListOptions{ + All: true, + }) + + if err != nil { + log.Printf("[ERROR] Failed creating imagelist: %s", err) + return err + } + + filteredImages := []types.ImageSummary{} + for _, image := range allImages { + found := false + for _, repoTag := range image.RepoTags { + if strings.Contains(repoTag, baseDockerName) { + found = true + break + } + } + + if found { + filteredImages = append(filteredImages, image) + } + } + + // FIXME: Continue fixing apps here + // https://github.com/frikky/Shuffle/issues/135 + // 1. Find if app exists + // 2. Create app if it doesn't + //log.Printf("Apps: %#v", filteredImages) + + return nil +} + func hookTest() { var hook Hook err := json.Unmarshal([]byte(webhook), &hook) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index f935a212..f361e5d6 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -42,14 +42,10 @@ var localBase = "http://localhost:5001" var baseEnvironment = "onprem" var cloudname = "cloud" - var defaultLocation = "europe-west2" var scheduledJobs = map[string]*newscheduler.Job{} // To test out firestore before potential merge -var shuffleTestProject = "shuffle-test-258209" -var shuffleTestPath = "./shuffle-test-258209-5a2e8d7e508a.json" - //var upgrader = websocket.Upgrader{ // ReadBufferSize: 1024, // WriteBufferSize: 1024, @@ -2146,7 +2142,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf //log.Printf("Execution data: %#v", execution) if len(execution.Start) == 36 { - log.Printf("SHOULD START ON NODE %s", execution.Start) + log.Printf("[INFO] Should start execution on node %s", execution.Start) workflowExecution.Start = execution.Start found := false @@ -2157,12 +2153,12 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } if !found { - log.Printf("ACTION %s WAS NOT FOUND!", workflow.Start) + log.Printf("[ERROR] ACTION %s WAS NOT FOUND!", workflow.Start) return WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", workflow.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", workflow.Start)) } } else if len(execution.Start) > 0 { - log.Printf("START ACTION %s IS WRONG ID LENGTH %d!", execution.Start, len(execution.Start)) + log.Printf("[ERROR] START ACTION %s IS WRONG ID LENGTH %d!", execution.Start, len(execution.Start)) return WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", execution.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", execution.Start)) } @@ -2292,10 +2288,10 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } if len(workflowExecution.ExecutionSource) == 0 { - log.Printf("No execution source specified. Setting to default") + log.Printf("[INFO] No execution source (trigger) specified. Setting to default") workflowExecution.ExecutionSource = "default" } else { - log.Printf("Execution source is %s for execution ID %s", workflowExecution.ExecutionSource, workflowExecution.ExecutionId) + log.Printf("[INFO] Execution source is %s for execution ID %s", workflowExecution.ExecutionSource, workflowExecution.ExecutionId) } workflowExecution.ExecutionVariables = workflow.ExecutionVariables @@ -2315,7 +2311,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if len(workflowExecution.Start) == 0 { workflowExecution.Start = workflowExecution.Workflow.Start } - log.Printf("STARTNODE: %s", workflowExecution.Start) + log.Printf("[INFO] New startnode: %s", workflowExecution.Start) childNodes := findChildNodes(workflowExecution, workflowExecution.Start) @@ -2423,6 +2419,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf environments := []string{} // Check if the actions are children of the startnode? + imageNames := []string{} for _, action := range workflowExecution.Workflow.Actions { if action.Environment != cloudname { found := false @@ -2433,6 +2430,11 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } } + // Check if the app exists? + newName := action.AppName + newName = strings.ReplaceAll(newName, " ", "-") + imageNames = append(imageNames, fmt.Sprintf("%s:%s_%s", baseDockerName, newName, action.AppVersion)) + if !found { environments = append(environments, action.Environment) } @@ -2441,6 +2443,12 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } } + err = imageCheckBuilder(imageNames) + if err != nil { + log.Printf("[ERROR] Failed building the required images from %#v: %s", imageNames, err) + return WorkflowExecution{}, "Failed building missing Docker images", err + } + err = setWorkflowExecution(ctx, workflowExecution) if err != nil { log.Printf("Error saving workflow execution for updates %s: %s", topic, err) @@ -2452,7 +2460,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if onpremExecution { // FIXME - tmp name based on future companyname-companyId for _, environment := range environments { - log.Printf("EXECUTION: %s should execute onprem with execution environment \"%s\"", workflowExecution.ExecutionId, environment) + log.Printf("[INFO] Execution: %s should execute onprem with execution environment \"%s\"", workflowExecution.ExecutionId, environment) executionRequest := ExecutionRequest{ ExecutionId: workflowExecution.ExecutionId, @@ -2539,7 +2547,7 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("STARTING EXEC OF %s!", fileId) + log.Printf("[INFO] Starting execution of %s!", fileId) workflowExecution, executionResp, err := handleExecution(fileId, *workflow, request) if err == nil { @@ -3051,26 +3059,6 @@ func getSpecificWorkflow(resp http.ResponseWriter, request *http.Request) { resp.Write(body) } -//func setWorkflowExecutionFS(ctx context.Context, reference string, workflowExecution WorkflowExecution) error { -// if len(workflowExecution.ExecutionId) == 0 { -// log.Printf("Workflowexeciton executionId can't be empty.") -// return errors.New("ExecutionId can't be empty.") -// } -// -// firestoreClient, err := firestore.NewClient(ctx, shuffleTestProject, option.WithCredentialsFile(shuffleTestPath)) -// if err != nil { -// return err -// } -// -// executionRef := firestoreClient.Doc(reference) -// _, err = executionRef.Set(ctx, workflowExecution) -// if err != nil { -// return err -// } -// -// return nil -//} - func setWorkflowExecution(ctx context.Context, workflowExecution WorkflowExecution) error { if len(workflowExecution.ExecutionId) == 0 { log.Printf("Workflowexeciton executionId can't be empty.") diff --git a/docker-compose.yml b/docker-compose.yml index 0b6ea2fc..ef24ab6f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: frontend: #build: ./frontend - image: frikky/shuffle:frontend + image: ghcr.io/frikky/shuffle-frontend:0.7.0 container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -17,7 +17,7 @@ services: - backend backend: #build: ./backend - image: frikky/shuffle:backend + image: ghcr.io/frikky/shuffle-backend:0.7.0 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: @@ -43,7 +43,7 @@ services: - database orborus: #build: ./functions/onprem/orborus - image: ghcr.io/frikky/orborus:0.6.0 + image: ghcr.io/frikky/orborus:0.6.1 container_name: shuffle-orborus hostname: shuffle-orborus networks: @@ -51,6 +51,8 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock environment: + - SHUFFLE_APP_SDK_VERSION=0.6.0 + - SHUFFLE_WORKER_VERSION=0.6.0 - ORG_ID=${ORG_ID} - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT} @@ -61,7 +63,7 @@ services: restart: unless-stopped database: #build: ./backend/database - image: frikky/shuffle:database + image: ghcr.io/frikky/shuffle-database:1.0.0 container_name: shuffle-database hostname: shuffle-database ports: diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index d24c2813..5c772da4 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -1031,7 +1031,7 @@ const AppCreator = (props) => { /> -
{deletePathQuery(index)}}> +
{deletePathQuery(index)}}> Delete
diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index f34c68f5..4902adce 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,5 +1,5 @@ NAME=orborus -VERSION=0.6.0 +VERSION=0.6.1 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 61c491cb..358f4441 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -30,11 +30,11 @@ var sleepTime = 3 // Timeout if something rashes var workerTimeout = 300 -var appSdkVersion = "0.6.0" -var workerVersion = "0.6.0" +var appSdkVersion = os.Getenv("SHUFFLE_APP_SDK_VERSION") +var workerVersion = os.Getenv("SHUFFLE_WORKER_VERSION") //var baseimagename = "docker.pkg.github.com/frikky/shuffle" -var baseimagename = "gchr.io/frikky" +var baseimagename = "ghcr.io/frikky" var orgId = os.Getenv("ORG_ID") var baseUrl = os.Getenv("BASE_URL") @@ -121,7 +121,7 @@ func deployWorker(image string, identifier string, env []string) { log.Printf("[INFO] Found container ID %s", containerId) hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) } else { - log.Printf("[WARNING] Empty self container id, continue without NetworkMode") + log.Printf("[INFO] Empty self container id, continue without NetworkMode") } config := &container.Config{ @@ -145,7 +145,7 @@ func deployWorker(image string, identifier string, env []string) { err = dockercli.ContainerStart(context.Background(), cont.ID, types.ContainerStartOptions{}) if err != nil { - log.Printf("Failed to start container in environment %s: %s", environment, err) + log.Printf("[ERROR] Failed to start container in environment %s: %s", environment, err) return //stats, err := cli.ContainerInspect(context.Background(), containerName) @@ -170,7 +170,7 @@ func deployWorker(image string, identifier string, env []string) { // } //} } else { - log.Printf("Container %s was created under environment %s", cont.ID, environment) + log.Printf("[INFO] Container %s was created under environment %s", cont.ID, environment) } return @@ -184,7 +184,7 @@ func stopWorker(containername string) error { // }) if err := dockercli.ContainerStop(ctx, containername, nil); err != nil { - log.Printf("Unable to stop container %s - running removal anyway, just in case: %s", containername, err) + log.Printf("[ERROR] Unable to stop container %s - running removal anyway, just in case: %s", containername, err) } removeOptions := types.ContainerRemoveOptions{ @@ -193,7 +193,7 @@ func stopWorker(containername string) error { } if err := dockercli.ContainerRemove(ctx, containername, removeOptions); err != nil { - log.Printf("Unable to remove container: %s", err) + log.Printf("[ERROR] Unable to remove container: %s", err) } return nil @@ -202,31 +202,42 @@ func stopWorker(containername string) error { func initializeImages() { ctx := context.Background() + if appSdkVersion == "" { + appSdkVersion = "0.6.0" + log.Printf("[WARNING] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %s", appSdkVersion) + } + if workerVersion == "" { + workerVersion = "0.6.0" + log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %s", workerVersion) + } + // check whether theyre the same first images := []string{ // fmt.Sprintf("docker.io/%s:app_sdk", baseimagename), // fmt.Sprintf("docker.io/%s:worker", baseimagename), + fmt.Sprintf("%s/worker:%s", baseimagename, workerVersion), fmt.Sprintf("%s/app_sdk:%s", baseimagename, appSdkVersion), } pullOptions := types.ImagePullOptions{} for _, image := range images { + log.Printf("[INFO] Pulling image %s", image) reader, err := dockercli.ImagePull(ctx, image, pullOptions) if err != nil { - log.Printf("Failed getting image %s: %s", image, err) + log.Printf("[ERROR] Failed getting image %s: %s", image, err) continue } io.Copy(os.Stdout, reader) - log.Printf("Successfully downloaded and built %s", image) + log.Printf("[INFO] Successfully downloaded and built %s", image) } } // Initial loop etc func main() { go zombiecheck() - log.Println("Setting up execution environment") + log.Println("[INFO] Setting up execution environment") //FIXME if baseUrl == "" { @@ -235,30 +246,30 @@ func main() { } if orgId == "" { - log.Printf("Org not defined. Set variable ORG_ID based on your org") + log.Printf("[ERROR] Org not defined. Set variable ORG_ID based on your org") os.Exit(3) } - log.Printf("Running towards %s with Org %s", baseUrl, orgId) + log.Printf("[INFO] Running towards %s with Org %s", baseUrl, orgId) httpProxy := os.Getenv("HTTP_PROXY") httpsProxy := os.Getenv("HTTPS_PROXY") if environment == "" { environment = "onprem" - log.Printf("Defaulting to environment name %s. Set environment variable ENVIRONMENT_NAME to change. This should be the same as in the frontend action.", environment) + log.Printf("[INFO] Defaulting to environment name %s. Set environment variable ENVIRONMENT_NAME to change. This should be the same as in the frontend action.", environment) } // FIXME - during init, BUILD and/or LOAD worker and app_sdk // Build/load app_sdk so it can be loaded as 127.0.0.1:5000/walkoff_app_sdk - log.Printf("--- Setting up Docker environment. Downloading worker and App SDK! ---") + log.Printf("[INFO] Setting up Docker environment. Downloading worker and App SDK!") initializeImages() //workerName := "worker" //workerVersion := "0.1.0" //workerImage := fmt.Sprintf("docker.pkg.github.com/frikky/shuffle/%s:%s", workerName, workerVersion) - workerImage := fmt.Sprintf("%s:worker", baseimagename) + workerImage := fmt.Sprintf("%s/worker:%s", baseimagename, workerVersion) - log.Printf("--- Finished configuring docker environment ---\n") + log.Printf("[INFO] Finished configuring docker environment") // FIXME - time limit client := &http.Client{ @@ -271,10 +282,10 @@ func main() { client = &http.Client{} } else { if len(httpProxy) > 0 { - log.Printf("Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy) + log.Printf("[INFO] Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy) } if len(httpsProxy) > 0 { - log.Printf("Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy) + log.Printf("[INFO] Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy) } } @@ -286,21 +297,21 @@ func main() { ) if err != nil { - log.Printf("Failed making request builder: %s", err) + log.Printf("[ERROR] Failed making request builder: %s", err) os.Exit(3) } zombiecounter := 0 req.Header.Add("Content-Type", "application/json") req.Header.Add("Org-Id", orgId) - log.Printf("Getting data from %s", fullUrl) + log.Printf("[INFO] Waiting for executions at %s", fullUrl) hasStarted := false for { //log.Printf("Prerequest") newresp, err := client.Do(req) //log.Printf("Postrequest") if err != nil { - log.Printf("Failed making request: %s", err) + log.Printf("[WARNING] Failed making request: %s", err) zombiecounter += 1 if zombiecounter*sleepTime > workerTimeout { go zombiecheck() @@ -313,7 +324,7 @@ func main() { // FIXME - add check for StatusCode if newresp.StatusCode != 200 { if hasStarted { - log.Printf("Bad statuscode: %d", newresp.StatusCode) + log.Printf("[WARNING] Bad statuscode: %d", newresp.StatusCode) } } else { hasStarted = true @@ -321,7 +332,7 @@ func main() { body, err := ioutil.ReadAll(newresp.Body) if err != nil { - log.Printf("Failed reading body: %s", err) + log.Printf("[ERROR] Failed reading body: %s", err) zombiecounter += 1 if zombiecounter*sleepTime > workerTimeout { go zombiecheck() @@ -334,7 +345,7 @@ func main() { var executionRequests ExecutionRequestWrapper err = json.Unmarshal(body, &executionRequests) if err != nil { - log.Printf("Failed executionrequest in queue unmarshaling: %s", err) + log.Printf("[WARNING] Failed executionrequest in queue unmarshaling: %s", err) sleepTime = 10 zombiecounter += 1 if zombiecounter*sleepTime > workerTimeout { @@ -346,7 +357,7 @@ func main() { } if hasStarted && len(executionRequests.Data) > 0 { - log.Printf("Body: %s", string(body)) + log.Printf("[INFO] Body: %s", string(body)) // Type string `json:"type"` } @@ -364,16 +375,16 @@ func main() { var toBeRemoved ExecutionRequestWrapper for _, execution := range executionRequests.Data { if len(execution.ExecutionArgument) > 0 { - log.Printf("Argument: %#v", execution.ExecutionArgument) + log.Printf("[INFO] Argument: %#v", execution.ExecutionArgument) } if execution.Type == "schedule" { - log.Printf("SOMETHING ELSE :O: %s", execution.Type) + log.Printf("[INFO] SOMETHING ELSE :O: %s", execution.Type) continue } if execution.Status == "ABORT" || execution.Status == "FAILED" { - log.Printf("Executionstatus issue: ", execution.Status) + log.Printf("[INFO] Executionstatus issue: ", execution.Status) } // Now, how do I execute this one? // FIXME - if error, check the status of the running one. If it's bad, send data back. @@ -396,7 +407,7 @@ func main() { go deployWorker(workerImage, containerName, env) - log.Printf("%s is deployed and to be removed from queue.", execution.ExecutionId) + log.Printf("[INFO] %s is deployed and to be removed from queue.", execution.ExecutionId) zombiecounter += 1 toBeRemoved.Data = append(toBeRemoved.Data, execution) } @@ -407,7 +418,7 @@ func main() { data, err := json.Marshal(toBeRemoved) if err != nil { - log.Printf("Failed removal marshalling: %s", err) + log.Printf("[WARNING] Failed removal marshalling: %s", err) time.Sleep(time.Duration(sleepTime) * time.Second) continue } @@ -419,7 +430,7 @@ func main() { ) if err != nil { - log.Printf("Failed building confirm request: %s", err) + log.Printf("[ERROR] Failed building confirm request: %s", err) time.Sleep(time.Duration(sleepTime) * time.Second) continue } @@ -429,14 +440,14 @@ func main() { resultResp, err := client.Do(result) if err != nil { - log.Printf("Failed making confirm request: %s", err) + log.Printf("[ERROR] Failed making confirm request: %s", err) time.Sleep(time.Duration(sleepTime) * time.Second) continue } body, err := ioutil.ReadAll(resultResp.Body) if err != nil { - log.Printf("Failed reading confirm body: %s", err) + log.Printf("[ERROR] Failed reading confirm body: %s", err) time.Sleep(time.Duration(sleepTime) * time.Second) continue } @@ -450,7 +461,7 @@ func main() { if len(toBeRemoved.Data) == len(executionRequests.Data) { //log.Println("Should remove ALL!") } else { - log.Printf("NOT IMPLEMENTED: Should remove %d workflows from backend because they're executed!", len(toBeRemoved.Data)) + log.Printf("[INFO] NOT IMPLEMENTED: Should remove %d workflows from backend because they're executed!", len(toBeRemoved.Data)) } } @@ -461,7 +472,7 @@ func main() { // FIXME - add this to remove exited workers // Should it check what happened to the execution? idk func zombiecheck() error { - log.Println("Looking for old containers") + log.Println("[INFO] Looking for old containers") ctx := context.Background() containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{ @@ -469,7 +480,7 @@ func zombiecheck() error { }) if err != nil { - log.Printf("Failed creating Containerlist: %s", err) + log.Printf("[ERROR] Failed creating Containerlist: %s", err) return err } @@ -502,7 +513,7 @@ func zombiecheck() error { continue } - log.Printf("NAME: %s", name) + log.Printf("[INFO] NAME: %s", name) // Need to check time here too because a container can be removed the same instant as its created currenttime := time.Now().Unix() @@ -522,7 +533,7 @@ func zombiecheck() error { // FIXME - add killing of apps with same execution ID too for _, containername := range stopContainers { - log.Printf("Stopping and removing container %s", containerNames[containername]) + log.Printf("[INFO] Stopping and removing container %s", containerNames[containername]) go dockercli.ContainerStop(ctx, containername, nil) removeContainers = append(removeContainers, containername) } diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 4a4492a6..247a2369 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -44,7 +44,7 @@ func init() { if len(containerId) == 0 { log.Printf("[ERROR] No container ID found.") } else { - log.Printf("Found container ID: %s", containerId) + log.Printf("[INFO] Found container ID: %s", containerId) } } @@ -359,7 +359,7 @@ type ExecutionRequestWrapper struct { func shutdown(executionId, workflowId string) { dockercli, err := dockerclient.NewEnvClient() if err != nil { - log.Printf("Unable to create docker client: %s", err) + log.Printf("[ERROR] Unable to create docker client: %s", err) os.Exit(3) } @@ -396,7 +396,7 @@ func shutdown(executionId, workflowId string) { ) if err != nil { - log.Println("Failed building request: %s", err) + log.Println("[INFO] Failed building request: %s", err) } // FIXME: Add an API call to the backend @@ -404,7 +404,7 @@ func shutdown(executionId, workflowId string) { if len(authorization) > 0 { req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", authorization)) } else { - log.Printf("No authorization specified for abort") + log.Printf("[ERROR] No authorization specified for abort") } req.Header.Add("Content-Type", "application/json") @@ -420,18 +420,18 @@ func shutdown(executionId, workflowId string) { client = &http.Client{} } else { if len(httpProxy) > 0 { - log.Printf("Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy) + log.Printf("[INFO] Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy) } if len(httpsProxy) > 0 { - log.Printf("Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy) + log.Printf("[INFO] Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy) } } _, err = client.Do(req) if err != nil { - log.Printf("Failed abort request: %s", err) + log.Printf("[INFO] Failed abort request: %s", err) } - log.Printf("Finished shutdown.") + log.Printf("[INFO] Finished shutdown.") os.Exit(3) } @@ -467,13 +467,12 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] ) if err != nil { - log.Println(err) + log.Printf("Container error: %s", err) return err } cli.ContainerStart(context.Background(), cont.ID, types.ContainerStartOptions{}) - fmt.Printf("\n") - log.Printf("Container %s is created", cont.ID) + log.Printf("[INFO] Container %s is created", cont.ID) return nil } @@ -482,7 +481,7 @@ func removeContainer(containername string) error { cli, err := dockerclient.NewEnvClient() if err != nil { - log.Printf("Unable to create docker client: %s", err) + log.Printf("[INFO] Unable to create docker client: %s", err) return err } @@ -901,9 +900,11 @@ func handleExecution(client *http.Client, req *http.Request, workflowExecution W err = deployApp(dockercli, image, identifier, env) if err != nil { - log.Printf("Failed deploying %s from image %s: %s", identifier, image, err) - log.Printf("Should send status and exit the entire thing?") - //shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + log.Printf("[ERROR] Failed deploying %s from image %s: %s", identifier, image, err) + if strings.Contains(err.Error(), "No such image") { + log.Printf("[ERROR] Image doesn't exist. Shutting down") + shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + } } log.Printf("Adding visited (3): %s", action.Label) @@ -1091,7 +1092,7 @@ func runTestExecution(client *http.Client, workflowId, apikey string) (string, s return "", "" } - log.Printf("Body: %s", string(body)) + log.Printf("[INFO] Body: %s", string(body)) var workflowExecution WorkflowExecution err = json.Unmarshal(body, &workflowExecution) if err != nil { @@ -1104,7 +1105,7 @@ func runTestExecution(client *http.Client, workflowId, apikey string) (string, s // Initial loop etc func main() { - log.Printf("Setting up worker environment") + log.Printf("[INFO] Setting up worker environment") sleepTime := 5 client := &http.Client{ @@ -1133,7 +1134,7 @@ func main() { shuffle_apikey := os.Getenv("WORKER_TESTING_APIKEY") if len(testing) > 0 && len(shuffle_apikey) > 0 { // Execute a workflow and use that info - log.Printf("!! Running test environment for worker by executing workflow %s", testing) + log.Printf("[WARNING] Running test environment for worker by executing workflow %s", testing) authorization, executionId = runTestExecution(client, testing, shuffle_apikey) //os.Exit(3) @@ -1144,12 +1145,12 @@ func main() { } if len(authorization) == 0 { - log.Println("No AUTHORIZATION key set in env") + log.Println("[INFO] No AUTHORIZATION key set in env") shutdown(executionId, "") } if len(executionId) == 0 { - log.Println("No EXECUTIONID key set in env") + log.Println("[INFO] No EXECUTIONID key set in env") shutdown(executionId, "") } @@ -1163,7 +1164,7 @@ func main() { ) if err != nil { - log.Println("Failed making request builder") + log.Println("[ERROR] Failed making request builder for backend") shutdown(executionId, "") } @@ -1172,20 +1173,20 @@ func main() { // Removed request requirement from app_sdk newresp, err := client.Do(req) if err != nil { - log.Printf("Failed request: %s", err) + log.Printf("[ERROR] Failed request: %s", err) time.Sleep(time.Duration(sleepTime) * time.Second) continue } body, err := ioutil.ReadAll(newresp.Body) if err != nil { - log.Printf("Failed reading body: %s", err) + log.Printf("[ERROR] Failed reading body: %s", err) time.Sleep(time.Duration(sleepTime) * time.Second) continue } if newresp.StatusCode != 200 { - log.Printf("Err: %s\nStatusCode: %d", string(body), newresp.StatusCode) + log.Printf("[ERROR] %s\nStatusCode: %d", string(body), newresp.StatusCode) time.Sleep(time.Duration(sleepTime) * time.Second) continue } @@ -1193,13 +1194,13 @@ func main() { var workflowExecution WorkflowExecution err = json.Unmarshal(body, &workflowExecution) if err != nil { - log.Printf("Failed workflowExecution unmarshal: %s", err) + log.Printf("[ERROR] Failed workflowExecution unmarshal: %s", err) time.Sleep(time.Duration(sleepTime) * time.Second) continue } if workflowExecution.Status == "FINISHED" || workflowExecution.Status == "SUCCESS" { - log.Printf("Workflow %s is finished. Exiting worker.", workflowExecution.ExecutionId) + log.Printf("[INFO] Workflow %s is finished. Exiting worker.", workflowExecution.ExecutionId) shutdown(executionId, workflowExecution.Workflow.ID) } @@ -1207,15 +1208,14 @@ func main() { //log.Printf("Status: %s", workflowExecution.Status) err = handleExecution(client, req, workflowExecution) if err != nil { - log.Printf("Workflow %s is finished: %s", workflowExecution.ExecutionId, err) + log.Printf("[INFO] Workflow %s is finished: %s", workflowExecution.ExecutionId, err) shutdown(executionId, workflowExecution.Workflow.ID) } } else { - log.Printf("Workflow %s has status %s. Exiting worker.", workflowExecution.ExecutionId, workflowExecution.Status) + log.Printf("[INFO] Workflow %s has status %s. Exiting worker.", workflowExecution.ExecutionId, workflowExecution.Status) shutdown(executionId, workflowExecution.Workflow.ID) } - //log.Println(string(body)) time.Sleep(time.Duration(sleepTime) * time.Second) } }