From c7a626bdbd4e5268d0186f6815fd262e9f231a9f Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 17 Nov 2020 15:54:42 +0100 Subject: [PATCH] Migrated docker images to be versioned --- .env | 6 ++-- backend/app_sdk/build.sh | 4 +-- backend/go-app/walkoff.go | 32 +++++++++++++---- docker-compose.yml | 12 +++---- frontend/src/views/AngularWorkflow.jsx | 11 +++--- functions/onprem/orborus/Dockerfile | 3 +- functions/onprem/orborus/build.sh | 8 ++--- functions/onprem/orborus/orborus.go | 49 ++++++++++++++++---------- functions/onprem/worker/build.sh | 4 +-- functions/onprem/worker/worker.go | 19 +++++----- 10 files changed, 93 insertions(+), 55 deletions(-) diff --git a/.env b/.env index 4fd9eced..93b05920 100644 --- a/.env +++ b/.env @@ -35,6 +35,6 @@ SHUFFLE_HTTP_PROXY= SHUFFLE_HTTPS_PROXY= SHUFFLE_PASS_WORKER_PROXY=TRUE -SHUFFLE_BASE_IMAGE_REGISTRY=docker.io -SHUFFLE_BASE_IMAGE_NAME=frikky/shuffle -SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-0.6.0" \ No newline at end of file +SHUFFLE_BASE_IMAGE_REGISTRY=ghcr.io +SHUFFLE_BASE_IMAGE_NAME=frikky +SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-0.8.0" diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index bfd63f22..5e2fc3aa 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,6 +1,6 @@ #!/bin/bash -NAME=app_sdk -VERSION=0.7.6 +NAME=shuffle-app_sdk +VERSION=0.8.0 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -t frikky/shuffle:$NAME -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 729fbdd8..08f8846b 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -919,6 +919,16 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { err := handleUserInput(trigger, orgId, workflowExecution.Workflow.ID, workflowExecution.ExecutionId) if err != nil { log.Printf("Failed userinput handler: %s", err) + actionResult.Result = fmt.Sprintf("Cloud error: %s", err) + workflowExecution.Results = append(workflowExecution.Results, actionResult) + workflowExecution.Status = "ABORTED" + err = setWorkflowExecution(ctx, *workflowExecution) + if err != nil { + log.Printf("Failed ") + } else { + log.Printf("Successfully set the execution to waiting.") + } + resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error: %s"}`, err))) } else { @@ -1311,7 +1321,7 @@ func handleExecutionStatistics(execution WorkflowExecution) { log.Printf("Added %d exampleresults to backend", successful) } else { - log.Printf("No examplresults necessary to be added for execution %s", execution.ExecutionId) + log.Printf("No example results necessary to be added for execution %s", execution.ExecutionId) } } @@ -2756,7 +2766,17 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf // If the node is NOT found, it's supposed to be set to SKIPPED, // as it's not a childnode of the startnode // This is a configuration item for the workflow itself. - if !workflowExecution.Workflow.Configuration.StartFromTop { + if len(workflowExecution.Results) > 0 { + defaultResults = []ActionResult{} + for _, result := range workflowExecution.Results { + if result.Status == "WAITING" { + result.Status = "FINISHED" + result.Result = "Continuing" + } + + defaultResults = append(defaultResults, result) + } + } else if len(workflowExecution.Results) == 0 && !workflowExecution.Workflow.Configuration.StartFromTop { found := false for _, nodeId := range childNodes { if nodeId == action.ID { @@ -6167,13 +6187,13 @@ func handleUserInput(trigger Trigger, organizationId string, workflowId string, org, err := getOrg(ctx, organizationId) if err != nil { - log.Printf("Failed email send to cloud: %s", err) + log.Printf("Failed email send to cloud (1): %s", err) return err } err = executeCloudAction(action, org.SyncConfig.Apikey) if err != nil { - log.Printf("Failed email send to cloud", err) + log.Printf("Failed email send to cloud (2): %s", err) return err } @@ -6193,13 +6213,13 @@ func handleUserInput(trigger Trigger, organizationId string, workflowId string, org, err := getOrg(ctx, organizationId) if err != nil { - log.Printf("Failed email send to cloud", err) + log.Printf("Failed sms send to cloud (3): %s", err) return err } err = executeCloudAction(action, org.SyncConfig.Apikey) if err != nil { - log.Printf("Failed email send to cloud", err) + log.Printf("Failed sms send to cloud (4): %s", err) return err } diff --git a/docker-compose.yml b/docker-compose.yml index 37e61122..f57f1e90 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,8 @@ version: '3' services: frontend: - #build: ./frontend - image: frikky/shuffle:frontend + build: ./frontend + image: ghcr.io/frikky/shuffle-frontend:0.8.0 container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -17,7 +17,7 @@ services: - backend backend: build: ./backend - image: frikky/shuffle:backend + image: ghcr.io/frikky/shuffle-backend:0.8.0 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: @@ -43,7 +43,7 @@ services: - database orborus: #build: ./functions/onprem/orborus - image: frikky/shuffle:orborus + image: ghcr.io/frikky/shuffle-orborus:0.8.0 container_name: shuffle-orborus hostname: shuffle-orborus networks: @@ -51,8 +51,8 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock environment: - - SHUFFLE_APP_SDK_VERSION=0.6.0 - - SHUFFLE_WORKER_VERSION=0.6.0 + - SHUFFLE_APP_SDK_VERSION=0.8.0 + - SHUFFLE_WORKER_VERSION=0.8.0 - ORG_ID=${ORG_ID} - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT} diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index ba4d6199..3e417534 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1074,10 +1074,13 @@ const AngularWorkflow = (props) => { // FIXME - do this for both actions and other types? targetnode = workflow.triggers.findIndex(data => data.id === edge.target) if (targetnode !== -1) { - alert.error("Can't have triggers as target of branch") - event.target.remove() - found = true - break + if (workflow.triggers[targetnode].app_name !== "User Input") { + + alert.error("Can't have triggers as target of branch") + event.target.remove() + found = true + break + } } } } diff --git a/functions/onprem/orborus/Dockerfile b/functions/onprem/orborus/Dockerfile index e5bc379e..3db4c027 100644 --- a/functions/onprem/orborus/Dockerfile +++ b/functions/onprem/orborus/Dockerfile @@ -5,7 +5,8 @@ WORKDIR /app RUN go get github.com/docker/docker/api/types github.com/docker/docker/api/types/container github.com/docker/docker/client COPY orborus.go /app/orborus.go - +RUN go mod init orborus +RUN go build RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o orborus . FROM alpine:3.12 diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index 9c139d11..14499755 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,11 +1,11 @@ -NAME=orborus -VERSION=0.6.2 +NAME=shuffle-orborus +VERSION=0.8.0 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force docker build . -t frikky/shuffle:$NAME -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION #docker push frikky/$NAME:$VERSION -docker push frikky/shuffle:$NAME +#docker push frikky/shuffle:$NAME # docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -#docker push ghcr.io/frikky/$NAME:$VERSION +docker push ghcr.io/frikky/$NAME:$VERSION diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 412f0263..78984425 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -22,6 +22,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" dockerclient "github.com/docker/docker/client" + "github.com/satori/go.uuid" //network "github.com/docker/docker/api/types/network" //natting "github.com/docker/go-connections/nat" ) @@ -134,30 +135,34 @@ func deployWorker(image string, identifier string, env []string) { Env: env, } + log.Printf("Identifier: %s", identifier) cont, err := dockercli.ContainerCreate( context.Background(), config, hostConfig, nil, - nil, identifier, ) if err != nil { - log.Printf("[ERROR] Container create error: %s", err) + if strings.Contains(fmt.Sprintf("%s", err), "Conflict. The container name ") { + uuid := uuid.NewV4() + identifier = fmt.Sprintf("%s-%s", identifier, uuid) + log.Printf("2 - Identifier: %s", identifier) + cont, err = dockercli.ContainerCreate( + context.Background(), + config, + hostConfig, + nil, + identifier, + ) - identifier := fmt.Sprintf("%s-new", identifier) - cont, err = dockercli.ContainerCreate( - context.Background(), - config, - hostConfig, - nil, - nil, - identifier, - ) - - if err != nil { - log.Printf("[ERROR] Container create error(2): %s", err) + if err != nil { + log.Printf("[ERROR] Container create error(2): %s", err) + return + } + } else { + log.Printf("[ERROR] Container create error: %s", err) return } } @@ -222,27 +227,32 @@ func initializeImages() { ctx := context.Background() if appSdkVersion == "" { - appSdkVersion = "0.6.0" + appSdkVersion = "0.8.0" log.Printf("[WARNING] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %s", appSdkVersion) } if workerVersion == "" { - workerVersion = "0.6.0" + workerVersion = "0.8.0" log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %s", workerVersion) } if baseimageregistry == "" { baseimageregistry = "docker.io" + baseimageregistry = "ghcr.io" log.Printf("Setting baseimageregistry") } if baseimagename == "" { baseimagename = "frikky/shuffle" + baseimagename = "frikky" log.Printf("Setting baseimagename") } // check whether they are the same first images := []string{ - fmt.Sprintf("%s/%s:app_sdk%s", baseimageregistry, baseimagename, baseimagetagsuffix), - fmt.Sprintf("%s/%s:worker%s", baseimageregistry, baseimagename, baseimagetagsuffix), + //fmt.Sprintf("%s/%s:app_sdk%s", baseimageregistry, baseimagename, baseimagetagsuffix), + //fmt.Sprintf("%s/%s:worker%s", baseimageregistry, baseimagename, baseimagetagsuffix), + + fmt.Sprintf("%s/%s/shuffle-app_sdk:%s", baseimageregistry, baseimagename, appSdkVersion), + fmt.Sprintf("%s/%s/shuffle-worker:%s", baseimageregistry, baseimagename, workerVersion), // fmt.Sprintf("docker.io/%s:app_sdk", baseimagename), // fmt.Sprintf("docker.io/%s:worker", baseimagename), @@ -315,7 +325,8 @@ func main() { //workerImage := fmt.Sprintf("%s/worker:%s", baseimagename, workerVersion) // workerImage := fmt.Sprintf("docker.io/%s:worker", baseimagename) // fmt.Sprintf("%s/%s:app_sdk%s", baseimageregistry, baseimagename, baseimagetagsuffix), - workerImage := fmt.Sprintf("%s/%s:worker%s", baseimageregistry, baseimagename, baseimagetagsuffix) + //workerImage := fmt.Sprintf("%s/%s:worker%s", baseimageregistry, baseimagename, baseimagetagsuffix) + workerImage := fmt.Sprintf("%s/%s/shuffle-worker:%s", baseimageregistry, baseimagename, workerVersion) log.Printf("[INFO] Finished configuring docker environment") diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh index 5872707f..363df000 100644 --- a/functions/onprem/worker/build.sh +++ b/functions/onprem/worker/build.sh @@ -1,4 +1,4 @@ -NAME=worker +NAME=shuffle-worker VERSION=0.8.0 echo "Running docker build with $NAME:$VERSION" @@ -9,4 +9,4 @@ docker build . -t frikky/shuffle:$NAME -t docker.pkg.github.com/frikky/shuffle/$ #docker push frikky/$NAME:$VERSION docker push frikky/shuffle:$NAME #docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -#docker push ghcr.io/frikky/$NAME:$VERSION +docker push ghcr.io/frikky/$NAME:$VERSION diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 244ecc4b..dbaea55f 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -792,11 +792,20 @@ func handleExecution(client *http.Client, req *http.Request, workflowExecution W // SKIP if it's not onprem for _, nextAction := range nextActions { action := getAction(workflowExecution, nextAction, environment) + // check visited and onprem + if arrayContains(visited, nextAction) { + log.Printf("ALREADY VISITIED (%s): %s", action.Label, nextAction) + continue + } + if action.AppName == "User Input" { log.Printf("USER INPUT!") if action.ID == workflowExecution.Start { log.Printf("Skipping because it's the startnode") + visited = append(visited, action.ID) + executed = append(executed, action.ID) + continue } else { log.Printf("Should stop after this iteration because it's user-input based. %#v", action) trigger := Trigger{} @@ -827,12 +836,6 @@ func handleExecution(client *http.Client, req *http.Request, workflowExecution W } } - // check visited and onprem - if arrayContains(visited, nextAction) { - log.Printf("ALREADY VISITIED (%s): %s", action.Label, nextAction) - continue - } - // Not really sure how this edgecase happens. // FIXME @@ -866,7 +869,7 @@ func handleExecution(client *http.Client, req *http.Request, workflowExecution W } if continueOuter { - //log.Printf("Parents of %s aren't finished: %s", nextAction, strings.Join(parents[nextAction], ", ")) + log.Printf("Parents of %s aren't finished: %s", nextAction, strings.Join(parents[nextAction], ", ")) //for _, tmpaction := range parents[nextAction] { // action := getAction(workflowExecution, tmpaction) // _ = action @@ -1029,7 +1032,7 @@ func handleExecution(client *http.Client, req *http.Request, workflowExecution W shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) } - if len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions) { + if len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions)+extra { shutdownCheck := true ctx := context.Background() for _, result := range workflowExecution.Results {