From 698b4f54a50bcb90038cb1c9c2c1842816263c00 Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 19 Feb 2022 16:45:57 +0100 Subject: [PATCH] Fixed new caching system in worker to ensure failed apps retry multiple times --- functions/onprem/worker/build.sh | 2 +- functions/onprem/worker/worker.go | 29 ++++++++++++++++++++++++++--- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh index 6d482956..e828cc0e 100644 --- a/functions/onprem/worker/build.sh +++ b/functions/onprem/worker/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-worker -VERSION=0.9.55 +VERSION=0.9.59 echo "Running docker build with $NAME:$VERSION" #CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin . diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 390a2012..9b634e90 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -358,6 +358,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] log.Printf("\n\n[DEBUG] Result for %s already found - returning\n\n", newExecId) return nil } + cacheData := []byte("1") err = shuffle.SetCache(ctx, newExecId, cacheData) if err != nil { @@ -371,17 +372,17 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] waitTime := time.Duration(action.ExecutionDelay) * time.Second time.AfterFunc(waitTime, func() { - DeployContainer(ctx, cli, config, hostConfig, identifier, workflowExecution) + DeployContainer(ctx, cli, config, hostConfig, identifier, workflowExecution, newExecId) }) } else { log.Printf("[DEBUG] Running app %s in docker NORMALLY as there is no delay set", action.Name) - return DeployContainer(ctx, cli, config, hostConfig, identifier, workflowExecution) + return DeployContainer(ctx, cli, config, hostConfig, identifier, workflowExecution, newExecId) } return nil } -func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *container.Config, hostConfig *container.HostConfig, identifier string, workflowExecution shuffle.WorkflowExecution) error { +func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *container.Config, hostConfig *container.HostConfig, identifier string, workflowExecution shuffle.WorkflowExecution, newExecId string) error { cont, err := cli.ContainerCreate( ctx, config, @@ -396,6 +397,11 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont if !strings.Contains(err.Error(), "Conflict. The container name") { log.Printf("[ERROR] Container CREATE error (1): %s", err) + err = shuffle.DeleteCache(ctx, newExecId) + if err != nil { + log.Printf("[ERROR] FAILED Deleting cache for %s: %s", newExecId, err) + } + return err } else { parsedUuid := uuid.NewV4() @@ -414,6 +420,11 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont if err != nil { log.Printf("[ERROR] Container create error (2): %s", err) + err = shuffle.DeleteCache(ctx, newExecId) + if err != nil { + log.Printf("[ERROR] FAILED Deleting cache for %s: %s", newExecId, err) + } + return err } @@ -445,6 +456,12 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont if err != nil { log.Printf("[ERROR] Container create error (3): %s", err) + + err = shuffle.DeleteCache(ctx, newExecId) + if err != nil { + log.Printf("[ERROR] FAILED Deleting cache for %s: %s", newExecId, err) + } + return err } @@ -454,6 +471,12 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont if err != nil { log.Printf("[ERROR] Failed to start container in environment %s: %s", environment, err) + + err = shuffle.DeleteCache(ctx, newExecId) + if err != nil { + log.Printf("[ERROR] FAILED Deleting cache for %s: %s", newExecId, err) + } + //shutdown(workflowExecution, workflowExecution.Workflow.ID, true) return err }