From 1adf20b43c857a110a78beb2b4c25e6478f79a1c Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 26 Apr 2021 18:55:52 +0200 Subject: [PATCH] #189: Fixed proxy and tagging issues with Docker build and runs --- .env | 1 + backend/app_sdk/app_base.py | 3 ++- backend/go-app/docker.go | 20 +++++++++++++++++++- backend/go-app/go.mod | 2 +- backend/go-app/go.sum | 2 ++ docker-compose.yml | 9 +++++---- frontend/src/views/Workflows.jsx | 2 ++ functions/onprem/orborus/build.sh | 2 +- functions/onprem/orborus/orborus.go | 10 ++++++---- functions/onprem/worker/build.sh | 2 +- functions/onprem/worker/worker.go | 10 ++++++++-- 11 files changed, 48 insertions(+), 15 deletions(-) diff --git a/.env b/.env index 1b646f6f..fc773f9a 100644 --- a/.env +++ b/.env @@ -38,6 +38,7 @@ DB_LOCATION=./shuffle-database SHUFFLE_HTTP_PROXY= SHUFFLE_HTTPS_PROXY= SHUFFLE_PASS_WORKER_PROXY=TRUE +SHUFFLE_PASS_APP_PROXY=FALSE SHUFFLE_BASE_IMAGE_REGISTRY=ghcr.io SHUFFLE_BASE_IMAGE_NAME=frikky diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index ecd87b9e..df4154b2 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -1621,13 +1621,14 @@ class AppBase: return True, "" + # THE START IS ACTUALLY RIGHT HERE :O # Checks whether conditions are met, otherwise set branchcheck, tmpresult = check_branch_conditions(action, fullexecution) if isinstance(tmpresult, object) or isinstance(tmpresult, list): print("Fixing branch return as object -> string") try: - tmpresult = tmpresult.replace("'", "\"") + #tmpresult = tmpresult.replace("'", "\"") tmpresult = json.dumps(tmpresult) except json.decoder.JSONDecodeError as e: print(f"[WARNING] Failed condition parsing {tmpresult} to string") diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index 214f81c9..ac25f9e6 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -214,11 +214,28 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin // Dockerfile is inside the TAR itself. Not local context // docker build --build-arg http_proxy=http://my.proxy.url + // Attempt at setting name according to #359: https://github.com/frikky/Shuffle/issues/359 + labels := map[string]string{} + target := "" + if len(tags) > 0 { + if strings.Contains(tags[0], ":") { + version := strings.Split(tags[0], ":") + if len(version) == 2 { + target = fmt.Sprintf("shuffle-build-%s", version[1]) + tags = append(tags, target) + labels["name"] = target + } + } + } + + _ = labels buildOptions := types.ImageBuildOptions{ Remove: true, Tags: tags, BuildArgs: map[string]*string{}, + Labels: labels, } + // NetworkMode: "host", httpProxy := os.Getenv("HTTP_PROXY") @@ -231,13 +248,14 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin } // Build the actual image - log.Printf("[INFO] Building %s. This may take up to a few minutes.", dockerfileFolder) + log.Printf(`[INFO] Building %s with proxy "%s". Tags: "%s". This may take up to a few minutes.`, dockerfileFolder, httpsProxy, strings.Join(tags, ",")) imageBuildResponse, err := client.ImageBuild( ctx, dockerFileTarReader, buildOptions, ) + log.Printf("RESPONSE: %#v", imageBuildResponse) //log.Printf("Response: %#v", imageBuildResponse.Body) //log.Printf("IMAGERESPONSE: %#v", imageBuildResponse.Body) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 3ac40e66..58802d79 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -2,7 +2,7 @@ module shuffle go 1.13 -//replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared +replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index ac20c7f5..1e8d5548 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -111,6 +111,8 @@ github.com/frikky/shuffle-shared v0.0.28 h1:VQqL3+ePwKSUxCOiCC8DpOEgbb2GhXI8XzFB github.com/frikky/shuffle-shared v0.0.28/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= github.com/frikky/shuffle-shared v0.0.32 h1:Uy/zcAetSVYtRr3HEkUb7aE7Ggm0oSFxVeUNsi6q4uc= github.com/frikky/shuffle-shared v0.0.32/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= +github.com/frikky/shuffle-shared v0.0.37 h1:6nN1Im22TBuWUCG5L619xTgEPIPXCY68QThIfNiuG8k= +github.com/frikky/shuffle-shared v0.0.37/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= github.com/getkin/kin-openapi v0.8.0 h1:a6TQjTqwkyscC4/hShJX7WhCVE+4bi9lzw61XHQW5hE= github.com/getkin/kin-openapi v0.8.0/go.mod h1:zZQMFkVgRHCdhgb6ihCTIo9dyDZFvX0k/xAKqw1FhPw= github.com/getkin/kin-openapi v0.52.0 h1:6WqsF5d6PfJ8AscdD+9Rtb2RP2iBWyC7V6GcjssWg7M= diff --git a/docker-compose.yml b/docker-compose.yml index b17892a9..980d6be2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,7 +47,7 @@ services: - database orborus: #build: ./functions/onprem/orborus - image: ghcr.io/frikky/shuffle-orborus:0.8.75 + image: ghcr.io/frikky/shuffle-orborus:0.8.76 container_name: shuffle-orborus hostname: shuffle-orborus networks: @@ -61,13 +61,14 @@ services: - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT} - DOCKER_API_VERSION=1.40 - - HTTP_PROXY=${SHUFFLE_HTTP_PROXY} - - HTTPS_PROXY=${SHUFFLE_HTTPS_PROXY} - - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY} - SHUFFLE_ORBORUS_EXECUTION_TIMEOUT=600 - SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME} - SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY} - SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX} + - HTTP_PROXY=${SHUFFLE_HTTP_PROXY} + - HTTPS_PROXY=${SHUFFLE_HTTPS_PROXY} + - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY} + - SHUFFLE_PASS_APP_PROXY=${SHUFFLE_PASS_APP_PROXY} - CLEANUP=${SHUFFLE_CONTAINER_AUTO_CLEANUP} restart: unless-stopped database: diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 946a827a..d7a8a833 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -1467,6 +1467,8 @@ const Workflows = (props) => { color="primary" defaultValue={newWorkflowDescription} placeholder="Description" + rows="6" + multiline margin="dense" fullWidth /> diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index 2f2136b0..539389a9 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-orborus -VERSION=0.8.75 +VERSION=0.8.76 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index c2cafa0f..89ab6dac 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -244,11 +244,11 @@ func initializeImages() { ctx := context.Background() if appSdkVersion == "" { - appSdkVersion = "0.8.60" + appSdkVersion = "0.8.75" log.Printf("[WARNING] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %s", appSdkVersion) } if workerVersion == "" { - workerVersion = "0.8.72" + workerVersion = "0.8.76" log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %s", workerVersion) } @@ -553,9 +553,11 @@ func main() { fmt.Sprintf("ENVIRONMENT_NAME=%s", environment), fmt.Sprintf("BASE_URL=%s", baseUrl), fmt.Sprintf("CLEANUP=%s", cleanupEnv), + fmt.Sprintf("SHUFFLE_PASS_APP_PROXY=%s", os.Getenv("SHUFFLE_PASS_APP_PROXY")), } - if strings.ToLower(os.Getenv("SHUFFLE_PASS_WORKER_PROXY")) != "false" { + //log.Printf("Running worker with proxy? %s", os.Getenv("SHUFFLE_PASS_WORKER_PROXY")) + if strings.ToLower(os.Getenv("SHUFFLE_PASS_WORKER_PROXY")) == "true" { env = append(env, fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY"))) env = append(env, fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY"))) } @@ -715,7 +717,7 @@ func zombiecheck(ctx context.Context, workerTimeout int) error { // Check image name if !shuffleFound { - log.Printf("Skipping: %s, %s", container.Labels, container.Image) + log.Printf("[WARNING] Zombie container skip: %#v, %s", container.Labels, container.Image) continue } //} else { diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh index d0742a19..b746fd5a 100644 --- a/functions/onprem/worker/build.sh +++ b/functions/onprem/worker/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-worker -VERSION=0.8.75 +VERSION=0.8.76 echo "Running docker build with $NAME:$VERSION" #CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin . diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 5272b052..af893ab9 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -934,6 +934,12 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { fmt.Sprintf("BASE_URL=%s", appCallbackUrl), } + if strings.ToLower(os.Getenv("SHUFFLE_PASS_APP_PROXY")) == "true" { + //log.Printf("APPENDING PROXY TO THE APP!") + env = append(env, fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY"))) + env = append(env, fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY"))) + } + // Fixes issue: // standard_init_linux.go:185: exec user process caused "argument list too long" // https://devblogs.microsoft.com/oldnewthing/20100203-00/?p=15083 @@ -1255,7 +1261,7 @@ func handleDefaultExecution(client *http.Client, req *http.Request, workflowExec for { //fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s/executions/%s/abort", baseUrl, workflowExecution.Workflow.ID, workflowExecution.ExecutionId) fullUrl := fmt.Sprintf("%s/api/v1/streams/results", baseUrl) - log.Printf("URL: %s", fullUrl) + //log.Printf("[INFO] URL: %s", fullUrl) req, err := http.NewRequest( "POST", fullUrl, @@ -1597,7 +1603,7 @@ func getWorkflowExecution(ctx context.Context, id string) (*shuffle.WorkflowExec func sendResult(workflowExecution shuffle.WorkflowExecution, data []byte) { if workflowExecution.ExecutionSource == "default" { - log.Printf("Not sending backend info since source is default") + log.Printf("[INFO] Not sending backend info since source is default") return }