From 1334a292fa4d0bda7a85ad3bb4c12f96c603b6cc Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 18 Mar 2022 10:56:24 +0100 Subject: [PATCH] Added NO_PROXY passing to apps and worker --- backend/app_sdk/app_base.py | 3 ++- functions/extensions/aws-s3-lambda/s3_function.py | 3 ++- functions/onprem/orborus/orborus.go | 13 ++++++++++--- functions/onprem/worker/build.sh | 2 +- functions/onprem/worker/worker.go | 10 ++++++++++ 5 files changed, 25 insertions(+), 6 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 3e889533..7937387c 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -279,6 +279,7 @@ class AppBase: # Try it with some magic + action_result["completed_at"] = int(time.time()) self.logger.info(f"""[DEBUG] Inside Send result with status {action_result["status"]}""") #if isinstance(action_result, @@ -300,7 +301,7 @@ class AppBase: self.logger.info(f"[INFO] URL FOR RESULT (URL): {url}") try: - log_contents = "disabled" + log_contents = "disabled: add env SHUFFLE_LOGS_DISABLED=true to Orborus to re-enable logs for apps" if not os.getenv("SHUFFLE_LOGS_DISABLED") == "true": log_contents = self.log_capture_string.getvalue() diff --git a/functions/extensions/aws-s3-lambda/s3_function.py b/functions/extensions/aws-s3-lambda/s3_function.py index 82838a4e..d83d6154 100644 --- a/functions/extensions/aws-s3-lambda/s3_function.py +++ b/functions/extensions/aws-s3-lambda/s3_function.py @@ -16,6 +16,7 @@ def lambda_handler(event, context): http = urllib3.PoolManager() ret = http.request('POST', webhook, body=json.dumps(event["Records"][0]).encode("utf-8")) if ret.status != 200: - return "Bad status code for webhook: %d" % ret.status_code + return "Bad status code for webhook: %d" % ret.status print("Status code: %d\nData: %s" % (ret.status, ret.data)) + return "Successfully started with data %s" % ret.data diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 56831574..4eb9c9fe 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -398,6 +398,12 @@ func deployServiceWorkers(image string) { serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_SCALE_REPLICAS=%s", os.Getenv("SHUFFLE_SCALE_REPLICAS"))) } + if strings.ToLower(os.Getenv("SHUFFLE_PASS_WORKER_PROXY")) == "true" { + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY"))) + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY"))) + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("NO_PROXY=%s", os.Getenv("NO_PROXY"))) + } + if len(os.Getenv("DOCKER_HOST")) > 0 { serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("DOCKER_HOST=%s", os.Getenv("DOCKER_HOST"))) } else { @@ -568,10 +574,10 @@ func deployWorker(image string, identifier string, env []string, executionReques } if err != nil { - log.Printf("[ERROR] Failed to start container in environment %s: %s", environment, err) + log.Printf("[ERROR] Failed to start worker container in environment %s: %s", environment, err) return err } else { - log.Printf("[INFO] Container %s was created under environment %s for execution %s", cont.ID, environment, executionRequest.ExecutionId) + log.Printf("[INFO] Worker Container %s was created under environment %s for execution %s: docker logs %s", cont.ID, environment, executionRequest.ExecutionId, cont.ID) } //stats, err := cli.ContainerInspect(context.Background(), containerName) @@ -596,7 +602,7 @@ func deployWorker(image string, identifier string, env []string, executionReques // } //} } else { - log.Printf("[INFO] Container %s was created under environment %s", cont.ID, environment) + log.Printf("[INFO] Worker Container %s was created under environment %s: docker logs %s", cont.ID, environment, cont.ID) } return nil @@ -1040,6 +1046,7 @@ func main() { if strings.ToLower(os.Getenv("SHUFFLE_PASS_WORKER_PROXY")) == "true" { env = append(env, fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY"))) env = append(env, fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY"))) + env = append(env, fmt.Sprintf("NO_PROXY=%s", os.Getenv("NO_PROXY"))) } if dockerApiVersion != "" { diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh index 0a9fb0ff..88fcbc9e 100644 --- a/functions/onprem/worker/build.sh +++ b/functions/onprem/worker/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-worker -VERSION=0.9.62 +VERSION=0.9.64 echo "Running docker build with $NAME:$VERSION" #CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin . diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 29e911ff..efec6e73 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -1237,6 +1237,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { //log.Printf("APPENDING PROXY TO THE APP!") env = append(env, fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY"))) env = append(env, fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY"))) + env = append(env, fmt.Sprintf("NO_PROXY=%s", os.Getenv("NO_PROXY"))) } // Fixes issue: @@ -1940,6 +1941,7 @@ func runUserInput(client *http.Client, action shuffle.Action, workflowId string, //log.Printf("APPENDING PROXY TO THE APP!") env = append(env, fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY"))) env = append(env, fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY"))) + env = append(env, fmt.Sprintf("NO_PROXY=%s", os.Getenv("NO_PROXY"))) } err = deployApp(dockercli, "frikky/shuffle:shuffle-subflow_1.0.0", identifier, env, workflowExecution, newAction) @@ -2666,6 +2668,13 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl Target: "shuffle_shuffle", }) } + + if strings.ToLower(os.Getenv("SHUFFLE_PASS_APP_PROXY")) == "true" { + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY"))) + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY"))) + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("NO_PROXY=%s", os.Getenv("NO_PROXY"))) + } + /* Mounts: []mount.Mount{ mount.Mount{ @@ -3016,6 +3025,7 @@ func baseDeploy() { //log.Printf("APPENDING PROXY TO THE APP!") env = append(env, fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY"))) env = append(env, fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY"))) + env = append(env, fmt.Sprintf("NO_PROXY=%s", os.Getenv("NO_PROXY"))) } identifier := fmt.Sprintf("%s_%s_%s_%s", appname, appversion, action.ID, workflowExecution.ExecutionId)