diff --git a/.env b/.env index 33c6c318..35edd59a 100644 --- a/.env +++ b/.env @@ -65,7 +65,6 @@ SHUFFLE_CHAT_DISABLED=false # Controls support chat SHUFFLE_RERUN_SCHEDULE=300 SHUFFLE_DISABLE_RERUN_AND_ABORT=false SHUFFLE_WORKER_SERVER_URL= # Definition in case Worker & Orborus is talking to the wrong server -SHUFFLE_WORKER_SERVER_URL= # Definition in case Worker & Orborus is talking to the wrong server SHUFFLE_ORBORUS_PULL_TIME= # Definition in case Orborus is pulling too often/not often enough # DATABASE CONFIGURATIONS diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 787bf8b5..955c0f0b 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -467,7 +467,7 @@ class AppBase: finished = False for i in range (0, 10): try: - ret = requests.post(url, headers=headers, json=action_result, timeout=10) + ret = requests.post(url, headers=headers, json=action_result, timeout=10, verify=False) self.logger.info(f"[DEBUG] Result: {ret.status_code} (break on 200 or 201)") if ret.status_code == 200 or ret.status_code == 201: @@ -514,7 +514,7 @@ class AppBase: action_result["status"] = "FAILURE" action_result["result"] = json.dumps({"success": False, "reason": "POST error: Failed connecting to %s over 10 retries to the backend" % url}) self.logger.info(f"[DEBUG] Before typeerror stream result - NOT finished after 10 requests") - ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result) + ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result, verify=False) self.logger.info(f"""[DEBUG] Successful request result request: Status= {ret.status_code} & Response= {ret.text}. Action status: {action_result["status"]}""") except requests.exceptions.ConnectionError as e: @@ -524,7 +524,7 @@ class AppBase: action_result["result"] = json.dumps({"success": False, "reason": "Typeerror when sending to backend URL %s" % url}) self.logger.info(f"[DEBUG] Before typeerror stream result: {e}") - ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result) + ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result, verify=False) #self.logger.info(f"[DEBUG] Result: {ret.status_code}") #if ret.status_code != 200: # pr @@ -653,7 +653,7 @@ class AppBase: #self.logger.info(f"RET: {ret.text}") #self.logger.info(f"ID: {ret.status_code}") url = f"{self.url}/api/v1/orgs/{org_id}/validate_app_values" - ret = requests.post(url, json=data) + ret = requests.post(url, json=data, verify=False) if ret.status_code == 200: json_value = ret.json() if len(json_value["found"]) > 0: @@ -1129,7 +1129,7 @@ class AppBase: "User-Agent": "Shuffle 1.1.0", } - ret = requests.get("%s%s" % (self.url, get_path), headers=headers) + ret = requests.get("%s%s" % (self.url, get_path), headers=headers, verify=False) return ret.json() #if ret1.status_code != 200: # return { @@ -1155,7 +1155,7 @@ class AppBase: "User-Agent": "Shuffle 1.1.0", } - ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers) + ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers, verify=False) if ret1.status_code != 200: return None @@ -1218,7 +1218,7 @@ class AppBase: "User-Agent": "Shuffle 1.1.0", } - ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers) + ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers, verify=False) self.logger.info("RET1 (file get): %s" % ret1.text) if ret1.status_code != 200: returns.append({ @@ -1229,7 +1229,7 @@ class AppBase: continue content_path = "/api/v1/files/%s/content?execution_id=%s" % (item, full_execution["execution_id"]) - ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers) + ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers, verify=False) self.logger.info("RET2 (file get) done") if ret2.status_code == 200: tmpdata = ret1.json() @@ -1265,7 +1265,7 @@ class AppBase: "value": str(value), } - response = requests.post(url, json=data) + response = requests.post(url, json=data, verify=False) try: allvalues = response.json() allvalues["key"] = key @@ -1287,7 +1287,7 @@ class AppBase: "key": key, } - value = requests.post(url, json=data) + value = requests.post(url, json=data, verify=False) try: allvalues = value.json() self.logger.info("VAL1: ", allvalues) @@ -1341,7 +1341,7 @@ class AppBase: self.logger.info(f"KeyError in file setup: {e}") pass - ret = requests.post("%s%s" % (self.url, create_path), headers=headers, json=data) + ret = requests.post("%s%s" % (self.url, create_path), headers=headers, json=data, verify=False) #self.logger.info(f"Ret CREATE: {ret.text}") cur_id = "" if ret.status_code == 200: @@ -1373,7 +1373,7 @@ class AppBase: files={"shuffle_file": (filename, curfile["data"])} #open(filename,'rb')} - ret = requests.post("%s%s" % (self.url, upload_path), files=files, headers=new_headers) + ret = requests.post("%s%s" % (self.url, upload_path), files=files, headers=new_headers, verify=False) self.logger.info("Ret UPLOAD: %s" % ret.text) self.logger.info("Ret2 UPLOAD: %d" % ret.status_code) @@ -1440,7 +1440,7 @@ class AppBase: # FIXME: Shouldn't skip this, but it's good for minimzing API calls #try: - # ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result) + # ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result, verify=False) # self.logger.info("Workflow: %d" % ret.status_code) # if ret.status_code != 200: # self.logger.info(ret.text) @@ -1469,7 +1469,8 @@ class AppBase: ret = requests.post( "%s/api/v1/streams/results" % (self.base_url), headers=headers, - json=tmpdata + json=tmpdata, + verify=False ) if ret.status_code == 200: diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 8e754571..ff5c31d3 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -19,7 +19,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.3.52 + github.com/shuffle/shuffle-shared v0.3.59 golang.org/x/crypto v0.3.0 google.golang.org/api v0.103.0 google.golang.org/appengine v1.6.7 diff --git a/docker-compose.yml b/docker-compose.yml index 73881519..f64e5899 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -58,7 +58,7 @@ services: security_opt: - seccomp:unconfined opensearch: - image: opensearchproject/opensearch:2.4.0 + image: opensearchproject/opensearch:2.5.0 hostname: shuffle-opensearch container_name: shuffle-opensearch environment: diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index ce48bd6a..0f3264cb 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -8,7 +8,7 @@ require ( github.com/docker/docker v23.0.0+incompatible github.com/mackerelio/go-osstat v0.2.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.3.52 + github.com/shuffle/shuffle-shared v0.3.59 ) require ( diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 6c968756..a2892bc1 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -929,8 +929,6 @@ func main() { zombiecheck(ctx, workerTimeout) log.Printf("[INFO] Running towards %s (BASE_URL) with environment name %s", baseUrl, environment) - httpProxy := os.Getenv("HTTP_PROXY") - httpsProxy := os.Getenv("HTTPS_PROXY") if environment == "" { environment = "onprem" @@ -965,27 +963,7 @@ func main() { log.Printf("[INFO] Finished configuring docker environment") - // FIXME - time limit - client := &http.Client{ - Transport: &http.Transport{ - Proxy: nil, - }, - } - - //getStats() - - if (len(httpProxy) > 0 || len(httpsProxy) > 0) && baseUrl != "http://shuffle-backend:5001" { - client = &http.Client{} - } else { - if len(httpProxy) > 0 { - log.Printf("[INFO] Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy) - } - if len(httpsProxy) > 0 { - log.Printf("[INFO] Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy) - } - } - - client.Timeout = 30 * time.Second + client := shuffle.GetExternalClient(baseUrl) fullUrl := fmt.Sprintf("%s/api/v1/workflows/queue", baseUrl) req, err := http.NewRequest( diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index a40c77a7..a545a936 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -11,7 +11,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.3.52 + github.com/shuffle/shuffle-shared v0.3.59 ) require ( diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 63ad7a8d..ee3c2766 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -144,24 +144,7 @@ func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason } req.Header.Add("Content-Type", "application/json") - client := &http.Client{ - Transport: &http.Transport{ - Proxy: nil, - }, - } - - httpProxy := os.Getenv("HTTP_PROXY") - httpsProxy := os.Getenv("HTTPS_PROXY") - if (len(httpProxy) > 0 || len(httpsProxy) > 0) && baseUrl != "http://shuffle-backend:5001" { - client = &http.Client{} - } else { - if len(httpProxy) > 0 { - log.Printf("[INFO][%s] Running with HTTP proxy %s (env: HTTP_PROXY)", workflowExecution.ExecutionId, httpProxy) - } - if len(httpsProxy) > 0 { - log.Printf("[INFO][%s] Running with HTTPS proxy %s (env: HTTPS_PROXY)", workflowExecution.ExecutionId, httpsProxy) - } - } + client := shuffle.GetExternalClient(baseUrl) //log.Printf("[DEBUG][%s] All App Logs: %#v", workflowExecution.ExecutionId, allLogs) _, err = client.Do(req) @@ -2023,24 +2006,7 @@ func main() { log.Printf("[INFO] Setting up worker environment") sleepTime := 5 - client := &http.Client{ - Transport: &http.Transport{ - Proxy: nil, - }, - } - - httpProxy := os.Getenv("HTTP_PROXY") - httpsProxy := os.Getenv("HTTPS_PROXY") - if (len(httpProxy) > 0 || len(httpsProxy) > 0) && baseUrl != "http://shuffle-backend:5001" { - client = &http.Client{} - } else { - if len(httpProxy) > 0 { - log.Printf("Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy) - } - if len(httpsProxy) > 0 { - log.Printf("Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy) - } - } + client := shuffle.GetExternalClient(baseUrl) if timezone == "" { timezone = "Europe/Amsterdam"