From e3a8f1553e0164192f6cb5c945a94bd9cfaefac3 Mon Sep 17 00:00:00 2001 From: frikky Date: Thu, 24 Feb 2022 02:06:38 +0100 Subject: [PATCH] Fixed retries in sdk, file ordering in admin and editing issues in app creator --- backend/app_sdk/app_base.py | 64 ++++++++++++++++++++++------- backend/app_sdk/build.sh | 2 +- docker-compose.yml | 3 +- frontend/src/views/Admin.jsx | 2 +- frontend/src/views/AppCreator.jsx | 21 ++++------ functions/onprem/orborus/orborus.go | 6 ++- 6 files changed, 65 insertions(+), 33 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 040f1ba4..5595b379 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -39,8 +39,16 @@ for key, value in standard_filter_manager.filters.items(): @shuffle_filters.register def plus(a, b): - a = int(a) - b = int(b) + try: + a = int(a) + except: + a = 0 + + try: + b = int(b) + except: + b = 0 + return standard_filter_manager.filters["plus"](a, b) @shuffle_filters.register @@ -274,11 +282,42 @@ class AppBase: self.logger.info(f"[DEBUG] Before last stream result") url = "%s%s" % (self.base_url, stream_path) self.logger.info("[INFO] URL FOR RESULT (URL): %s" % url) + + # FIXME: Adding retries here. try: - ret = requests.post(url, headers=headers, json=action_result) - #self.logger.info(f"[DEBUG] Result: {ret.status_code}") - #if ret.status_code != 200: - # self.logger.info(f"[DEBUG] Shuffle Response: {ret.text}") + finished = False + for i in range (0, 5): + try: + ret = requests.post(url, headers=headers, json=action_result, timeout=10) + + self.logger.info(f"[DEBUG] Result: {ret.status_code} (break on 200)") + if ret.status_code == 200 or ret.status_code == 201: + finished = True + break + else: + self.logger.info(f"[DEBUG] RESP: {ret.text}") + + except (requests.exceptions.RequestException, TimeoutError) as e: + time.sleep(5) + continue + except requests.exceptions.ConnectionError as e: + time.sleep(5) + continue + except http.client.RemoteDisconnected as e: + time.sleep(5) + continue + except urllib3.exceptions.ProtocolError as e: + time.sleep(5) + continue + + time.sleep(5) + + if not finished: + # Not sure why this would work tho :) + action_result["status"] = "FAILURE" + action_result["result"] = f"POST error: {e}" + self.logger.info(f"[DEBUG] Before typeerror stream result: {e}") + ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result) self.logger.info(f"""[DEBUG] Successful request result request: Status= {ret.status_code} & Response= {ret.text}. Action status: {action_result["status"]}""") except requests.exceptions.ConnectionError as e: @@ -2445,7 +2484,7 @@ class AppBase: # THE START IS ACTUALLY RIGHT HERE :O # Checks whether conditions are met, otherwise set branchcheck, tmpresult = check_branch_conditions(action, fullexecution, self) - if isinstance(tmpresult, object) or isinstance(tmpresult, list): + if isinstance(tmpresult, object) or isinstance(tmpresult, list) or isinstance(tmpresult, dict): self.logger.info("[DEBUG] Fixing branch return as object -> string") try: #tmpresult = tmpresult.replace("'", "\"") @@ -2453,19 +2492,14 @@ class AppBase: except json.decoder.JSONDecodeError as e: self.logger.info(f"[WARNING] Failed condition parsing {tmpresult} to string") + # IF branches fail: Exit! if not branchcheck: self.logger.info("Failed one or more branch conditions.") self.action_result["result"] = tmpresult self.action_result["status"] = "SKIPPED" - try: - ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=self.action_result) - self.logger.info("Result: %d" % ret.status_code) - if ret.status_code != 200: - self.logger.info(ret.text) - except requests.exceptions.ConnectionError as e: - self.logger.exception(e) + self.action_result["completed_at"] = int(time.time()) - self.logger.info("\n\n[DEBUG] RETURNING BECAUSE A BRANCH FAILED: %s\n\n" % tmpresult) + self.send_result(self.action_result, headers, stream_path) return # Replace name cus there might be issues diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index a8be1c76..4cfe3e2c 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -3,7 +3,7 @@ ### DEFAULT NAME=shuffle-app_sdk -VERSION=0.9.59 +VERSION=0.9.61 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly diff --git a/docker-compose.yml b/docker-compose.yml index 7b2bf6c3..02e07e9a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: frontend: - build: ./frontend + #build: ./frontend image: ghcr.io/frikky/shuffle-frontend:nightly container_name: shuffle-frontend hostname: shuffle-frontend @@ -94,4 +94,3 @@ networks: driver: bridge #driver: overlay -#driver: bridge diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index 81d7e623..14b153ed 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -3185,7 +3185,7 @@ const Admin = (props) => { } return ( - + { const [urlPathQueries, setUrlPathQueries] = useState([]); const [update, setUpdate] = useState(""); const [urlPathParameters] = useState([]); - const [firstrequest, setFirstrequest] = React.useState(true); const [basedata, setBasedata] = React.useState({}); const [actions, setActions] = useState([]); const [filteredActions, setFilteredActions] = useState([]); @@ -343,16 +342,13 @@ const AppCreator = (defaultprops) => { window.location.host === "shuffler.io"; useEffect(() => { - if (firstrequest) { - setFirstrequest(false); - if (window.location.pathname.includes("apps/edit")) { - setIsEditing(true); - handleEditApp(); - } else { - checkQuery(); - } - } - }); + if (window.location.pathname.includes("apps/edit")) { + setIsEditing(true); + handleEditApp(); + } else { + checkQuery(); + } + }, []); const handleEditApp = () => { fetch(globalUrl + "/api/v1/apps/" + props.match.params.appid + "/config", { @@ -1635,6 +1631,7 @@ const AppCreator = (defaultprops) => { if (urlParams !== undefined && urlParams !== null && urlParams.has("id")) { data.id = urlParams.get("id") } + //id: props.match.params.appid, } @@ -5267,7 +5264,7 @@ const AppCreator = (defaultprops) => { ); const loadedCheck = - isLoaded && isAppLoaded && !firstrequest ? ( + isLoaded && isAppLoaded ? (
{landingpageDataBrowser}
{newActionModal} diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 93316155..ee733a77 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -201,7 +201,7 @@ func deployServiceWorkers(image string) { // Looks for and cleans up all existing items in swarm we can't re-use (Shuffle only) cleanupExistingNodes(ctx) // frikky@debian:~/git/shuffle/functions/onprem/worker$ docker service create --replicas 5 --name shuffle-workers --env SHUFFLE_SWARM_CONFIG=run --publish published=33333,target=33333 ghcr.io/frikky/shuffle-worker:nightly - networkName := "shuffle-executions" + networkName := "shuffle_swarm_executions" if len(swarmNetworkName) > 0 { networkName = swarmNetworkName } @@ -809,7 +809,7 @@ func main() { // Run by default from now zombiecheck(ctx, workerTimeout) - log.Printf("[INFO] Running towards %s (BASE_URL) with Org %s", baseUrl, environment) + log.Printf("[INFO] Running towards %s (BASE_URL) with environment name %s", baseUrl, environment) httpProxy := os.Getenv("HTTP_PROXY") httpsProxy := os.Getenv("HTTPS_PROXY") @@ -857,6 +857,8 @@ func main() { } } + client.Timeout = 10 * time.Second + fullUrl := fmt.Sprintf("%s/api/v1/workflows/queue", baseUrl) req, err := http.NewRequest( "GET",