From caaf4a5a40807e4e9f352114b3398977d31e6f94 Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 16 Nov 2023 18:22:53 +0100 Subject: [PATCH 01/33] Added Queue size to Environments to understand size of queue currently --- backend/go-app/go.mod | 4 +- docker-compose.yml | 6 +- frontend/src/views/Admin.jsx | 103 ++++++++++++++++++----------------- 3 files changed, 58 insertions(+), 55 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index a69a127b..71086381 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -1,6 +1,6 @@ module shuffle-shared -//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared +replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared go 1.19 @@ -19,7 +19,7 @@ require ( github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 github.com/shuffle/shuffle-shared v0.4.79 - golang.org/x/crypto v0.9.0 + golang.org/x/crypto v0.14.0 google.golang.org/api v0.125.0 google.golang.org/grpc v1.55.0 gopkg.in/src-d/go-git.v4 v4.13.1 diff --git a/docker-compose.yml b/docker-compose.yml index 8bde6ff6..c35a6138 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: frontend: - image: ghcr.io/shuffle/shuffle-frontend:latest + image: ghcr.io/shuffle/shuffle-frontend:nightly container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -15,7 +15,7 @@ services: depends_on: - backend backend: - image: ghcr.io/shuffle/shuffle-backend:latest + image: ghcr.io/shuffle/shuffle-backend:nightly container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: @@ -34,7 +34,7 @@ services: - SHUFFLE_FILE_LOCATION=/shuffle-files restart: unless-stopped orborus: - image: ghcr.io/shuffle/shuffle-orborus:latest + image: ghcr.io/shuffle/shuffle-orborus:nightly container_name: shuffle-orborus hostname: shuffle-orborus networks: diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index 9631fd1e..565597cc 100755 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -3941,7 +3941,7 @@ If you're interested, please let me know a time that works for you, or set up a style={{ minWidth: 125, maxWidth: 125 }} /> 100 ? ">100" : environment.queue : 0 + return ( @@ -4091,7 +4093,8 @@ If you're interested, please let me know a time that works for you, or set up a overflow: "hidden", marginLeft: 10, }} - primary={environment.archived.toString()} + + primary={isCloud ? environment.archived.toString() : queueSize} />
- - - + - + if (isCloud && environment.Name.toLowerCase() === "cloud") { + rerunCloudWorkflows(environment); + } else { + abortEnvironmentWorkflows(environment); + } + }} + color="primary" + > + {isCloud && environment.Name.toLowerCase() === "cloud" ? "Rerun" : "Clear"} + +
- {showCPUAlert === false ? null : + {showCPUAlert === false ? null : -
-
- - 90% CPU the server(s) hosting the Shuffle App Runner (Orborus) was found. - - - Need help with High Availability and Scale? Read documentation and Get in touch. - -
-
- -
-
- - } - +
+
+ + 90% CPU the server(s) hosting the Shuffle App Runner (Orborus) was found. + + + Need help with High Availability and Scale? Read documentation and Get in touch. + +
+
+ +
+
+ + } + ); })} From d1bf64ac74b9b6560c8170067b38f1aad0b277a0 Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 16 Nov 2023 19:00:14 +0100 Subject: [PATCH 02/33] Added workflow run debugger search properly to onprem and added queue tracker for environments --- backend/go-app/go.mod | 4 ++-- backend/go-app/go.sum | 2 ++ backend/go-app/main.go | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 71086381..6dd36169 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -1,6 +1,6 @@ module shuffle-shared -replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared +//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared go 1.19 @@ -18,7 +18,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.4.79 + github.com/shuffle/shuffle-shared v0.4.80 golang.org/x/crypto v0.14.0 google.golang.org/api v0.125.0 google.golang.org/grpc v1.55.0 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index e55ea13a..0dfe108a 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -406,6 +406,8 @@ github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shuffle/shuffle-shared v0.4.66 h1:Aw4qOp0VsVJrRzW1sJhEy4OY4fRGlFErUD5+93RXL6g= github.com/shuffle/shuffle-shared v0.4.66/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= +github.com/shuffle/shuffle-shared v0.4.80 h1:03OL+O8prwL9zq6Gnb9SRORPWi5+ThO0jPoxk+xctOo= +github.com/shuffle/shuffle-shared v0.4.80/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 3fbd4f70..e7756dc0 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -4824,6 +4824,7 @@ func initHandlers() { /* Everything below here increases the counters*/ r.HandleFunc("/api/v1/workflows", shuffle.GetWorkflows).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/workflows", shuffle.SetNewWorkflow).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/workflows/search", shuffle.HandleWorkflowRunSearch).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/schedules", shuffle.HandleGetSchedules).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/executions", shuffle.GetWorkflowExecutions).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/executions/{key}/abort", shuffle.AbortExecution).Methods("GET", "OPTIONS") From 3c34e36be4aa741d182b948cc4ad3478e16c4c9e Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 16 Nov 2023 19:51:56 +0100 Subject: [PATCH 03/33] Added catchall and str() for regex --- backend/app_sdk/app_base.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 298be08f..9174cc0d 100755 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -2803,9 +2803,12 @@ class AppBase: return False elif check.lower() == "re" or check.lower() == "matches regex": try: - found = re.search(destinationvalue, sourcevalue) + found = re.search(str(destinationvalue), str(sourcevalue)) except re.error as e: - print("[WARNING] Regex error in condition: %s" % e) + print("[WARNING] Regex error in condition (re.error): %s" % e) + return False + except Exception as e: + print("[WARNING] Regex error in condition (catchall): %s" % e) return False if found == None: From deba8f15a274c1589111149986ea03a89d59c20d Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 17 Nov 2023 01:31:10 +0100 Subject: [PATCH 04/33] Made startnode also care about conditions --- backend/app_sdk/app_base.py | 13 +++++++------ backend/go-app/go.mod | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 9174cc0d..9892aa95 100755 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -2829,12 +2829,13 @@ class AppBase: return True, "" # Startnode should always run - no need to check incoming - try: - if action["id"] == fullexecution["start"]: - return True, "" - except Exception as error: - self.logger.info(f"[WARNING] Failed checking startnode: {error}") - return True, "" + # Removed November 2023 due to people wanting startnode to also check + #try: + # if action["id"] == fullexecution["start"]: + # return True, "" + #except Exception as error: + # self.logger.info(f"[WARNING] Failed checking startnode: {error}") + # return True, "" available_checks = [ "=", diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 6dd36169..14cecd74 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -1,6 +1,6 @@ module shuffle-shared -//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared +replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared go 1.19 From 7c90456ba39651a6b37d09b2bb14aa067b8a12da Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 17 Nov 2023 01:44:11 +0100 Subject: [PATCH 05/33] Minor tests with skipping of startnode in app --- backend/app_sdk/app_base.py | 41 +++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 9892aa95..d3ae8d8c 100755 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -2830,12 +2830,41 @@ class AppBase: # Startnode should always run - no need to check incoming # Removed November 2023 due to people wanting startnode to also check - #try: - # if action["id"] == fullexecution["start"]: - # return True, "" - #except Exception as error: - # self.logger.info(f"[WARNING] Failed checking startnode: {error}") - # return True, "" + # This is to make it possible ot + try: + if action["id"] == fullexecution["start"]: + pass + # Need to validate if the source is a trigger or not + # need to remove branches that are not from trigger to the startnode to make it all work + + #if "workflow" in fullexecution["workflow"] and "triggers" in fullexecution["workflow"]: + # cnt = 0 + # found_branch_indexes = [] + # for branch in fullexecution["workflow"]["branches"]: + # if branch["destination_id"] != action["id"]: + # continue + + # # Check if the source is a trigger + # # if we can't find it as trigger, remove the branch + # print("Found relevant branch: %s" % branch) + # for action in fullexecution["workflow"]["actions"]: + # if action["id"] == branch["source_id"]: + # found_branch_indexes.append(branch["source_id"]) + # break + + # if len(found_branch_indexes) > 0: + # for i in sorted(found_branch_indexes, reverse=True): + # fullexecution["workflow"]["branches"].pop(i) + + # print("Removed %d branches" % len(found_branch_indexes)) + #else: + # print("[WARNING] No branches or triggers found in fullexecution for startnode") + + #if should_skip: + # return True, "" + except Exception as error: + self.logger.info(f"[WARNING] Failed checking startnode: {error}") + #return True, "" available_checks = [ "=", From d6e37aa3fe3386b6ca6a21b36365165be113c528 Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 17 Nov 2023 01:51:34 +0100 Subject: [PATCH 06/33] Removed the startnode changes again. Too many possible failures --- backend/app_sdk/app_base.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index d3ae8d8c..1d75a027 100755 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -2833,10 +2833,10 @@ class AppBase: # This is to make it possible ot try: if action["id"] == fullexecution["start"]: - pass + return True, "" + # Need to validate if the source is a trigger or not # need to remove branches that are not from trigger to the startnode to make it all work - #if "workflow" in fullexecution["workflow"] and "triggers" in fullexecution["workflow"]: # cnt = 0 # found_branch_indexes = [] @@ -2859,12 +2859,10 @@ class AppBase: # print("Removed %d branches" % len(found_branch_indexes)) #else: # print("[WARNING] No branches or triggers found in fullexecution for startnode") - - #if should_skip: - # return True, "" except Exception as error: self.logger.info(f"[WARNING] Failed checking startnode: {error}") #return True, "" + #return True, "" available_checks = [ "=", From a0711dd453313a4113f42fb2fb71fc0b0726910d Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 17 Nov 2023 05:19:18 +0100 Subject: [PATCH 07/33] Fixed a few UI inconsistencies --- frontend/src/components/ParsedAction.jsx | 6 ++ frontend/src/components/RuntimeDebugger.jsx | 84 +++++++++++++++++- frontend/src/views/Admin.jsx | 97 +++++++++++---------- frontend/src/views/AngularWorkflow.jsx | 23 ++++- 4 files changed, 157 insertions(+), 53 deletions(-) diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index c069499f..e10c6e5f 100755 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -3409,6 +3409,12 @@ const ParsedAction = (props) => { setSelectedActionEnvironment(env); selectedAction.environment = env.Name; setSelectedAction(selectedAction); + + for (let actionkey in workflow.actions) { + workflow.actions[actionkey].environment = env.Name + } + setWorkflow(workflow) + toast("Set environment for ALL actions to " + env.Name) }} style={{ backgroundColor: theme.palette.inputColor, diff --git a/frontend/src/components/RuntimeDebugger.jsx b/frontend/src/components/RuntimeDebugger.jsx index fa149570..8bb82514 100644 --- a/frontend/src/components/RuntimeDebugger.jsx +++ b/frontend/src/components/RuntimeDebugger.jsx @@ -21,6 +21,7 @@ import theme from '../theme.jsx'; import dayjs from 'dayjs'; import { AdapterDayjs } from '@mui/x-date-pickers/AdapterDayjs' import Pagination from '@mui/material/Pagination'; +import { triggers as alltriggers } from "../views/AngularWorkflow.jsx" import { DatePicker, DateTimePicker, @@ -29,6 +30,7 @@ import { import { OpenInNew as OpenInNewIcon, + PlayArrow as PlayArrowIcon, } from '@mui/icons-material'; import { DataGrid, GridColDef, GridValueGetterParams } from '@mui/x-data-grid' @@ -176,11 +178,42 @@ const RuntimeDebugger = (props) => { } }, []) + const imageSize = 30 const columns: GridColDef[] = [ + { + field: 'execution_source', + headerName: 'Source', + width: 75, + renderCell: (params) => { + var foundSource = + + var source = params.row.execution_source + if (source === "schedule") { + foundSource = schedule + } else if (source === "webhook") { + foundSource = webhook + } else if (source === "subflow" || source.length === 36) { + foundSource = subflow + source = "subflow" + } else { + source = "manual" + } + + return ( + { + //setStatus(params.row.status) + }}> + + {foundSource} + + + ) + }, + }, { field: 'status', headerName: 'Status', - width: 150, + width: 100, renderCell: (params) => ( { setStatus(params.row.status) @@ -208,6 +241,7 @@ const RuntimeDebugger = (props) => { ), }, + { field: 'workflow results', headerName: 'Results', @@ -235,6 +269,52 @@ const RuntimeDebugger = (props) => { ) }, }, + { + field: 'finished', + headerName: 'Finished', + width: 75, + renderCell: (params) => { + var foundItems = 0 + var extraItems = 0 + if (params.row.results !== null && params.row.results !== undefined) { + for (let key in params.row.results) { + if (params.row.results[key].status === "SUCCESS") { + foundItems += 1 + } + } + } + + return ( + { + }}> + {foundItems} + + ) + }, + }, + { + field: 'skipped', + headerName: 'Skipped', + width: 75, + renderCell: (params) => { + var foundItems = 0 + var extraItems = 0 + if (params.row.results !== null && params.row.results !== undefined) { + for (let key in params.row.results) { + if (params.row.results[key].status === "SKIPPED") { + foundItems += 1 + } + } + } + + return ( + { + }}> + {foundItems} + + ) + }, + }, { field: 'startTimestamp', headerName: 'Start time', width: 160, }, { field: 'endTimestamp', headerName: 'End time', width: 160, }, { @@ -288,7 +368,7 @@ const RuntimeDebugger = (props) => { } return ( -
+

Workflow Run Debugger

{ submitSearch(workflowId, status, startTime, endTime, rowCursor, rowsPerPage) diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index 565597cc..1aab3b4c 100755 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -3937,21 +3937,21 @@ If you're interested, please let me know a time that works for you, or set up a style={{ minWidth: 125, maxWidth: 125 }} /> + - {environments === undefined || environments === null ? null @@ -3969,20 +3969,20 @@ If you're interested, please let me know a time that works for you, or set up a bgColor = "#1f2023"; } - // Check if there's a notification for it in userdata.priorities - var showCPUAlert = false - var foundIndex = -1 - if (userdata !== undefined && userdata !== null && userdata.priorities !== undefined && userdata.priorities !== null && userdata.priorities.length > 0) { - foundIndex = userdata.priorities.findIndex(prio => prio.name.includes("CPU") && prio.active === true) + // Check if there's a notification for it in userdata.priorities + var showCPUAlert = false + var foundIndex = -1 + if (userdata !== undefined && userdata !== null && userdata.priorities !== undefined && userdata.priorities !== null && userdata.priorities.length > 0) { + foundIndex = userdata.priorities.findIndex(prio => prio.name.includes("CPU") && prio.active === true) - if (foundIndex >= 0 && userdata.priorities[foundIndex].name.endsWith(environment.Name)) { - showCPUAlert = true - } - } + if (foundIndex >= 0 && userdata.priorities[foundIndex].name.endsWith(environment.Name)) { + showCPUAlert = true + } + } - console.log("Show CPU alert: ", showCPUAlert) + //console.log("Show CPU alert: ", showCPUAlert) - const queueSize = isCloud ? -1 : environment.queue !== undefined && environment.queue !== null ? environment.queue < 0 ? 0 : environment.queue > 100 ? ">100" : environment.queue : 0 + const queueSize = environment.queue !== undefined && environment.queue !== null ? environment.queue < 0 ? 0 : environment.queue > 99 ? ">99" : environment.queue : 0 return ( @@ -4069,8 +4069,18 @@ If you're interested, please let me know a time that works for you, or set up a /> + setDefaultEnvironment(environment)} color="primary" > - Make default + Set Default )} - - {isCloud && environment.Name.toLowerCase() === "cloud" ? "Rerun" : "Clear"} +
+ {showCPUAlert === false ? null : diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 8133d485..cb973f31 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -395,9 +395,10 @@ const AngularWorkflow = (defaultprops) => { var to_be_copied = ""; const [firstrequest, setFirstrequest] = React.useState(true); const [cystyle] = useState(cytoscapestyle); + const [cy, setCy] = React.useState(); - const [toolsApp, setToolsApp] = React.useState({}); + const [toolsApp, setToolsApp] = React.useState({}); const [currentView, setCurrentView] = React.useState(0); const [triggerAuthentication, setTriggerAuthentication] = React.useState({}); const [triggerFolders, setTriggerFolders] = React.useState([]); @@ -6344,6 +6345,9 @@ const AngularWorkflow = (defaultprops) => { } const fetchRecommendations = (inputWorkflow) => { + //console.log("Disabled recommendations") + //return + const parsedWorkflow = JSON.parse(JSON.stringify(inputWorkflow)) fetch(globalUrl + "/api/v1/workflows/recommend", { @@ -14992,6 +14996,18 @@ const AngularWorkflow = (defaultprops) => { ) : null}
+ {executionData.workflow !== undefined && executionData.workflow !== null && executionData.workflow.actions !== undefined && executionData.workflow.actions !== null && executionData.workflow.actions.length > 0 && executionData.workflow.actions[0].environment !== "Cloud" ? +
+ + Env      + + { + window.open("/admin?tab=environments", "_blank") + }}> + {executionData.workflow.actions[0].environment} + +
+ : null} {executionData.status !== undefined && executionData.status.length > 0 ? (
@@ -15973,8 +15989,9 @@ const AngularWorkflow = (defaultprops) => { cy={(incy) => { // FIXME: There's something specific loading when // you do the first hover of a node. Why is this different? - //console.log("CY: ", incy) - setCy(incy); + + + setCy(incy); }} /> From c1bfb159c63dbd8259204f80415d6abf9f0769d2 Mon Sep 17 00:00:00 2001 From: Frikky Date: Sun, 19 Nov 2023 18:03:53 +0100 Subject: [PATCH 08/33] Fixed oauth2, docs and popups for onboarding --- backend/app_sdk/app_base.py | 29 +--- frontend/src/components/AppSearchButtons.jsx | 33 +++-- frontend/src/components/Oauth2Auth.jsx | 99 +++++++++++-- frontend/src/components/SearchData.jsx | 138 +++++++----------- frontend/src/components/Searchfield.jsx | 16 +- .../src/components/WorkflowTemplatePopup.jsx | 7 +- frontend/src/views/AngularWorkflow.jsx | 19 ++- frontend/src/views/AppCreator.jsx | 113 ++++++++++---- frontend/src/views/Dashboard.jsx | 112 +++++++++----- frontend/src/views/Docs.jsx | 126 ++++------------ 10 files changed, 367 insertions(+), 325 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 1d75a027..84a0d9a5 100755 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -3585,36 +3585,9 @@ class AppBase: }) break - - - - #thread = threading.Thread(target=func, args=(**params,)) - #thread.start() - - #thread.join(timeout) - - #if thread.is_alive(): - # # The thread is still running, so we need to stop it - # # You can handle this as needed, such as raising an exception - # timeout_handler() - - - #with Timeout(timeout): - # newres = func(**params) - # break - #except Timeout.Timeout as e: - # self.logger.info(f"[DEBUG] Timeout error: {e}") - # newres = json.dumps({ - # "success": False, - # "reason": "Timeout error within %d seconds. This typically happens if we can't reach the API you're trying to reach." % timeout, - # "exception": str(e), - # }) - - # break - except TypeError as e: newres = "" - self.logger.info(f"[DEBUG] Got exec type error: {e}") + self.logger.info(f"[ERROR] Got function exec type error: {e}") try: e = json.loads(f"{e}") except: diff --git a/frontend/src/components/AppSearchButtons.jsx b/frontend/src/components/AppSearchButtons.jsx index dbd2c055..553b7e41 100644 --- a/frontend/src/components/AppSearchButtons.jsx +++ b/frontend/src/components/AppSearchButtons.jsx @@ -47,8 +47,6 @@ const AppSearchButtons = (props) => { const [newSelectedApp, setNewSelectedApp] = useState(undefined) useEffect(() => { - console.log("AppSearchButtons: newSelectedApp: " + JSON.stringify(newSelectedApp)) - if (newSelectedApp !== undefined && setMissing != undefined) { console.log("AppSearchButtons: setMissing is defined!") @@ -131,9 +129,12 @@ const AppSearchButtons = (props) => { //setFrameworkLoaded(true) }) } + const icon = foundApp.large_image - console.log("index:", moreButton) - console.log("totalApps:", totalApps) + var foundAppImage = AppImage + if (foundApp.name !== undefined && foundApp.name !== null && !foundApp.name.includes(":default")) { + foundAppImage = foundApp.large_image + } let xsValue = 12; if (index === totalApps - 1 || index === totalApps - 2 || index === totalApps - 3 || index === totalApps - 4) { @@ -142,6 +143,8 @@ const AppSearchButtons = (props) => { if (index === totalApps - 5) { xsValue = 12; } + + // This is silly huh if (moreButton) { switch (index) { case totalApps - 1: @@ -159,7 +162,6 @@ const AppSearchButtons = (props) => { xsValue = 12; break; default: - // Handle other cases if needed } } @@ -221,10 +223,11 @@ const AppSearchButtons = (props) => { > { e.preventDefault(); setLocalSearchOpen(false) - setDefaultSearch("") + const submitDeletedApp = { "description": "", "id": "remove", @@ -233,15 +236,23 @@ const AppSearchButtons = (props) => { } setFrameworkItem(submitDeletedApp) setNewSelectedApp({}) + + if (setDefaultSearch !== undefined) { + setDefaultSearch("") + } + setTimeout(() => { - setDiscoveryData({}) + if (setDiscoveryData !== undefined) { + setDiscoveryData({}) + } + setFrameworkItem(submitDeletedApp) //setNewSelectedApp({}) }, 1000) //setAppName(discoveryData.cases.name) }} > - +
@@ -283,12 +294,12 @@ const AppSearchButtons = (props) => { }} >
- {AppImage === undefined || AppImage === null || AppImage.length === 0 ? -
+ {foundAppImage === undefined || foundAppImage === null || foundAppImage.length === 0 ? +
: - + }
{ authenticationType.client_secret.length > 0 ); - const [clientId, setClientId] = React.useState( - defaultConfigSet ? authenticationType.client_id : "" - ); - const [clientSecret, setClientSecret] = React.useState( - defaultConfigSet ? authenticationType.client_secret : "" - ); + console.log("AUTH: ", authenticationType) + + const [clientId, setClientId] = React.useState(defaultConfigSet ? authenticationType.client_id : ""); + const [clientSecret, setClientSecret] = React.useState(defaultConfigSet ? authenticationType.client_secret : ""); + + const [username, setUsername] = React.useState(""); + const [password, setPassword] = React.useState(""); + const [oauthUrl, setOauthUrl] = React.useState(""); const [buttonClicked, setButtonClicked] = React.useState(false); - const [offlineAccess, setOfflineAccess] = React.useState(true); - const allscopes = authenticationType.scope !== undefined ? authenticationType.scope : []; + const allscopes = authenticationType.scope !== undefined && authenticationType.scope !== null ? authenticationType.scope : []; + const [selectedScopes, setSelectedScopes] = React.useState(allscopes !== null && allscopes !== undefined ? allscopes.length > 0 && allscopes.length <= 3 ? [allscopes[0]] : [] : []) - const [selectedScopes, setSelectedScopes] = React.useState(allscopes.length > 0 && allscopes.length <= 3 ? [allscopes[0]] : []) const [manuallyConfigure, setManuallyConfigure] = React.useState( defaultConfigSet ? false : true ); @@ -158,6 +159,7 @@ const AuthenticationOauth2 = (props) => { return null; } + const startOauth2Request = (admin_consent) => { // Admin consent also means to add refresh tokens console.log("Inside oauth2 request for app: ", selectedApp.name) @@ -319,6 +321,32 @@ const AuthenticationOauth2 = (props) => { "value": authenticationType.token_uri, }] + if (authenticationType.grant_type !== undefined && authenticationType.grant_type !== null && authenticationType.grant_type.length > 0) { + if (authenticationType.grant_type === "client_credentials") { + parsedFields.push({ + "key": "grant_type", + "value": authenticationType.grant_type, + }) + } else if (authenticationType.grant_type === "password") { + parsedFields.push({ + "key": "grant_type", + "value": authenticationType.grant_type, + }) + + parsedFields.push({ + "key": "username", + "value": username, + }) + + parsedFields.push({ + "key": "password", + "value": password, + }) + } else { + toast("Unknown grant type: " + authenticationType.grant_type) + } + } + const appAuthData = { "label": "OAuth2 for " + selectedApp.name, "app": { @@ -478,8 +506,6 @@ const AuthenticationOauth2 = (props) => { } return; - //do { - //} while ( }; authenticationOption.app.actions = []; @@ -647,7 +673,7 @@ const AuthenticationOauth2 = (props) => { )} - if (authButtonOnly === true) { + if (authButtonOnly === true && (authenticationType.grant_type === undefined || authenticationType.grant_type === null || authenticationType.grant_type === "")) { return autoAuthButton } @@ -872,8 +898,51 @@ const AuthenticationOauth2 = (props) => { //authenticationOption.label = event.target.value }} /> - {allscopes.length === 0 ? null : "Scopes (access rights)"} - {allscopes.length === 0 ? null : ( + + {authenticationType.grant_type !== "password" ? null : +
+ { + setUsername(event.target.value); + //authenticationOption.label = event.target.value + }} + /> + { + setPassword(event.target.value); + //authenticationOption.label = event.target.value + }} + /> +
+ } + + {allscopes === undefined || allscopes === null || allscopes.length === 0 ? null : "Scopes (access rights)"} + {allscopes === undefined || allscopes === null || allscopes.length === 0 ? null : (
{authenticationOption === "Oauth2" ? - + {/* - Delegated: The user will get a popup for access their personal data. - Application: Permissions are set by the app creator in the 3rd party platform. */} - { + setOauth2Type(e.target.value); + }} + value={oauth2Type} + style={{ + backgroundColor: inputColor, + color: "white", + height: "50px", + }} + > + {["delegated", "application"].map((data, index) => ( + + {data} + + ))} + +
+ + {oauth2Type === "application" ? +
+ Grant Type + + {["client_credentials", "password"].map((data, index) => ( + + {data} + + ))} + +
+ : null} : null} diff --git a/frontend/src/views/Dashboard.jsx b/frontend/src/views/Dashboard.jsx index f961076b..21c2d513 100755 --- a/frontend/src/views/Dashboard.jsx +++ b/frontend/src/views/Dashboard.jsx @@ -102,6 +102,8 @@ const UsecaseListComponent = (props) => { const [expandedItem, setExpandedItem] = useState(-1); const [inputUsecase, setInputUsecase] = useState({}); + const [prevSubcase, setPrevSubcase] = useState({}) + const [editing, setEditing] = useState(false); const [description, setDescription] = useState(""); const [video, setVideo] = useState(""); @@ -117,6 +119,42 @@ const UsecaseListComponent = (props) => { const [mitreTags, setMitreTags] = useState([]); + const parseUsecase = (subcase) => { + const srcdata = findSpecificApp(frameworkData, subcase.type) + const dstdata = findSpecificApp(frameworkData, subcase.last) + + if (srcdata !== undefined && srcdata !== null) { + subcase.srcimg = srcdata.large_image + subcase.srcapp = srcdata.name + } + + if (dstdata !== undefined && dstdata !== null) { + subcase.dstimg = dstdata.large_image + subcase.dstapp = dstdata.name + } + + return subcase + } + + useEffect(() => { + console.log("In frameworkData useEffect: frameworkData: ", frameworkData) + if (frameworkData === undefined || prevSubcase === undefined) { + return + } + + console.log("PAST!") + + var parsedUsecase = inputUsecase + const subcase = parseUsecase(prevSubcase) + + parsedUsecase.srcimg = subcase.srcimg + parsedUsecase.srcapp = subcase.srcapp + parsedUsecase.dstimg = subcase.dstimg + parsedUsecase.dstapp = subcase.dstapp + + setInputUsecase(parsedUsecase) + }, [frameworkData]) + const loadApps = () => { fetch(`${globalUrl}/api/v1/apps`, { method: "GET", @@ -163,35 +201,14 @@ const UsecaseListComponent = (props) => { if (keys === undefined || keys === null || keys.length === 0) { return null - } - - - const parseUsecase = (subcase) => { - //console.log("parseUsecase: ", subcase) - const srcdata = findSpecificApp(frameworkData, subcase.type) - const dstdata = findSpecificApp(frameworkData, subcase.last) - - if (srcdata !== undefined && srcdata !== null) { - subcase.srcimg = srcdata.large_image - subcase.srcapp = srcdata.name - } - - if (dstdata !== undefined && dstdata !== null) { - subcase.dstimg = dstdata.large_image - subcase.dstapp = dstdata.name - } - - return subcase - } + } + + // Timeout 50ms to delay it slightly const getUsecase = (subcase, index, subindex) => { subcase = parseUsecase(subcase) - - // Timeout 50ms to delay it slightly - //setTimeout(() => { - // setInputUsecase(subcase) - //}, 50) + setPrevSubcase(subcase) fetch(`${globalUrl}/api/v1/workflows/usecases/${escape(subcase.name.replaceAll(" ", "_"))}`, { method: "GET", @@ -214,8 +231,6 @@ const UsecaseListComponent = (props) => { if (responseJson.success === false) { parsedUsecase = subcase } else { - console.log("FOUND: ", JSON.parse(JSON.stringify(responseJson))) - parsedUsecase = responseJson parsedUsecase.srcimg = subcase.srcimg @@ -314,7 +329,7 @@ const UsecaseListComponent = (props) => { }) .catch((error) => { //toast(error.toString()); - //setFrameworkLoaded(true) + //setFrameworkLoaded(true) }) } @@ -416,15 +431,16 @@ const UsecaseListComponent = (props) => { return ( { + if (fixedName === "increase authentication") { + getUsecase(subcase, index, subindex) + return + } //setSelectedWorkflows([]) if (selectedItem) { } else { getUsecase(subcase, index, subindex) navigate(`/usecases?selected_object=${fixedName}`) - - //const newitem = removeParam("selected_object", cursearch); - //navigate(curpath + newitem) } }}> { @@ -594,11 +610,12 @@ const UsecaseListComponent = (props) => { > { - setExpandedItem(-1) - setExpandedIndex(-1) - setEditing(false) - setInputUsecase({}) + setExpandedItem(-1) + setExpandedIndex(-1) + setEditing(false) + setInputUsecase({}) }} > @@ -1172,13 +1189,28 @@ const Dashboard = (props) => { if (foundQuery !== null && foundQuery !== undefined) { setSelectedUsecaseCategory(foundQuery) - const newitem = removeParam("selected", cursearch); + const newitem = removeParam("selected", cursearch); navigate(curpath + newitem) } + const baseItem = document.getElementById("increase authentication") + if (baseItem !== undefined && baseItem !== null) { + baseItem.click() + + // Find close window button -> go to top + const foundButton = document.getElementById("close_selection") + if (foundButton !== undefined && foundButton !== null) { + foundButton.click() + } + + // Scroll back to top + window.scrollTo(0, 0) + } + const foundQuery2 = params["selected_object"] if (foundQuery2 !== null && foundQuery2 !== undefined) { - //console.log("Got selected_object: ", foundQuery2) + // Take a random object, quickly click it, then go to this one + // Something is weird with loading apps without it const queryName = foundQuery2.toLowerCase().replaceAll("_", " ") // Waiting a bit for it to render @@ -1198,7 +1230,7 @@ const Dashboard = (props) => { } else { //console.log("Couldn't find item with name ", queryName) } - }, 100); + }, 1000); } } @@ -1251,6 +1283,7 @@ const Dashboard = (props) => { }) } + const getAvailableWorkflows = () => { fetch(globalUrl + "/api/v1/workflows", { method: "GET", @@ -1385,8 +1418,7 @@ const Dashboard = (props) => { useEffect(() => { getAvailableWorkflows() - getFramework() - //fetchUsecases() + getFramework() }, []); const fetchdata = (stats_id) => { diff --git a/frontend/src/views/Docs.jsx b/frontend/src/views/Docs.jsx index ba56b835..350853ee 100755 --- a/frontend/src/views/Docs.jsx +++ b/frontend/src/views/Docs.jsx @@ -1,13 +1,11 @@ import React, { useEffect, useState } from "react"; -import ReactMarkdown from "react-markdown"; +import Markdown from 'react-markdown' + import { BrowserView, MobileView } from "react-device-detect"; import { useParams, useNavigate, Link } from "react-router-dom"; import { isMobile } from "react-device-detect"; import theme from '../theme.jsx'; -import remarkGfm from 'remark-gfm' -import KeyboardArrowRightIcon from '@mui/icons-material/KeyboardArrowRight'; -import ExpandMoreIcon from '@mui/icons-material/ExpandMore'; import { Grid, @@ -29,6 +27,8 @@ import { import { Link as LinkIcon, Edit as EditIcon, + KeyboardArrowRight as KeyboardArrowRightIcon, + ExpandMore as ExpandMoreIcon, } from "@mui/icons-material"; const Body = { @@ -166,8 +166,11 @@ const Docs = (defaultprops) => { }) .then((response) => response.json()) .then((responseJson) => { - if (responseJson.success) { - setData(responseJson.reason); + if (responseJson.success && responseJson.reason !== undefined) { + // Find tags and translate them into ![]() format + const imgRegex = / { }; function OuterLink(props) { - console.log("Link: ", props.href) if (props.href.includes("http") || props.href.includes("mailto")) { return ( { } function CodeHandler(props) { - console.log("PROPS: ", props) + //console.log("Codehandler PROPS: ", props) const propvalue = props.value !== undefined && props.value !== null ? props.value : props.children !== undefined && props.children !== null && props.children.length > 0 ? props.children[0] : "" @@ -603,7 +605,6 @@ const Docs = (defaultprops) => { const [hover, setHover] = useState(false); - console.log("Link: ", link) if (link === undefined || link === null) { return null } @@ -681,60 +682,20 @@ const Docs = (defaultprops) => { Organize. Whether an organization of 1000 or 1, management tools are necessary. In Shuffle we offer full user management, MFA and single-signon options, multi-tenancy and a lot more - for free!
- - {/* - - {list.map((data, index) => { - const item = data.name; - if (item === undefined) { - return null; - } - - const path = "/docs/" + item; - const newname = - item.charAt(0).toUpperCase() + - item.substring(1).split("_").join(" ").split("-").join(" "); - - const itemMatching = props.match.params.key === undefined ? false : - props.match.params.key.toLowerCase() === item.toLowerCase(); - - return ( - - - - ) - })} - - */} - - {/* - { - console.log("Change: ", event.target.value) - }} - /> - */}
+ const markdownComponents = { + img: Img, + code: CodeHandler, + h1: Heading, + h2: Heading, + h3: Heading, + h4: Heading, + h5: Heading, + h6: Heading, + a: OuterLink, + } + // PostDataBrowser Section const postDataBrowser = list === undefined || list === null ? null : ( @@ -812,32 +773,22 @@ const Docs = (defaultprops) => { mainpageInfo :
- {data} - +
}
); - // remarkPlugins={[remarkGfm]} const mobileStyle = { color: "white", @@ -849,6 +800,7 @@ const Docs = (defaultprops) => { flexDirection: "column", }; + const postDataMobile = list === undefined || list === null ? null : (
@@ -899,18 +851,8 @@ const Docs = (defaultprops) => { mainpageInfo :
- { }} > {data} - +
} {
); - //const imageModal = - // - // {imageModal} - // Padding and zIndex etc set because of footer in cloud. const loadedCheck = ( -
+
{postDataBrowser} {postDataMobile}
From 8c6a39f658eadfda38f3cc0a82af278eeee26335 Mon Sep 17 00:00:00 2001 From: Frikky Date: Mon, 20 Nov 2023 13:33:44 +0100 Subject: [PATCH 09/33] Made orborus use latest shuffle-shared to get SHUFFLE_INTERNAL_HTTP_PROXY utilized. This is still missing for the SDK --- functions/onprem/orborus/go.mod | 2 +- functions/onprem/orborus/go.sum | 2 ++ functions/onprem/orborus/orborus.go | 36 +++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index f74e495b..97d638d9 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -9,7 +9,7 @@ require ( github.com/mackerelio/go-osstat v0.2.3 github.com/satori/go.uuid v1.2.0 github.com/shirou/gopsutil v3.21.11+incompatible - github.com/shuffle/shuffle-shared v0.4.62 + github.com/shuffle/shuffle-shared v0.4.86 k8s.io/api v0.28.1 k8s.io/apimachinery v0.28.1 k8s.io/client-go v0.28.1 diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index 70af6a61..cd9f9880 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -272,6 +272,8 @@ github.com/shuffle/shuffle-shared v0.4.59 h1:5Sv8aorgQJFZr3cCKltfycdXzp9v5zlF2l3 github.com/shuffle/shuffle-shared v0.4.59/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= github.com/shuffle/shuffle-shared v0.4.62 h1:L76zWCD/7gIBuhr3feWZwzT4I8VCiLRd8ZAub/3EiO0= github.com/shuffle/shuffle-shared v0.4.62/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= +github.com/shuffle/shuffle-shared v0.4.86 h1:QrFx3j+maUgeU/dP48WMx+NBcWdwe3Ov2yKlLUUZRHw= +github.com/shuffle/shuffle-shared v0.4.86/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index e5b7dc8a..050620b7 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -556,6 +556,23 @@ func deployServiceWorkers(image string) { } } + // Look for SHUFFLE_VOLUME_BINDS + if len(os.Getenv("SHUFFLE_VOLUME_BINDS")) > 0 { + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_VOLUME_BINDS=%s", os.Getenv("SHUFFLE_VOLUME_BINDS"))) + } + + overrideHttpProxy := os.Getenv("SHUFFLE_INTERNAL_HTTP_PROXY") + overrideHttpsProxy := os.Getenv("SHUFFLE_INTERNAL_HTTPS_PROXY") + if len(overrideHttpProxy) > 0 { + log.Printf("[DEBUG] Added internal proxy: %s", overrideHttpProxy) + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_INTERNAL_HTTP_PROXY=%s", overrideHttpProxy)) + } + + if len(overrideHttpsProxy) > 0 { + log.Printf("[DEBUG] Added internal proxy: %s", overrideHttpsProxy) + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_INTERNAL_HTTPS_PROXY=%s", overrideHttpsProxy)) + } + serviceOptions := types.ServiceCreateOptions{} _, err = dockercli.ServiceCreate( ctx, @@ -1574,6 +1591,25 @@ func main() { env = append(env, fmt.Sprintf("SHUFFLE_DEBUG_MEMORY=%s", os.Getenv("SHUFFLE_DEBUG_MEMORY"))) } + // Look for volume binds + if len(os.Getenv("SHUFFLE_VOLUME_BINDS")) > 0 { + log.Printf("[DEBUG] Added volume binds: %s", os.Getenv("SHUFFLE_VOLUME_BINDS")) + env = append(env, fmt.Sprintf("SHUFFLE_VOLUME_BINDS=%s", os.Getenv("SHUFFLE_VOLUME_BINDS"))) + } + + // Setting up internal proxy config for Shuffle -> shuffle comms + overrideHttpProxy := os.Getenv("SHUFFLE_INTERNAL_HTTP_PROXY") + overrideHttpsProxy := os.Getenv("SHUFFLE_INTERNAL_HTTPS_PROXY") + if len(overrideHttpProxy) > 0 { + log.Printf("[DEBUG] Added internal proxy: %s", overrideHttpProxy) + env = append(env, fmt.Sprintf("HTTP_PROXY=%s", overrideHttpProxy)) + } + + if len(overrideHttpsProxy) > 0 { + log.Printf("[DEBUG] Added internal proxy: %s", overrideHttpsProxy) + env = append(env, fmt.Sprintf("HTTPS_PROXY=%s", overrideHttpsProxy)) + } + err = deployWorker(workerImage, containerName, env, execution) zombiecounter += 1 if err == nil { From 1fd46a7307c2b19b94b37b8b30ed9fe161f6c57d Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 24 Nov 2023 12:03:20 +0100 Subject: [PATCH 10/33] Many fixes for frontend and backend related to Oauth2 and environment control --- backend/go-app/main.go | 15 +- backend/go-app/walkoff.go | 11 +- frontend/package.json | 1 + frontend/src/components/EditWorkflow.jsx | 59 ++-- frontend/src/components/NewHeader.jsx | 7 +- frontend/src/components/Oauth2Auth.jsx | 60 ++-- frontend/src/components/ParsedAction.jsx | 8 + frontend/src/components/RuntimeDebugger.jsx | 20 +- frontend/src/components/ShuffleCodeEditor.jsx | 270 +++++++++--------- frontend/src/views/Admin.jsx | 162 +++++++---- frontend/src/views/AngularWorkflow.jsx | 149 +++++++--- frontend/src/views/AppCreator.jsx | 10 +- frontend/src/views/Docs.jsx | 4 +- frontend/src/views/Workflows.jsx | 20 +- 14 files changed, 494 insertions(+), 302 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index e7756dc0..33383850 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -1965,6 +1965,7 @@ func executeCloudAction(action shuffle.CloudSyncJob, apikey string) error { return err } + defer newresp.Body.Close() respBody, err := ioutil.ReadAll(newresp.Body) if err != nil { return err @@ -3513,15 +3514,13 @@ func remoteOrgJobHandler(org shuffle.Org, interval int) error { ) req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, org.SyncConfig.Apikey)) - - //log.Printf("[INFO] Sending org sync with autho %s", org.SyncConfig.Apikey) - newresp, err := client.Do(req) if err != nil { //log.Printf("Failed request in org sync: %s", err) return err } + defer newresp.Body.Close() respBody, err := ioutil.ReadAll(newresp.Body) if err != nil { log.Printf("[ERROR] Failed body read in job sync: %s", err) @@ -3574,6 +3573,8 @@ func runInitEs(ctx context.Context) { log.Printf("[DEBUG] Getting organizations for Elasticsearch/Opensearch") activeOrgs, err := shuffle.GetAllOrgs(ctx) + log.Printf("[DEBUG] Got %d organizations to look into", len(activeOrgs)) + setUsers := false _ = setUsers if err != nil { @@ -3697,7 +3698,7 @@ func runInitEs(ctx context.Context) { for _, schedule := range schedules { if strings.ToLower(schedule.Environment) == "cloud" { - log.Printf("Skipping cloud schedule") + log.Printf("[DEBUG] Skipping cloud schedule") continue } @@ -3705,7 +3706,9 @@ func runInitEs(ctx context.Context) { //log.Printf("Schedule time: every %d seconds", schedule.Seconds) jobret, err := newscheduler.Every(schedule.Seconds).Seconds().NotImmediately().Run(job(schedule)) if err != nil { - log.Printf("Failed to schedule workflow: %s", err) + log.Printf("[ERROR] Failed to start schedule for workflow %s: %s", schedule.WorkflowId, err) + } else { + log.Printf("[DEBUG] Successfully started schedule for workflow %s", schedule.WorkflowId) } scheduledJobs[schedule.Id] = jobret @@ -4725,7 +4728,7 @@ func initHandlers() { log.Printf("[DEBUG] Initialized Shuffle database connection. Setting up environment.") if elasticConfig == "elasticsearch" { - time.Sleep(5 * time.Second) + time.Sleep(10 * time.Second) go runInitEs(ctx) } else { //go shuffle.runInit(ctx) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 32fb67ef..83656aa4 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -705,7 +705,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { // Will make sure transactions are always ran for an execution. This is recursive if it fails. Allowed to fail up to 5 times func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workflowExecutionId string, actionResult shuffle.ActionResult, resp http.ResponseWriter) { - log.Printf("[DEBUG] Running workflow execution transaction for %s", workflowExecutionId) + log.Printf("[DEBUG][%s] Running workflow execution update", workflowExecutionId) // Should start a tx for the execution here workflowExecution, err := shuffle.GetWorkflowExecution(ctx, workflowExecutionId) @@ -1063,10 +1063,6 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request } } - err = shuffle.SetWorkflowExecution(ctx, workflowExecution, true) - if err != nil { - log.Printf("[ERROR] Failed setting workflow execution during init (2): %s", err) - } err = imageCheckBuilder(execInfo.ImageNames) if err != nil { @@ -1573,6 +1569,11 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request workflowExecution.ExecutionOrg = workflow.ExecutingOrg.Id } + err = shuffle.SetWorkflowExecution(ctx, workflowExecution, true) + if err != nil { + log.Printf("[ERROR] Failed setting workflow execution during init (2): %s", err) + } + var allEnvs []shuffle.Environment if len(workflowExecution.ExecutionOrg) > 0 { //log.Printf("[INFO] Executing ORG: %s", workflowExecution.ExecutionOrg) diff --git a/frontend/package.json b/frontend/package.json index 0cb2e764..fbababec 100755 --- a/frontend/package.json +++ b/frontend/package.json @@ -14,6 +14,7 @@ "@mui/styles": "^5.14.0", "@mui/x-data-grid": "^5.17.11", "@mui/x-date-pickers": "^6.11.1", + "@uiw/codemirror-theme-vscode": "^4.21.20", "@uiw/codemirror-themes": "^4.21.9", "@uiw/react-codemirror": "^4.21.9", "@use-it/interval": "^1.0.0", diff --git a/frontend/src/components/EditWorkflow.jsx b/frontend/src/components/EditWorkflow.jsx index 81dacd85..05e57184 100644 --- a/frontend/src/components/EditWorkflow.jsx +++ b/frontend/src/components/EditWorkflow.jsx @@ -269,35 +269,8 @@ const EditWorkflow = (props) => { />
- { - console.log("Chip: ", chip) - //newWorkflowTags.push(chip); - setNewWorkflowTags(chip); - }} - onAdd={(chip) => { - newWorkflowTags.push(chip); - setNewWorkflowTags(newWorkflowTags); - }} - onDelete={(chip, index) => { - console.log("Deleting: ", chip, index) - newWorkflowTags.splice(index, 1); - setNewWorkflowTags(newWorkflowTags); - setUpdate(Math.random()); - }} - /> {usecases !== null && usecases !== undefined && usecases.length > 0 ? - + Usecases : null} + { + console.log("Chip: ", chip) + //newWorkflowTags.push(chip); + setNewWorkflowTags(chip); + }} + onAdd={(chip) => { + newWorkflowTags.push(chip); + setNewWorkflowTags(newWorkflowTags); + }} + onDelete={(chip, index) => { + console.log("Deleting: ", chip, index) + newWorkflowTags.splice(index, 1); + setNewWorkflowTags(newWorkflowTags); + setUpdate(Math.random()); + }} + />
{showMoreClicked === true ? @@ -365,7 +365,8 @@ const EditWorkflow = (props) => { onChange={(e) => { console.log("Data: ", e.target.value) - innerWorkflow.workflow_type = e.target.value + //innerWorkflow.workflow_type = e.target.value + innerWorkflow.status = e.target.value setInnerWorkflow(innerWorkflow) }} > diff --git a/frontend/src/components/NewHeader.jsx b/frontend/src/components/NewHeader.jsx index 7b72f0bd..004b3132 100644 --- a/frontend/src/components/NewHeader.jsx +++ b/frontend/src/components/NewHeader.jsx @@ -1352,14 +1352,17 @@ const Header = (props) => { ); // - return !isMobile ? - isLoggedIn ? + // + /* + !isLoggedIn ?
{loginTextBrowser}
: + */ + return !isMobile ? { authenticationType.client_secret.length > 0 ); - console.log("AUTH: ", authenticationType) - const [clientId, setClientId] = React.useState(defaultConfigSet ? authenticationType.client_id : ""); const [clientSecret, setClientSecret] = React.useState(defaultConfigSet ? authenticationType.client_secret : ""); @@ -125,7 +123,7 @@ const AuthenticationOauth2 = (props) => { const [offlineAccess, setOfflineAccess] = React.useState(true); const allscopes = authenticationType.scope !== undefined && authenticationType.scope !== null ? authenticationType.scope : []; - const [selectedScopes, setSelectedScopes] = React.useState(allscopes !== null && allscopes !== undefined ? allscopes.length > 0 && allscopes.length <= 3 ? [allscopes[0]] : [] : []) + const [selectedScopes, setSelectedScopes] = React.useState(allscopes !== null && allscopes !== undefined ? allscopes.length > 0 && allscopes.length <= 3 ? allscopes : [] : []) const [manuallyConfigure, setManuallyConfigure] = React.useState( defaultConfigSet ? false : true @@ -303,6 +301,23 @@ const AuthenticationOauth2 = (props) => { if ((authenticationType.redirect_uri === undefined || authenticationType.redirect_uri === null || authenticationType.redirect_uri.length === 0) && (authenticationType.token_uri !== undefined && authenticationType.token_uri !== null && authenticationType.token_uri.length > 0)) { console.log("No redirect URI found, and token URI found. Assuming client credentials flow and saving directly in the database") + + var tokenUri = authenticationType.token_uri; + if (oauthUrl !== undefined && oauthUrl !== null && oauthUrl.length > 0 && selectedApp !== undefined && selectedApp !== null) { + var same = false + for (var i = 0; i < selectedApp.authentication.parameters.length; i++) { + const param = selectedApp.authentication.parameters[i]; + if (param.name === "url" && (param.value === oauthUrl || param.example === oauthUrl)) { + same = true + break + } + } + + if (!same) { + tokenUri = oauthUrl + } + } + // Find app.configuration=true fields in the app.paramters var parsedFields = [{ "key": "client_id", @@ -318,7 +333,7 @@ const AuthenticationOauth2 = (props) => { }, { "key": "token_uri", - "value": authenticationType.token_uri, + "value": tokenUri, }] if (authenticationType.grant_type !== undefined && authenticationType.grant_type !== null && authenticationType.grant_type.length > 0) { @@ -360,14 +375,14 @@ const AuthenticationOauth2 = (props) => { "reference_workflow": workflowId, } - setNewAppAuth(appAuthData) + setNewAppAuth(appAuthData, true) + // Wait 1 second, then get app auth with update - // - if (getAppAuthentication !== undefined) { - setTimeout(() => { - getAppAuthentication(true, true, true); - }, 1000) - } + //if (getAppAuthentication !== undefined) { + // setTimeout(() => { + // getAppAuthentication(true, true, true); + // }, 1000) + //} return } @@ -397,8 +412,6 @@ const AuthenticationOauth2 = (props) => { } const authentication_url = authenticationType.token_uri; - //console.log("AUTH: ", authenticationType) - //console.log("SCOPES2: ", resources) const redirectUri = `${window.location.protocol}//${window.location.host}/set_authentication`; const workflowId = workflow !== undefined ? workflow.id : ""; var state = `workflow_id%3D${workflowId}%26reference_action_id%3d${selectedAction.app_id}%26app_name%3d${selectedAction.app_name}%26app_id%3d${selectedAction.app_id}%26app_version%3d${selectedAction.app_version}%26authentication_url%3d${authentication_url}%26scope%3d${resources}%26client_id%3d${client_id}%26client_secret%3d${client_secret}`; @@ -523,7 +536,6 @@ const AuthenticationOauth2 = (props) => { } const handleSubmitCheck = () => { - console.log("NEW AUTH: ", authenticationOption); if (authenticationOption.label.length === 0) { authenticationOption.label = `Auth for ${selectedApp.name}`; //toast("Label can't be empty") @@ -673,7 +685,7 @@ const AuthenticationOauth2 = (props) => { )} - if (authButtonOnly === true && (authenticationType.grant_type === undefined || authenticationType.grant_type === null || authenticationType.grant_type === "")) { + if (authButtonOnly === true && (authenticationType.redirect_uri !== undefined && authenticationType.redirect_uri !== null && authenticationType.redirect_uri.length > 0) && (authenticationType.token_uri !== undefined && authenticationType.token_uri !== null && authenticationType.token_uri.length > 0)) { return autoAuthButton } @@ -777,10 +789,14 @@ const AuthenticationOauth2 = (props) => { setOauthUrl(data.value); } + const defaultValue = data.name === "url" && authenticationType.token_uri !== undefined && authenticationType.token_uri !== null && authenticationType.token_uri.length > 0 && (authenticationType.authorizationUrl === undefined || authenticationType.authorizationUrl === null || authenticationType.authorizationUrl.length === 0) ? authenticationType.token_uri : data.value === undefined || data.value === null ? "" : data.value + const fieldname = data.name === "url" && authenticationType.grant_type !== undefined && authenticationType.grant_type !== null && authenticationType.grant_type.length > 0 ? "Token URL" : data.name + return (
- {data.name} + + {fieldname} {data.schema !== undefined && data.schema !== null && @@ -793,6 +809,7 @@ const AuthenticationOauth2 = (props) => { }} defaultValue={"false"} fullWidth + label={fieldname} onChange={(e) => { console.log("Value: ", e.target.value); authenticationOption.fields[data.name] = e.target.value; @@ -842,16 +859,11 @@ const AuthenticationOauth2 = (props) => { : "text" } color="primary" - defaultValue={ - data.value !== undefined && data.value !== null - ? data.value - : "" - } + defaultValue={defaultValue} placeholder={data.example} onChange={(event) => { - authenticationOption.fields[data.name] = - event.target.value; - console.log("Setting oauth url"); + authenticationOption.fields[data.name] = event.target.value; + console.log("Setting oauth url: ", event.target.value); setOauthUrl(event.target.value); //const [oauthUrl, setOauthUrl] = React.useState("") }} diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index e10c6e5f..53f40b79 100755 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -181,6 +181,8 @@ const ParsedAction = (props) => { const [fieldCount, setFieldCount] = React.useState(0); const [hiddenDescription, setHiddenDescription] = React.useState(true); + const [autoCompleting, setAutocompleting] = React.useState(false); + useEffect(() => { if (setLastSaved !== undefined) { @@ -2909,9 +2911,11 @@ const ParsedAction = (props) => { marginLeft: 15, paddingRight: 0, }} + disabled={autoCompleting} onClick={() => { // aiSubmit(aiMsg, undefined, undefined, newSelectedAction) aiSubmit("Fill based on previous values", undefined, undefined, selectedAction) + setAutocompleting(true) }} > { title={"Autocompletes fields. Uses NAME of the action and previous values' results."} placement="top" > + {autoCompleting ? + + : + }
diff --git a/frontend/src/components/RuntimeDebugger.jsx b/frontend/src/components/RuntimeDebugger.jsx index 8bb82514..47462d2b 100644 --- a/frontend/src/components/RuntimeDebugger.jsx +++ b/frontend/src/components/RuntimeDebugger.jsx @@ -315,16 +315,26 @@ const RuntimeDebugger = (props) => { ) }, }, - { field: 'startTimestamp', headerName: 'Start time', width: 160, }, - { field: 'endTimestamp', headerName: 'End time', width: 160, }, + { field: 'startTimestamp', headerName: 'Start time (UTC)', width: 160, }, + { field: 'endTimestamp', headerName: 'End time (UTC)', width: 160, }, { field: 'id', headerName: 'Explore', width: 65, renderCell: (params) => ( - - - + + {params.row.result !== null && params.row.result !== undefined && params.row.result !== "" ? + params.row.result + : + null + } + + } > + + + + ), }, ] diff --git a/frontend/src/components/ShuffleCodeEditor.jsx b/frontend/src/components/ShuffleCodeEditor.jsx index 5fb9b152..91db5f49 100644 --- a/frontend/src/components/ShuffleCodeEditor.jsx +++ b/frontend/src/components/ShuffleCodeEditor.jsx @@ -22,6 +22,7 @@ import { isMobile } from "react-device-detect" import { NestedMenuItem } from "mui-nested-menu" import { GetParsedPaths, FindJsonPath } from "../views/Apps.jsx"; import { SetJsonDotnotation } from "../views/AngularWorkflow.jsx"; +import { vscodeDark, vscodeDarkInit } from '@uiw/codemirror-theme-vscode'; import { FullscreenExit as FullscreenExitIcon, @@ -82,6 +83,7 @@ const pythonFilters = [ {"name": "Handle JSON", "value": `{% python %}\nimport json\njsondata = json.loads(r"""$nodename""")\n{% endpython %}`, "example": ``}, ] +/* const shuffleTheme = createTheme({ theme: 'dark', settings: { @@ -110,6 +112,7 @@ const shuffleTheme = createTheme({ { tag: t.attributeName, color: '#5c6166' }, ], }); +*/ const CodeEditor = (props) => { const { @@ -547,7 +550,7 @@ const CodeEditor = (props) => { var code_lines = localcodedata.split('\n') for (var i = 0; i < code_lines.length; i++){ var current_code_line = code_lines[i] - // console.log(current_code_line) + console.log("Codeline: ", current_code_line) var variable_occurence = current_code_line.match(/[\\]{0,1}[$]{1}([a-zA-Z0-9_-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,}/g) @@ -609,9 +612,10 @@ const CodeEditor = (props) => { const fixedVariable = fixVariable(variable_occurence[occ]) var correctVariable = availableVariables.includes(fixedVariable) if(!correctVariable) { + console.log("Line: ", i, "ch: ", dollar_occurence[occ]) + value.markText({line:i, ch:dollar_occurence[occ]}, {line:i, ch:dollar_occurence_len[occ]+dollar_occurence[occ]}, {"css": "background-color: rgb(248, 106, 62, 0.9); padding-top: 2px; padding-bottom: 2px; color: white"}) - } - else{ + } else { value.markText({line:i, ch:dollar_occurence[occ]}, {line:i, ch:dollar_occurence_len[occ]+dollar_occurence[occ]}, {"css": "background-color: #8b8e26; padding-top: 2px; padding-bottom: 2px; color: white"}) } // console.log(correctVariables) @@ -674,36 +678,33 @@ const CodeEditor = (props) => { try { for (var i = 0; i < found.length; i++) { try { - // For found specifically, should replace .#\d with .# with regex - - - //found[i] = found[i].toLowerCase() const fixedVariable = fixVariable(found[i]) - //var correctVariable = availableVariables.includes(fixedVariable) - - // var valuefound = false for (var j = 0; j < actionlist.length; j++) { - if(fixedVariable.slice(1,).toLowerCase() === actionlist[j].autocomplete.toLowerCase()){ - valuefound = true + if(fixedVariable.slice(1,).toLowerCase() !== actionlist[j].autocomplete.toLowerCase()){ + continue + } - try { - if (typeof actionlist[j].example === "object") { - input = input.replace(found[i], JSON.stringify(actionlist[j].example), -1); + valuefound = true - } else if (actionlist[j].example.trim().startsWith("{") || actionlist[j].example.trim().startsWith("[")) { - input = input.replace(found[i], JSON.stringify(actionlist[j].example), -1); - } else { - input = input.replace(found[i], actionlist[j].example, -1) - } - } catch (e) { + try { + if (typeof actionlist[j].example === "object") { + input = input.replace(found[i], JSON.stringify(actionlist[j].example), -1); + + } else if (actionlist[j].example.trim().startsWith("{") || actionlist[j].example.trim().startsWith("[")) { + input = input.replace(found[i], JSON.stringify(actionlist[j].example), -1); + } else { input = input.replace(found[i], actionlist[j].example, -1) } - } else { - // Couldn't find the correct example value + } catch (e) { + input = input.replace(found[i], actionlist[j].example, -1) } } + //if (!valuefound) { + // console.log("Couldn't find value "+fixedVariable) + //} + if (!valuefound && availableVariables.includes(fixedVariable)) { var shouldbreak = false for (var k=0; k < actionlist.length; k++){ @@ -714,46 +715,48 @@ const CodeEditor = (props) => { for (var key in parsedPaths) { const fullpath = "$"+actionlist[k].autocomplete.toLowerCase()+parsedPaths[key].autocomplete - if (fullpath === fixedVariable) { - //if (actionlist[k].example === undefined) { - // actionlist[k].example = "TMP" - //} + if (fullpath !== fixedVariable) { + continue + } - var new_input = "" - try { - new_input = FindJsonPath(fullpath, actionlist[k].example) - } catch (e) { - console.log("ERR IN INPUT: ", e) - } + //if (actionlist[k].example === undefined) { + // actionlist[k].example = "TMP" + //} - //console.log("Got output for: ", fullpath, new_input, actionlist[k].example, typeof new_input) + var new_input = "" + try { + new_input = FindJsonPath(fullpath, actionlist[k].example) + } catch (e) { + console.log("ERR IN INPUT: ", e) + } - if (typeof new_input === "object") { - new_input = JSON.stringify(new_input) + console.log("Got output for: ", fullpath, new_input, actionlist[k].example, typeof new_input) + + if (typeof new_input === "object") { + new_input = JSON.stringify(new_input) + } else { + if (typeof new_input === "string") { + new_input = new_input } else { - if (typeof new_input === "string") { - new_input = new_input - } else { - console.log("NO TYPE? ", typeof new_input) - try { - new_input = new_input.toString() - } catch (e) { - new_input = "" - } + console.log("NO TYPE? ", typeof new_input) + try { + new_input = new_input.toString() + } catch (e) { + new_input = "" } } - - //console.log("FOUND2: ", fixedVariable, actionlist[j].example) - input = input.replace(fixedVariable, new_input, -1) - input = input.replace(found[i], new_input, -1) - - //} catch (e) { - // input = input.replace(found[i], actionlist[k].example) - //} - - shouldbreak = true - break } + + //console.log("FOUND2: ", fixedVariable, actionlist[j].example) + input = input.replace(fixedVariable, new_input, -1) + input = input.replace(found[i], new_input, -1) + + //} catch (e) { + // input = input.replace(found[i], actionlist[k].example) + //} + + shouldbreak = true + break } if (shouldbreak) { @@ -766,7 +769,7 @@ const CodeEditor = (props) => { } } } catch (e) { - //console.log("Outer replace error: ", e) + console.log("Outer replace error: ", e) } } @@ -895,7 +898,7 @@ const CodeEditor = (props) => { aria-labelledby="draggable-code-modal" disableBackdropClick={true} disableEnforceFocus={true} - //style={{ pointerEvents: "none" }} + //style={{ pointerEvents: "none" }} hideBackdrop={true} open={expansionModalOpen} onClose={() => { @@ -964,6 +967,7 @@ const CodeEditor = (props) => { }} >
+ {/* { > Code Editor - { - - }} - > - - - - - - - { - autoFormat(localcodedata) - }} - > - - {isAiLoading ? - - : - - } - - -
- - } - - + */} { isFileEditor ? null : -
+
} - { + + }} + > + + + + + + + { + autoFormat(localcodedata) + }} + > + + {isAiLoading ? + + : + + } + + +
+ + } + + + +
{ - // console.log(value.getCursor()) + console.log("CURSOR: ", value.getCursor()) setCurrentCharacter(value.getCursor().ch) setCurrentLine(value.getCursor().line) // console.log(value.getCursor().ch, value.getCursor().line) findIndex(value.getCursor().line, value.getCursor().ch) + highlight_variables(value) }} onChange={(value, viewUpdate) => { @@ -1422,20 +1434,19 @@ const CodeEditor = (props) => { setlocalcodedata(value) expectedOutput(value) + highlight_variables(value) + //if(value.display.input.prevInput.startsWith('$') || value.display.input.prevInput.endsWith('$')){ // setEditorPopupOpen(true) //} }} - extensions={[]}//indentWithTab]} - theme={shuffleTheme} options={{ - styleSelectedText: true, - keyMap: 'sublime', mode: validation === true ? "json" : "python", lineWrapping: linewrap, + theme: vscodeDark, }} /> - +
{/*editorPopupOpen ? { -
+
{isFileEditor ? null :
{isMobile ? null : @@ -1564,7 +1576,7 @@ const CodeEditor = (props) => { Expected Output - { + { executeSingleAction(expOutput) }}> @@ -1613,8 +1625,8 @@ const CodeEditor = (props) => { borderRadius: theme.palette.borderRadius, maxHeight: 500, minHeight: 500, - minWidth: 500, - maxWidth: 500, + minWidth: 580, + maxWidth: 580, overflow: "auto", whiteSpace: "pre-wrap", }} diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index 1aab3b4c..ab9560fd 100755 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -16,6 +16,7 @@ import { OutlinedInput, Checkbox, Card, + Chip, Tooltip, FormControlLabel, Typography, @@ -485,6 +486,14 @@ If you're interested, please let me know a time that works for you, or set up a return `mailto:${admins}?bcc=frikky@shuffler.io,binu@shuffler.io&subject=${subject}&body=${body}` } + + const changeDistribution = (data) => { + //changeDistributed(data, !isDistributed) + console.log("Should change distribution to be shared among suborgs") + + editAuthenticationConfig(data.id, "suborg_distribute") + } + const deleteAuthentication = (data) => { toast("Deleting auth " + data.label); @@ -800,10 +809,10 @@ If you're interested, please let me know a time that works for you, or set up a }); }; - const editAuthenticationConfig = (id) => { + const editAuthenticationConfig = (id, parentAction) => { const data = { id: id, - action: "assign_everywhere", + action: parentAction !== undefined && parentAction !== null ? parentAction : "assign_everywhere", }; const url = globalUrl + "/api/v1/apps/authentication/" + id + "/config"; @@ -821,9 +830,9 @@ If you're interested, please let me know a time that works for you, or set up a .then((response) => response.json().then((responseJson) => { if (responseJson["success"] === false) { - toast("Failed overwriting appauth in workflows"); + toast("Failed overwriting appauth"); } else { - toast("Successfully updated auth everywhere!"); + toast("Successfully updated auth!"); setSelectedUserModalOpen(false); setTimeout(() => { getAppAuthentication(); @@ -1732,7 +1741,7 @@ If you're interested, please let me know a time that works for you, or set up a const userId = user.id; const data = { user_id: userId }; - console.log(user, userdata) + toast("Generating new API key") var fetchdata = { method: "POST", @@ -3440,6 +3449,7 @@ If you're interested, please let me know a time that works for you, or set up a style={{ minWidth: 300, maxWidth: 300, overflow: "hidden" }} /> + {schedules === undefined || schedules === null ? null @@ -3658,10 +3668,13 @@ If you're interested, please let me know a time that works for you, or set up a style={{ minWidth: 125, maxWidth: 125, overflow: "hidden" }} /> - + + {authentication === undefined || authentication === null ? null @@ -3693,6 +3706,8 @@ If you're interested, please let me know a time that works for you, or set up a ]; } + const isDistributed = data.suborg_distributed === true ? true : false; + return ( { updateAppAuthentication(data); }} + disabled={data.org_id !== selectedOrganization.id ? true : false} > - + {data.defined ? ( { editAuthenticationConfig(data.id); }} > @@ -3803,23 +3819,54 @@ If you're interested, please let me know a time that works for you, or set up a placement="top" > {}} + disabled={data.org_id !== selectedOrganization.id ? true : false} > )} { deleteAuthentication(data); }} > - + + + {selectedOrganization.id !== undefined && data.org_id !== selectedOrganization.id ? + + + + : + + { + changeDistribution(data, !isDistributed) + }} + /> + + } + ); })} @@ -4018,49 +4065,56 @@ If you're interested, please let me know a time that works for you, or set up a - { - if (environment.Type === "cloud") { - toast("No Orborus necessary for environment cloud. Create and use a different environment to run executions on-premises.") - return - } + + { + if (environment.Type === "cloud") { + toast("No Orborus necessary for environment cloud. Create and use a different environment to run executions on-premises.") + return + } - const elementName = "copy_element_shuffle"; - const auth = environment.auth === "" ? 'cb5st3d3Z!3X3zaJ*Pc' : environment.auth - const commandData = `docker run --volume "/var/run/docker.sock:/var/run/docker.sock" -e ENVIRONMENT_NAME="${environment.Name}" -e 'AUTH=${auth}' -e ORG="${props.userdata.active_org.id}" -e DOCKER_API_VERSION=1.40 -e BASE_URL="${globalUrl}" --name="shuffle-orborus" -d ghcr.io/shuffle/shuffle-orborus:latest` - var copyText = document.getElementById(elementName); - if (copyText !== null && copyText !== undefined) { - const clipboard = navigator.clipboard; - if (clipboard === undefined) { - toast("Can only copy over HTTPS (port 3443)"); - return; - } + if (props.userdata.active_org === undefined || props.userdata.active_org === null) { + toast("No active organization yet. Are you logged in?") + return + } - navigator.clipboard.writeText(commandData); - copyText.select(); - copyText.setSelectionRange( - 0, - 99999 - ); /* For mobile devices */ + const elementName = "copy_element_shuffle"; + const auth = environment.auth === "" ? 'cb5st3d3Z!3X3zaJ*Pc' : environment.auth + const newUrl = globalUrl === "https://shuffler.io" ? "https://shuffle-backend-stbuwivzoq-nw.a.run.app" : globalUrl - /* Copy the text inside the text field */ - document.execCommand("copy"); + const commandData = `docker run --volume "/var/run/docker.sock:/var/run/docker.sock" -e ENVIRONMENT_NAME="${environment.Name}" -e 'AUTH=${auth}' -e ORG="${props.userdata.active_org.id}" -e DOCKER_API_VERSION=1.40 -e BASE_URL="${newUrl}" --name="shuffle-orborus" -d ghcr.io/shuffle/shuffle-orborus:latest` + var copyText = document.getElementById(elementName); + if (copyText !== null && copyText !== undefined) { + const clipboard = navigator.clipboard; + if (clipboard === undefined) { + toast("Can only copy over HTTPS (port 3443)"); + return; + } - toast("Orborus command copied to clipboard"); - } - }} - > - - - - } + navigator.clipboard.writeText(commandData); + copyText.select(); + copyText.setSelectionRange( + 0, + 99999 + ); /* For mobile devices */ + + /* Copy the text inside the text field */ + document.execCommand("copy"); + + toast("Orborus command copied to clipboard"); + } + }} + > + + + + } /> { const [selectedAction, setSelectedAction] = React.useState({}); const [selectedActionEnvironment, setSelectedActionEnvironment] = React.useState({}); + const [streamDisabled, setStreamDisabled] = React.useState(false); const [executionRequest, setExecutionRequest] = React.useState({}); const [executionRunning, setExecutionRunning] = React.useState(false); @@ -551,10 +552,18 @@ const AngularWorkflow = (defaultprops) => { props.userdata.active_org !== undefined ? props.userdata.active_org.cloud_sync === true : false; - const isCloud = - window.location.host === "localhost:3002" || - window.location.host === "shuffler.io"; + const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; + useEffect(() => { + return () => { + console.log("UNMOUNTING USER!") + sendStreamRequest({ + "item": "workflow", + "type": "leave", + "id": workflow.id, + }) + } + }, []) /* useEffect(() => { console.log("In useeffect for workflow: ", workflow) @@ -918,7 +927,7 @@ const AngularWorkflow = (defaultprops) => { }); }; - const setNewAppAuth = (appAuthData) => { + const setNewAppAuth = (appAuthData, refresh) => { fetch(globalUrl + "/api/v1/apps/authentication", { method: "PUT", headers: { @@ -937,9 +946,14 @@ const AngularWorkflow = (defaultprops) => { }) .then((responseJson) => { if (!responseJson.success) { - toast("Failed to set app auth: " + responseJson.reason); + toast("Error: " + responseJson.reason); } else { - getAppAuthentication(true, false); + if (refresh === true) { + getAppAuthentication(true, true, true); + } else { + getAppAuthentication(true, false); + } + setAuthenticationModalOpen(false); // Needs a refresh with the new authentication.. @@ -1003,16 +1017,15 @@ const AngularWorkflow = (defaultprops) => { } setExecutionModalView(1); - start(); - setExecutionRequest({ execution_id: execution.execution_id, authorization: execution.authorization, }); - const newitem = removeParam("execution_id", cursearch); - navigate(curpath + newitem) - //props.history.push(curpath + newitem); + start(); + + //const newitem = removeParam("execution_id", cursearch); + //navigate(curpath + newitem) } else { console.log("Couldn't find execution for execution ID. Retrying as user to get ", tmpView) @@ -1025,8 +1038,8 @@ const AngularWorkflow = (defaultprops) => { setExecutionRequest(cur_execution); start(); - const newitem = removeParam("execution_id", cursearch); - navigate(curpath + newitem) + //const newitem = removeParam("execution_id", cursearch); + //navigate(curpath + newitem) setTimeout(() => { stop() @@ -1310,12 +1323,28 @@ const AngularWorkflow = (defaultprops) => { const sendStreamRequest = (body) => { //console.log("Stream not activated yet.") - return + if (!isCloud) { + console.log("Stream not activated yet for onprem") + return + } + + if (streamDisabled) { + console.log("Stream disabled") + return + } + // Session may be important here huh body.user_id = userdata.id - fetch(`${globalUrl}/api/v1/workflows/${props.match.params.key}/stream`, { + //const url = ${globalUrl}/api/v1/workflows/${props.match.params.key}/stream + //const streamUrl = "http://localhost:5002" + + console.log("Stream request: ", body) + const streamUrl = "https://stream.shuffler.io" + const url = `${streamUrl}/api/v1/workflows/${props.match.params.key}/stream` + + fetch(url, { method: "POST", headers: { "Content-Type": "application/json", @@ -1337,7 +1366,7 @@ const AngularWorkflow = (defaultprops) => { }) .catch((error) => { console.log("Stream send error: ", error.toString()) - //toast(error.toString()); + setStreamDisabled(true) }) } @@ -1872,6 +1901,7 @@ const AngularWorkflow = (defaultprops) => { setSelectedAction(selectedAction); setWorkflow(workflow); saveWorkflow(workflow); + toast("Added and updated authentication!"); shouldClose = true } else { @@ -2084,7 +2114,9 @@ const AngularWorkflow = (defaultprops) => { } const onChunkedResponseError = (err) => { - console.error(err) + if (streamDisabled) { + return + } } @@ -2570,14 +2602,30 @@ const AngularWorkflow = (defaultprops) => { } const startWorkflowStream = async (workflowId) => { - const timeout = 60000 + if (!isCloud) { + console.log("Not cloud, not starting workflow stream") + return + } - return - + if (streamDisabled) { + console.log("Stream disabled") + return + } + + const timeout = 60000 + //const url = `${globalUrl}/api/v1/workflows/${workflowId}/stream` + //const streamUrl = "https://shuffle-streaming-backend-stbuwivzoq-ew.a.run.app" + // + const streamUrl = "https://stream.shuffler.io" + const url = `${streamUrl}/api/v1/workflows/${workflowId}/stream` while (true) { + if (streamDisabled) { + break + } + // Wait 1 second before next request just in case of timeouts await new Promise(r => setTimeout(r, 1000)); - await fetchWithTimeout(`${globalUrl}/api/v1/workflows/${workflowId}/stream`, { + await fetchWithTimeout(url, { method: "GET", headers: { "Content-Type": "application/json", @@ -6261,7 +6309,6 @@ const AngularWorkflow = (defaultprops) => { } insertedNodes = insertedNodes.concat(newedges); - setWorkflow(inputworkflow); // Reset view for cytoscape @@ -6271,6 +6318,8 @@ const AngularWorkflow = (defaultprops) => { } else { setElements(insertedNodes); } + + console.log("Setupgraph done 2!") }; const removeNode = (nodeId) => { @@ -6576,6 +6625,7 @@ const AngularWorkflow = (defaultprops) => { } // preview: true, + console.log("In POST graph setup 2") cy.fit(null, 200); cy.on("boxselect", "node", (e) => { @@ -6624,6 +6674,7 @@ const AngularWorkflow = (defaultprops) => { document.title = "Workflow - " + workflow.name; + console.log("In POST graph setup 3") startWorkflowStream(props.match.params.key); registerKeys(); @@ -7516,14 +7567,30 @@ const AngularWorkflow = (defaultprops) => { description = app.actions[actionIndex].description } - const parsedEnvironments = + var parsedEnvironments = environments === null || environments === [] ? "cloud" : environments[defaultEnvironmentIndex] === undefined ? "cloud" : environments[defaultEnvironmentIndex].Name; - // activated: app.generated === true ? app.activated === false ? false : true : true, + // List other nodes in the workflow and see if they have an environment set. If they do, use that as the default + if (cy !== undefined && cy !== null) { + const foundnodes = cy.nodes().jsons() + if (foundnodes !== undefined && foundnodes !== null && foundnodes.length > 0) { + // As they should all be the same, this is just an override + for (let nodekey in foundnodes) { + const curnode = foundnodes[nodekey] + if (curnode.data.environment !== undefined && curnode.data.environment !== null && curnode.data.environment.length > 0) { + console.log("Found environment: ", curnode.data.environment) + parsedEnvironments = curnode.data.environment + break + } + } + } + } + + console.log("Discovered environment: ", parsedEnvironments) const newAppData = { name: app.actions[actionIndex].name, label: actionLabel, @@ -8139,6 +8206,25 @@ const AngularWorkflow = (defaultprops) => {
) })} + {visibleApps.length <= 4 ? ( +
{ + }} + > + + Click one of the relevant public apps below to Activate it for your organization. + + { + console.log("CLICKED") + }}> + + + + + +
+ ) : null}
) : apps.length > 0 ? (
{ }} > - Couldn't find the app you're looking for? Searching unactivated apps. Click one of the below apps to Activate it for your organization. + Couldn't find the apps you were looking for? Searching unactivated apps. Click one of the below apps to Activate it for your organization. { console.log("CLICKED") @@ -8364,7 +8450,6 @@ const AngularWorkflow = (defaultprops) => { } } - console.log("NEW ACTION: ", newSelectedAction); setSelectedAction(newSelectedAction); setUpdate(Math.random()); @@ -13259,10 +13344,6 @@ const AngularWorkflow = (defaultprops) => { "user": "Anonymous", "user_id": "user_id", "color": "blue", - }, { - "user": "frikky", - "user_id": "user_id", - "color": "red", }] @@ -13318,18 +13399,18 @@ const AngularWorkflow = (defaultprops) => { const showErrors = !isMobile && !workflow.public && workflow.errors !== undefined && workflow.errors !== null && workflow.errors.length > 0 ?
- - {workflow.errors.length} Potential Workflow Issue{workflow.errors.length > 1 ? "s" : ""} + + {workflow.errors.length} Workflow Issue{workflow.errors.length > 1 ? "s" : ""} { const [oauth2Scopes, setOauth2Scopes] = useState([]); const [oauth2Type, setOauth2Type] = useState("delegated"); - const [oauth2GrantType, setOauth2GrantType] = useState("client_credentials"); + + //client_credentials + const [oauth2GrantType, setOauth2GrantType] = useState(""); const defaultAuth = { name: "", type: "header", @@ -1757,7 +1759,7 @@ const AppCreator = (defaultprops) => { // Kind of fucked up, but it works for now? if (value["x-grant-type"] !== undefined && value["x-grant-type"] !== null && value["x-grant-type"].length !== 0) { - setOauth2Type(value["x-grant-type"]) + setOauth2GrantType(value["x-grant-type"]) } //console.log("FLOW2: ", value[flowkey][basekey]) @@ -5988,6 +5990,10 @@ const AppCreator = (defaultprops) => { fullWidth onChange={(e) => { setOauth2Type(e.target.value); + + if (e.target.value === "application" && oauth2GrantType === "") { + setOauth2GrantType("client_credentials") + } }} value={oauth2Type} style={{ diff --git a/frontend/src/views/Docs.jsx b/frontend/src/views/Docs.jsx index 350853ee..c948dd3a 100755 --- a/frontend/src/views/Docs.jsx +++ b/frontend/src/views/Docs.jsx @@ -368,14 +368,14 @@ const Docs = (defaultprops) => { } const markdownStyle = { - color: "rgba(255, 255, 255, 0.65)", + color: "rgba(255, 255, 255, 0.90)", overflow: "hidden", paddingBottom: 100, margin: "auto", maxWidth: "100%", minWidth: "100%", overflow: "hidden", - fontSize: isMobile ? "1.3rem" : "1.0rem", + fontSize: isMobile ? "1.3rem" : "1.1rem", }; function OuterLink(props) { diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 2bcd3b38..981bc7a0 100755 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -1982,16 +1982,16 @@ const Workflows = (props) => {
- {data.image !== undefined && data.image !== null && data.image.length > 0 ? - {data.name} - : null} - - Edit {data.name} - - - } placement="bottom"> - + {data.image !== undefined && data.image !== null && data.image.length > 0 ? + {data.name} + : null} + + Edit {data.name} + +
+ } placement="left"> + Date: Fri, 24 Nov 2023 12:22:49 +0100 Subject: [PATCH 11/33] Added timeout properly to orborus as well --- functions/onprem/orborus/orborus.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 050620b7..92ddc27f 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -1541,9 +1541,12 @@ func main() { log.Printf("[INFO] Execution already handled (rerun of old executions?): %s", execution.ExecutionId) toBeRemoved.Data = append(toBeRemoved.Data, execution) + // Should check when last this was ran, and if it's more than 10 minutes ago and it's not finished, we should run it again? + /* if swarmConfig != "run" && swarmConfig != "swarm" { continue } + */ } // Now, how do I execute this one? @@ -1597,6 +1600,11 @@ func main() { env = append(env, fmt.Sprintf("SHUFFLE_VOLUME_BINDS=%s", os.Getenv("SHUFFLE_VOLUME_BINDS"))) } + + if len(os.Getenv("SHUFFLE_APP_SDK_TIMEOUT")) > 0 { + env = append(env, fmt.Sprintf("SHUFFLE_APP_SDK_TIMEOUT=%s", os.Getenv("SHUFFLE_APP_SDK_TIMEOUT"))) + } + // Setting up internal proxy config for Shuffle -> shuffle comms overrideHttpProxy := os.Getenv("SHUFFLE_INTERNAL_HTTP_PROXY") overrideHttpsProxy := os.Getenv("SHUFFLE_INTERNAL_HTTPS_PROXY") From 3dc5f75d9d93c76ad074973d6aaa367499cf3e0e Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 24 Nov 2023 12:37:43 +0100 Subject: [PATCH 12/33] Gomod changes for orborus & backend to rerun workflows better --- backend/go-app/go.mod | 4 ++-- backend/go-app/go.sum | 4 ++++ functions/onprem/orborus/go.mod | 2 +- functions/onprem/orborus/go.sum | 4 ++++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 14cecd74..70551980 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -1,6 +1,6 @@ module shuffle-shared -replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared +//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared go 1.19 @@ -18,7 +18,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.4.80 + github.com/shuffle/shuffle-shared v0.4.96 golang.org/x/crypto v0.14.0 google.golang.org/api v0.125.0 google.golang.org/grpc v1.55.0 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 0dfe108a..9b087663 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -408,6 +408,10 @@ github.com/shuffle/shuffle-shared v0.4.66 h1:Aw4qOp0VsVJrRzW1sJhEy4OY4fRGlFErUD5 github.com/shuffle/shuffle-shared v0.4.66/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= github.com/shuffle/shuffle-shared v0.4.80 h1:03OL+O8prwL9zq6Gnb9SRORPWi5+ThO0jPoxk+xctOo= github.com/shuffle/shuffle-shared v0.4.80/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= +github.com/shuffle/shuffle-shared v0.4.95 h1:xr92/03/uQeJiDme9S8/vgF1KWyQgJ1KQXVE7nQMKis= +github.com/shuffle/shuffle-shared v0.4.95/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= +github.com/shuffle/shuffle-shared v0.4.96 h1:iaIB/HP9eKpw9DMMJZhSLDbKdHJt075kFYLHg9AaiiM= +github.com/shuffle/shuffle-shared v0.4.96/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 97d638d9..fef487a1 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -9,7 +9,7 @@ require ( github.com/mackerelio/go-osstat v0.2.3 github.com/satori/go.uuid v1.2.0 github.com/shirou/gopsutil v3.21.11+incompatible - github.com/shuffle/shuffle-shared v0.4.86 + github.com/shuffle/shuffle-shared v0.4.96 k8s.io/api v0.28.1 k8s.io/apimachinery v0.28.1 k8s.io/client-go v0.28.1 diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index cd9f9880..c9eea384 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -274,6 +274,10 @@ github.com/shuffle/shuffle-shared v0.4.62 h1:L76zWCD/7gIBuhr3feWZwzT4I8VCiLRd8ZA github.com/shuffle/shuffle-shared v0.4.62/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= github.com/shuffle/shuffle-shared v0.4.86 h1:QrFx3j+maUgeU/dP48WMx+NBcWdwe3Ov2yKlLUUZRHw= github.com/shuffle/shuffle-shared v0.4.86/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= +github.com/shuffle/shuffle-shared v0.4.95 h1:xr92/03/uQeJiDme9S8/vgF1KWyQgJ1KQXVE7nQMKis= +github.com/shuffle/shuffle-shared v0.4.95/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= +github.com/shuffle/shuffle-shared v0.4.96 h1:iaIB/HP9eKpw9DMMJZhSLDbKdHJt075kFYLHg9AaiiM= +github.com/shuffle/shuffle-shared v0.4.96/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= From 599215c1c842da0ad42942b083c3e83909784d9a Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 24 Nov 2023 13:32:02 +0100 Subject: [PATCH 13/33] Made 1.4.0 builds go back to nightly properly --- .github/workflows/dockerbuild.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/dockerbuild.yaml b/.github/workflows/dockerbuild.yaml index 4fd63f53..c00583ed 100644 --- a/.github/workflows/dockerbuild.yaml +++ b/.github/workflows/dockerbuild.yaml @@ -3,7 +3,7 @@ name: dockerbuild on: push: branches: - - main + - 1.4.0 paths: - "**" - "!.github/**" @@ -19,23 +19,23 @@ jobs: include: - app: frontend path: frontend - version: 1.3.0 + version: nightly experimental: true - app: backend path: backend - version: 1.3.0 + version: nightly experimental: true - app: app_sdk path: backend/app_sdk - version: 1.3.0 + version: nightly experimental: true - app: orborus path: functions/onprem/orborus - version: 1.3.0 + version: nightly experimental: true - app: worker path: functions/onprem/worker - version: 1.3.0 + version: nightly experimental: true steps: - name: Checkout From 27cedd9d227ee6ffb7f17657124187efb8a2e14f Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 24 Nov 2023 13:33:03 +0100 Subject: [PATCH 14/33] Changed health interval to 30 minutes onprem --- backend/go-app/go.mod | 2 +- backend/go-app/go.sum | 2 ++ backend/go-app/main.go | 2 +- backend/go-app/walkoff.go | 1 + 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 70551980..3622850d 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -18,7 +18,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.4.96 + github.com/shuffle/shuffle-shared v0.4.97 golang.org/x/crypto v0.14.0 google.golang.org/api v0.125.0 google.golang.org/grpc v1.55.0 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 9b087663..cafc82f8 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -412,6 +412,8 @@ github.com/shuffle/shuffle-shared v0.4.95 h1:xr92/03/uQeJiDme9S8/vgF1KWyQgJ1KQXV github.com/shuffle/shuffle-shared v0.4.95/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= github.com/shuffle/shuffle-shared v0.4.96 h1:iaIB/HP9eKpw9DMMJZhSLDbKdHJt075kFYLHg9AaiiM= github.com/shuffle/shuffle-shared v0.4.96/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= +github.com/shuffle/shuffle-shared v0.4.97 h1:1c8LdNteMykKNEV97vwP63oSP2tV/Uso3O4TC+oxdFQ= +github.com/shuffle/shuffle-shared v0.4.97/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 33383850..12f3372b 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -4033,7 +4033,7 @@ func runInitEs(ctx context.Context) { if os.Getenv("SHUFFLE_HEALTHCHECK_DISABLED") != "true" { - healthcheckInterval := 15 + healthcheckInterval := 30 log.Printf("[INFO] Starting healthcheck job every %d minute. Stats available on /api/v1/health/stats. Disable with SHUFFLE_HEALTHCHECK_DISABLED=true", healthcheckInterval) job := func() { // Prepare a fake http.responsewriter diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 83656aa4..05ced567 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -707,6 +707,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workflowExecutionId string, actionResult shuffle.ActionResult, resp http.ResponseWriter) { log.Printf("[DEBUG][%s] Running workflow execution update", workflowExecutionId) + // Should start a tx for the execution here workflowExecution, err := shuffle.GetWorkflowExecution(ctx, workflowExecutionId) if err != nil { From 32f5c6b9a7bd4e592efaab18c093df1ddca3d547 Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 24 Nov 2023 14:01:00 +0100 Subject: [PATCH 15/33] Update dockerbuild.yaml --- .github/workflows/dockerbuild.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dockerbuild.yaml b/.github/workflows/dockerbuild.yaml index c00583ed..bb2a29fc 100644 --- a/.github/workflows/dockerbuild.yaml +++ b/.github/workflows/dockerbuild.yaml @@ -77,9 +77,9 @@ jobs: cache-to: type=local,dest=/tmp/.buildx-cache tags: | ghcr.io/shuffle/shuffle-${{ matrix.app }}:${{ matrix.version }} - ghcr.io/shuffle/shuffle-${{ matrix.app }}:latest + ghcr.io/shuffle/shuffle-${{ matrix.app }}:nightly ${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:${{ matrix.version }} - ${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:latest + ${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:nightly - name: Image digest run: echo ${{ steps.docker_build.outputs.digest }} From 4a1e9316a88bed4ea2f5b398d2e06023c3ef290c Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 24 Nov 2023 14:12:35 +0100 Subject: [PATCH 16/33] Made it look for all schedules instead of just top 10 --- backend/go-app/go.mod | 2 +- backend/go-app/go.sum | 2 ++ backend/go-app/main.go | 7 ++++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 3622850d..8e6d42ad 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -18,7 +18,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.4.97 + github.com/shuffle/shuffle-shared v0.4.98 golang.org/x/crypto v0.14.0 google.golang.org/api v0.125.0 google.golang.org/grpc v1.55.0 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index cafc82f8..f31286be 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -414,6 +414,8 @@ github.com/shuffle/shuffle-shared v0.4.96 h1:iaIB/HP9eKpw9DMMJZhSLDbKdHJt075kFYL github.com/shuffle/shuffle-shared v0.4.96/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= github.com/shuffle/shuffle-shared v0.4.97 h1:1c8LdNteMykKNEV97vwP63oSP2tV/Uso3O4TC+oxdFQ= github.com/shuffle/shuffle-shared v0.4.97/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= +github.com/shuffle/shuffle-shared v0.4.98 h1:pgsLdWUpxZ/q+eHpAjOCH9icOsmuO5u2olmirOldy5A= +github.com/shuffle/shuffle-shared v0.4.98/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 12f3372b..3d1a7b71 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3573,7 +3573,12 @@ func runInitEs(ctx context.Context) { log.Printf("[DEBUG] Getting organizations for Elasticsearch/Opensearch") activeOrgs, err := shuffle.GetAllOrgs(ctx) - log.Printf("[DEBUG] Got %d organizations to look into", len(activeOrgs)) + log.Printf("[DEBUG] Got %d organizations to look into. If this is 0, we wait 10 more seconds until DB is ready and try again.", len(activeOrgs)) + if len(activeOrgs) == 0 { + time.Sleep(10 * time.Second) + activeOrgs, err = shuffle.GetAllOrgs(ctx) + return + } setUsers := false _ = setUsers From 9fc63263502d42473b35845ebced23c63fcff06c Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 24 Nov 2023 15:40:23 +0100 Subject: [PATCH 17/33] Made autocasting happen values in larger than, less than and is empty --- backend/app_sdk/app_base.py | 40 ++++++++++++++++++++++++++++++++----- backend/go-app/main.go | 1 - 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 84a0d9a5..1d6b1e35 100755 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -2758,11 +2758,14 @@ class AppBase: if destinationvalue.lower() in sourcevalue.lower(): return True - elif check.lower() == "is empty": - if len(sourcevalue) == 0: - return True + elif check.lower() == "is empty" or check.lower() == "is_empty": + try: + if len(list(sourcevalue)) == 0: + return True + except Exception as e: + self.logger.info(f"[WARNING] Failed to check if empty as list: {e}") - if str(sourcevalue) == 0: + if len(str(sourcevalue)) == 0: return True return False @@ -2784,15 +2787,41 @@ class AppBase: return False elif check.lower() == "larger than" or check.lower() == "bigger than": + try: + destinationvalue = len(list(destinationvalue)) + except Exception as e: + self.logger.info(f"[WARNING] Failed to convert destination to list: {e}") + + try: + # Check if it's a list in autocast and if so, check the length + if len(list(sourcevalue)) > int(destinationvalue): + return True + except Exception as e: + self.logger.info(f"[WARNING] Failed to check if larger than as list: {e}") + try: if str(sourcevalue).isdigit() and str(destinationvalue).isdigit(): if int(sourcevalue) > int(destinationvalue): return True except AttributeError as e: - print("[WARNING] Condition larger than failed with values %s and %s: %s" % (sourcevalue, destinationvalue, e)) + self.logger.info("[WARNING] Condition larger than failed with values %s and %s: %s" % (sourcevalue, destinationvalue, e)) return False + + elif check.lower() == "smaller than" or check.lower() == "less than": + try: + destinationvalue = len(list(destinationvalue)) + except Exception as e: + self.logger.info(f"[WARNING] Failed to convert destination to list: {e}") + + + try: + # Check if it's a list in autocast and if so, check the length + if len(list(sourcevalue)) < int(destinationvalue): + return True + except Exception as e: + self.logger.info(f"[WARNING] Failed to check if smaller than as list: {e}") try: if str(sourcevalue).isdigit() and str(destinationvalue).isdigit(): if int(sourcevalue) < int(destinationvalue): @@ -2801,6 +2830,7 @@ class AppBase: except AttributeError as e: print("[WARNING] Condition smaller than failed with values %s and %s: %s" % (sourcevalue, destinationvalue, e)) return False + elif check.lower() == "re" or check.lower() == "matches regex": try: found = re.search(str(destinationvalue), str(sourcevalue)) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 3d1a7b71..7f561852 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3577,7 +3577,6 @@ func runInitEs(ctx context.Context) { if len(activeOrgs) == 0 { time.Sleep(10 * time.Second) activeOrgs, err = shuffle.GetAllOrgs(ctx) - return } setUsers := false From c776c71fc6b62b697ddc2b6bdfe142cfdfa74e87 Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 24 Nov 2023 16:54:08 +0100 Subject: [PATCH 18/33] Made larger than, less than and is empty a bit smarter to not need {{ | size }} --- backend/app_sdk/app_base.py | 62 ++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 32 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 1d6b1e35..d1d42e34 100755 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -2740,7 +2740,7 @@ class AppBase: return "", parameter["value"], is_loop def run_validation(sourcevalue, check, destinationvalue): - print("[DEBUG] Checking %s %s %s" % (sourcevalue, check, destinationvalue)) + self.logger.info("[DEBUG] Checking %s '%s' %s" % (sourcevalue, check, destinationvalue)) if check == "=" or check.lower() == "equals": if str(sourcevalue).lower() == str(destinationvalue).lower(): @@ -2760,7 +2760,7 @@ class AppBase: elif check.lower() == "is empty" or check.lower() == "is_empty": try: - if len(list(sourcevalue)) == 0: + if len(json.loads(sourcevalue)) == 0: return True except Exception as e: self.logger.info(f"[WARNING] Failed to check if empty as list: {e}") @@ -2768,8 +2768,6 @@ class AppBase: if len(str(sourcevalue)) == 0: return True - return False - elif check.lower() == "contains_any_of": newvalue = [destinationvalue.lower()] if "," in destinationvalue: @@ -2785,20 +2783,7 @@ class AppBase: print("[INFO] Found %s in %s" % (item, sourcevalue)) return True - return False elif check.lower() == "larger than" or check.lower() == "bigger than": - try: - destinationvalue = len(list(destinationvalue)) - except Exception as e: - self.logger.info(f"[WARNING] Failed to convert destination to list: {e}") - - try: - # Check if it's a list in autocast and if so, check the length - if len(list(sourcevalue)) > int(destinationvalue): - return True - except Exception as e: - self.logger.info(f"[WARNING] Failed to check if larger than as list: {e}") - try: if str(sourcevalue).isdigit() and str(destinationvalue).isdigit(): if int(sourcevalue) > int(destinationvalue): @@ -2806,22 +2791,22 @@ class AppBase: except AttributeError as e: self.logger.info("[WARNING] Condition larger than failed with values %s and %s: %s" % (sourcevalue, destinationvalue, e)) - return False + + try: + destinationvalue = len(json.loads(destinationvalue)) + except Exception as e: + self.logger.info(f"[WARNING] Failed to convert destination to list: {e}") + try: + # Check if it's a list in autocast and if so, check the length + if len(json.loads(sourcevalue)) > int(destinationvalue): + return True + except Exception as e: + self.logger.info(f"[WARNING] Failed to check if larger than as list: {e}") elif check.lower() == "smaller than" or check.lower() == "less than": - try: - destinationvalue = len(list(destinationvalue)) - except Exception as e: - self.logger.info(f"[WARNING] Failed to convert destination to list: {e}") + self.logger.info("In smaller than check: %s %s" % (sourcevalue, destinationvalue)) - - try: - # Check if it's a list in autocast and if so, check the length - if len(list(sourcevalue)) < int(destinationvalue): - return True - except Exception as e: - self.logger.info(f"[WARNING] Failed to check if smaller than as list: {e}") try: if str(sourcevalue).isdigit() and str(destinationvalue).isdigit(): if int(sourcevalue) < int(destinationvalue): @@ -2829,7 +2814,18 @@ class AppBase: except AttributeError as e: print("[WARNING] Condition smaller than failed with values %s and %s: %s" % (sourcevalue, destinationvalue, e)) - return False + + try: + destinationvalue = len(json.loads(destinationvalue)) + except Exception as e: + self.logger.info(f"[WARNING] Failed to convert destination to list: {e}") + + try: + # Check if it's a list in autocast and if so, check the length + if len(json.loads(sourcevalue)) < int(destinationvalue): + return True + except Exception as e: + self.logger.info(f"[WARNING] Failed to check if smaller than as list: {e}") elif check.lower() == "re" or check.lower() == "matches regex": try: @@ -2846,7 +2842,7 @@ class AppBase: return True else: - print("[DEBUG] Condition: can't handle %s yet. Setting to true" % check) + self.logger.error("[DEBUG] Condition: can't handle %s yet. Setting to true" % check) return False @@ -2911,6 +2907,8 @@ class AppBase: "contains_any_of", "re", "matches regex", + "is empty", + "is_empty", ] relevantbranches = [] @@ -2970,7 +2968,7 @@ class AppBase: destinationvalue = parse_wrapper_start(destinationvalue, self) if not condition["condition"]["value"] in available_checks: - self.logger.warning("Skipping %s %s %s because %s is invalid." % (sourcevalue, condition["condition"]["value"], destinationvalue, condition["condition"]["value"])) + self.logger.error("[ERROR] Skipping '%s' -> %s -> '%s' because %s is invalid." % (sourcevalue, condition["condition"]["value"], destinationvalue, condition["condition"]["value"])) continue # Configuration = negated because of WorkflowAppActionParam.. From 10f5d202ea9fac9dc006d1ca31def8522763d2af Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 24 Nov 2023 18:35:23 +0100 Subject: [PATCH 19/33] Added more app sdk timeout debug info --- backend/app_sdk/app_base.py | 4 ++-- docker-compose.yml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index d1d42e34..92982747 100755 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -3596,7 +3596,7 @@ class AppBase: future.cancel() newres = json.dumps({ "success": False, - "reason": "Timeout error within %d seconds. This happens if we can't reach or use the API you're trying to use within the time limit." % timeout, + "reason": "Timeout error within %d seconds (1). This happens if we can't reach or use the API you're trying to use within the time limit. Configure SHUFFLE_APP_SDK_TIMEOUT=100 in Orborus to increase it to 100 seconds. Not changeable for cloud." % timeout, "exception": str(e), }) @@ -3609,7 +3609,7 @@ class AppBase: except concurrent.futures.TimeoutError as e: newres = json.dumps({ "success": False, - "reason": "Timeout error within %d seconds (2). This happens if we can't reach or use the API you're trying to use within the time limit" % timeout + "reason": "Timeout error within %d seconds (2). This happens if we can't reach or use the API you're trying to use within the time limit. Configure SHUFFLE_APP_SDK_TIMEOUT=100 in Orborus to increase it to 100 seconds. Not changeable for cloud." % timeout, }) break diff --git a/docker-compose.yml b/docker-compose.yml index c35a6138..ba3f4353 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -42,6 +42,7 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock environment: + - SHUFFLE_APP_SDK_TIMEOUT=300 #- DOCKER_HOST=tcp://docker-socket-proxy:2375 - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - BASE_URL=http://${OUTER_HOSTNAME}:5001 From 3ecfb6898ebcd8008a39627a5962741aa0061089 Mon Sep 17 00:00:00 2001 From: Frikky Date: Mon, 27 Nov 2023 04:27:53 +0100 Subject: [PATCH 20/33] Added automatic value sanitization if it is a string in a JSON. Point is to make newlines and quotes not screw it all up --- backend/app_sdk/app_base.py | 42 +++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 92982747..b7996c12 100755 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -1964,6 +1964,8 @@ class AppBase: # Parses JSON loops and such down to the item you're looking for # $nodename.#.id # $nodename.data.#min-max.info.id + # $nodename.data.#1-max.info.id + # $nodename.data.#min-1.info.id def recurse_json(basejson, parsersplit): match = "#([0-9a-z]+):?-?([0-9a-z]+)?#?" try: @@ -2080,6 +2082,8 @@ class AppBase: if (basejson[value].endswith("}") and basejson[value].endswith("}")) or (basejson[value].startswith("[") and basejson[value].endswith("]")): basejson = json.loads(basejson[value]) else: + # Should we sanitize here? + self.logger.info("[DEBUG] VALUE TO SANITIZE?: %s" % basejson[value]) return str(basejson[value]), False except json.decoder.JSONDecodeError as e: return str(basejson[value]), False @@ -2128,7 +2132,6 @@ class AppBase: actionname_lower = parsersplit[0][1:].lower() #Actionname: Start_node - #print(f"\n[INFO] Actionname: {actionname_lower}") # 1. Find the action baseresult = "" @@ -2558,6 +2561,27 @@ class AppBase: return data + # Makes JSON string values into valid strings in JSON + # Mainly by removing newlines and such + def fix_json_string_value(value): + try: + value = value.replace("\r\n", "\\r\\n") + value = value.replace("\n", "\\n") + value = value.replace("\r", "\\r") + + # Fix quotes in the string + value = value.replace("\\\"", "\"") + value = value.replace("\"", "\\\"") + + value = value.replace("\\\'", "\'") + value = value.replace("\'", "\\\'") + except Exception as e: + print(f"[WARNING] Failed to fix json string value: {e}") + + return value + + + # Parses parameters sent to it and returns whether it did it successfully with the values found def parse_params(action, fullexecution, parameter, self): # Skip if it starts with $? @@ -2621,6 +2645,20 @@ class AppBase: value, is_loop = get_json_value(fullexecution, to_be_replaced) #self.logger.info(f"\n\nType of value: {type(value)}") if isinstance(value, str): + # Could we take it here? + self.logger.info(f"[DEBUG] Got value %s for parameter {paramname}" % value) + # Should check if there is are quotes infront of and after the to_be_replaced + # If there are, then we need to sanitize the value + # 1. Look for the to_be_replaced in the data + # 2. Check if there is a quote infront of it and also if there are {} in the data to validate JSON + # 3. If there are, sanitize! + if data.find(f'"{to_be_replaced}"') != -1 and data.find("{") != -1 and data.find("}") != -1: + print(f"[DEBUG] Found quotes infront of and after {to_be_replaced}! This probably means it's JSON and should be sanitized.") + returnvalue = fix_json_string_value(value) + value = returnvalue + + + parameter["value"] = parameter["value"].replace(to_be_replaced, value) elif isinstance(value, dict) or isinstance(value, list): # Changed from JSON dump to str() 28.05.2021 @@ -2633,7 +2671,7 @@ class AppBase: # parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value)) # self.logger.info("Failed parsing value as string?") else: - self.logger.info("[WARNING] Unknown type %s" % type(value)) + self.logger.error("[ERROR] Unknown type %s" % type(value)) try: parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value)) except json.decoder.JSONDecodeError as e: From ce029bfb32299cf30007f136686dd895fc2e94a4 Mon Sep 17 00:00:00 2001 From: Frikky Date: Mon, 27 Nov 2023 22:24:05 +0100 Subject: [PATCH 21/33] Rolled back SDK to not get more string issues --- backend/app_sdk/app_base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index b7996c12..536fff21 100755 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -2652,10 +2652,10 @@ class AppBase: # 1. Look for the to_be_replaced in the data # 2. Check if there is a quote infront of it and also if there are {} in the data to validate JSON # 3. If there are, sanitize! - if data.find(f'"{to_be_replaced}"') != -1 and data.find("{") != -1 and data.find("}") != -1: - print(f"[DEBUG] Found quotes infront of and after {to_be_replaced}! This probably means it's JSON and should be sanitized.") - returnvalue = fix_json_string_value(value) - value = returnvalue + #if data.find(f'"{to_be_replaced}"') != -1 and data.find("{") != -1 and data.find("}") != -1: + # print(f"[DEBUG] Found quotes infront of and after {to_be_replaced}! This probably means it's JSON and should be sanitized.") + # returnvalue = fix_json_string_value(value) + # value = returnvalue From 1373bac875bbb38e1902948f9d80c2692850e54a Mon Sep 17 00:00:00 2001 From: Frikky Date: Tue, 28 Nov 2023 03:09:46 +0100 Subject: [PATCH 22/33] More header, notifications & debugger fixes --- frontend/src/components/NewHeader.jsx | 66 +++---- frontend/src/components/ParsedAction.jsx | 10 +- frontend/src/components/RuntimeDebugger.jsx | 86 +++++--- frontend/src/components/ShuffleCodeEditor.jsx | 35 +++- frontend/src/defaultCytoscapeStyle.jsx | 5 +- frontend/src/views/AngularWorkflow.jsx | 186 ++++++++++++++---- 6 files changed, 271 insertions(+), 117 deletions(-) diff --git a/frontend/src/components/NewHeader.jsx b/frontend/src/components/NewHeader.jsx index 004b3132..f6f663b3 100644 --- a/frontend/src/components/NewHeader.jsx +++ b/frontend/src/components/NewHeader.jsx @@ -294,49 +294,26 @@ const Header = (props) => { borderBottom: "1px solid rgba(255,255,255,0.4)", }} > - {/* - {new Date(data.updated_at).toISOString()} - */} - {data.reference_url !== undefined && - data.reference_url !== null && - data.reference_url.length > 0 ? ( - - {data.title} - - ) : ( - - {data.title} - - )} + {data.reference_url !== undefined && data.reference_url !== null && data.reference_url.length > 0 ? + + + {data.title} ({data.amount}) + + + : + + {data.title} + + } - {data.image !== undefined && - data.image !== null && - data.image.length > 0 ? ( - {data.title} - ) : null} - {data.description} - {/*data.tags !== undefined && data.tags !== null && data.tags.length > 0 ? - data.tags.map((tag, index) => { - return ( - { - }} - variant="outlined" - color="primary" - /> - ) - }) - : null */} + {data.image !== undefined && data.image !== null && data.image.length > 0 ? + {data.title} + : + null + } + + {data.description} +
{data.read === false ? (
- Notifications are made by Shuffle to help you discover issues or - improvements. + Notifications generated made by Shuffle to help you discover issues or + improvements. + Learn more {notifications.map((data, index) => { diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index 53f40b79..558fb983 100755 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -170,6 +170,7 @@ const ParsedAction = (props) => { setEditorData, setcodedata, + setAiQueryModalOpen, } = props; const classes = useStyles(); @@ -2913,14 +2914,17 @@ const ParsedAction = (props) => { }} disabled={autoCompleting} onClick={() => { - // aiSubmit(aiMsg, undefined, undefined, newSelectedAction) - aiSubmit("Fill based on previous values", undefined, undefined, selectedAction) + //if (setAiQueryModalOpen !== undefined) { + // setAiQueryModalOpen(true) + //} else { + aiSubmit("Fill based on previous values", undefined, undefined, selectedAction) + //} setAutocompleting(true) }} > {autoCompleting ? diff --git a/frontend/src/components/RuntimeDebugger.jsx b/frontend/src/components/RuntimeDebugger.jsx index 47462d2b..ff2706f6 100644 --- a/frontend/src/components/RuntimeDebugger.jsx +++ b/frontend/src/components/RuntimeDebugger.jsx @@ -46,11 +46,6 @@ const RuntimeDebugger = (props) => { const classes = useStyles(); - //const [workflowId, setWorkflowId] = useState(""); - //const [status, setStatus] = useState("FINISHED"); - //const [endTime, setEndTime] = useState(dayjs().subtract(0, 'day')) - //const [startTime, setStartTime] = useState(dayjs().subtract(30, 'day')) - const [workflowId, setWorkflowId] = useState("") const [status, setStatus] = useState("") const [endTime, setEndTime] = useState("") @@ -75,7 +70,7 @@ const RuntimeDebugger = (props) => { const submitSearch = (workflowId, status, startTime, endTime, cursor, limit) => { - setResultRows([]) + //setResultRows([]) setSearchLoading(true) const fetchData = { workflow_id: workflowId, @@ -107,22 +102,29 @@ const RuntimeDebugger = (props) => { //data.runs[key].endTimestamp = data.runs[key].ended_at.toISOString().slice(0, 19).replace('T', ' ') const startTimestamp = new Date(data.runs[key].started_at*1000) data.runs[key].startTimestamp = startTimestamp.toISOString().slice(0, 19).replace('T', ' ') + const endTimestamp = new Date(data.runs[key].completed_at*1000) data.runs[key].endTimestamp = endTimestamp.toISOString().slice(0, 19).replace('T', ' ') + if (data.runs[key].completed_at === 0 || data.runs[key].completed_at === null) { + data.runs[key].endTimestamp = "" + } } // Add 20 empty rows to the end of the resultRows array // This is to make sure that the scrollbar is always visible setResultRows(data.runs) + } else { + toast("No results found. Keeping old runs") } } else { - console.error("Search error: ", data.reason) + toast("Failed to search for runs. Please try again.") } }) .catch((error) => { setSearchLoading(false) console.error("Error:", error); + toast("Failed to search for runs. Please try again (2)") }) } @@ -315,27 +317,63 @@ const RuntimeDebugger = (props) => { ) }, }, - { field: 'startTimestamp', headerName: 'Start time (UTC)', width: 160, }, + { field: 'startTimestamp', headerName: 'Start time (UTC)', width: 160, + renderCell: (params) => { + const hasError = params.row.completed_at-params.row.started_at > 300 + + return ( + + { + console.log("Zoom in on end timestamp is this one: ", params.row.endTimestamp) + //setEndTimestamp(params.row.endTimestamp) + + // Make a new Date() from params.row.startTimestamp and set it in the endTime + const newEndTime = new Date(params.row.startTimestamp) + if (newEndTime !== null && newEndTime !== undefined && newEndTime !== "" && newEndTime !== "Invalid Date") { + // Translate newEndTime to UTC no matter what timezone we are in. Based it on local() + // Plus 1 minute to make sure it comes in + setEndTime(dayjs(newEndTime.setMinutes(newEndTime.getMinutes()+1))) + + // Use dayjs to translate it into something useful + + // Remove 5 minutes from it and set startTime + //newEndTime.setMinutes(newEndTime.getMinutes()-5) + //setStartTime(dayjs(newEndTime)) + } + }}> + {params.row.startTimestamp} + + + ) + } + }, { field: 'endTimestamp', headerName: 'End time (UTC)', width: 160, }, { field: 'id', headerName: 'Explore', width: 65, - renderCell: (params) => ( - - {params.row.result !== null && params.row.result !== undefined && params.row.result !== "" ? - params.row.result - : - null - } - - } > - - - - - ), + renderCell: (params) => { + const parsedResult = params.row.result === null || params.row.result === undefined || params.row.result === "" ? null : params.row.result + const hasError = parsedResult !== null && parsedResult !== undefined && parsedResult !== "" ? parsedResult.includes("{%") && parsedResult.includes("%}") : false + + return ( + + {params.row.result !== null && params.row.result !== undefined && params.row.result !== "" ? + params.row.result + : + null + } + + }> + + + + + + + ) + } }, ] @@ -522,6 +560,7 @@ const RuntimeDebugger = (props) => { minWidth: 240, maxWidth: 240, }} + ampm={false} label="Search from" format="YYYY-MM-DD HH:mm:ss" value={startTime} @@ -535,6 +574,7 @@ const RuntimeDebugger = (props) => { minWidth: 240, maxWidth: 240, }} + ampm={false} label="Search until" format="YYYY-MM-DD HH:mm:ss" value={endTime} diff --git a/frontend/src/components/ShuffleCodeEditor.jsx b/frontend/src/components/ShuffleCodeEditor.jsx index 91db5f49..3053083c 100644 --- a/frontend/src/components/ShuffleCodeEditor.jsx +++ b/frontend/src/components/ShuffleCodeEditor.jsx @@ -550,7 +550,6 @@ const CodeEditor = (props) => { var code_lines = localcodedata.split('\n') for (var i = 0; i < code_lines.length; i++){ var current_code_line = code_lines[i] - console.log("Codeline: ", current_code_line) var variable_occurence = current_code_line.match(/[\\]{0,1}[$]{1}([a-zA-Z0-9_-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,}/g) @@ -612,8 +611,6 @@ const CodeEditor = (props) => { const fixedVariable = fixVariable(variable_occurence[occ]) var correctVariable = availableVariables.includes(fixedVariable) if(!correctVariable) { - console.log("Line: ", i, "ch: ", dollar_occurence[occ]) - value.markText({line:i, ch:dollar_occurence[occ]}, {line:i, ch:dollar_occurence_len[occ]+dollar_occurence[occ]}, {"css": "background-color: rgb(248, 106, 62, 0.9); padding-top: 2px; padding-bottom: 2px; color: white"}) } else { value.markText({line:i, ch:dollar_occurence[occ]}, {line:i, ch:dollar_occurence_len[occ]+dollar_occurence[occ]}, {"css": "background-color: #8b8e26; padding-top: 2px; padding-bottom: 2px; color: white"}) @@ -664,6 +661,23 @@ const CodeEditor = (props) => { setlocalcodedata(updatedCode) } + const fixStringInput = (new_input) => { + // Newline fixes + new_input = new_input.replace(/\r\n/g, "\\n") + new_input = new_input.replace(/\n/g, "\\n") + + // Quote fixes + new_input = new_input.replace(/\\"/g, '"') + new_input = new_input.replace(/"/g, '\\"') + + new_input = new_input.replace(/\\'/g, "'") + new_input = new_input.replace(/'/g, "\\'") + + + return new_input + } + + const expectedOutput = (input) => { //const found = input.match(/[$]{1}([a-zA-Z0-9_-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,}/g) @@ -687,20 +701,26 @@ const CodeEditor = (props) => { valuefound = true + console.log("Here. Checking if we got an example?") try { if (typeof actionlist[j].example === "object") { + input = input.replace(found[i], JSON.stringify(actionlist[j].example), -1); } else if (actionlist[j].example.trim().startsWith("{") || actionlist[j].example.trim().startsWith("[")) { input = input.replace(found[i], JSON.stringify(actionlist[j].example), -1); } else { - input = input.replace(found[i], actionlist[j].example, -1) + console.log("This?") + + const newExample = fixStringInput(actionlist[j].example) + input = input.replace(found[i], newExample, -1) } } catch (e) { input = input.replace(found[i], actionlist[j].example, -1) } } + //if (!valuefound) { // console.log("Couldn't find value "+fixedVariable) //} @@ -736,7 +756,10 @@ const CodeEditor = (props) => { new_input = JSON.stringify(new_input) } else { if (typeof new_input === "string") { - new_input = new_input + // Check if it contains any newlines, and replace them with raw newlines + new_input = fixStringInput(new_input) + + // Replace quotes with nothing } else { console.log("NO TYPE? ", typeof new_input) try { @@ -747,7 +770,6 @@ const CodeEditor = (props) => { } } - //console.log("FOUND2: ", fixedVariable, actionlist[j].example) input = input.replace(fixedVariable, new_input, -1) input = input.replace(found[i], new_input, -1) @@ -1430,7 +1452,6 @@ const CodeEditor = (props) => { highlight_variables(value) }} onChange={(value, viewUpdate) => { - console.log("Value: ", value, viewUpdate) setlocalcodedata(value) expectedOutput(value) diff --git a/frontend/src/defaultCytoscapeStyle.jsx b/frontend/src/defaultCytoscapeStyle.jsx index 01c389f8..7e7132fd 100644 --- a/frontend/src/defaultCytoscapeStyle.jsx +++ b/frontend/src/defaultCytoscapeStyle.jsx @@ -4,11 +4,10 @@ const data = [ css: { label: "data(label)", "text-valign": "center", - "font-family": - "Segoe UI, Tahoma, Geneva, Verdana, sans-serif, sans-serif", + "font-family": "Segoe UI, Tahoma, Geneva, Verdana, sans-serif, sans-serif", "font-weight": "lighter", - "margin-right": "10px", "font-size": "18px", + "margin-right": "10px", width: "80px", height: "80px", color: "white", diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 65038a5b..d596ffe2 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -112,6 +112,7 @@ import { AutoFixHigh as AutoFixHighIcon, Polyline as PolylineIcon, QueryStats as QueryStatsIcon, + AutoAwesome as AutoAwesomeIcon, } from "@mui/icons-material"; @@ -438,6 +439,7 @@ const AngularWorkflow = (defaultprops) => { const [appAuthentication, setAppAuthentication] = React.useState(undefined); const [variablesModalOpen, setVariablesModalOpen] = React.useState(false); + const [aiQueryModalOpen, setAiQueryModalOpen] = React.useState(false) const [executionVariablesModalOpen, setExecutionVariablesModalOpen] = React.useState(false); const [authenticationModalOpen, setAuthenticationModalOpen] = React.useState(false); @@ -592,17 +594,22 @@ const AngularWorkflow = (defaultprops) => { const [elements, setElements] = useState([]); const [loopRunning, setLoopRunning] = useState(false) + var loopRunning2 = loopRunning const stop = () => { setLoopRunning(false) + loopRunning2 = false } const start = () => { setLoopRunning(true) + loopRunning2 = true } useEffect(() => { - //console.log("In useeffect for loopRunning: ", loopRunning) - if (loopRunning) { + // Current variable + future state controlled + // This is so that the loop can stop itself as well + console.log("In useeffect for loopRunning: ", loopRunning, loopRunning2) + if (loopRunning && loopRunning2) { const intervalId = setInterval(() => { if (!loopRunning) { clearInterval(intervalId); @@ -983,19 +990,21 @@ const AngularWorkflow = (defaultprops) => { return response.json(); }) .then((responseJson) => { - if (responseJson !== undefined && responseJson !== null && responseJson.executions !== undefined && responseJson.executions !== null && responseJson.executions.length > 0) { + console.log("GOT A RESPONSE??") + if (responseJson !== undefined && responseJson !== null && responseJson.executions !== undefined && responseJson.executions !== null) { // - means it's opposite const newkeys = sortByKey(responseJson.executions, "-started_at"); setWorkflowExecutions(newkeys); const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; - var tmpView = new URLSearchParams(cursearch).get("execution_id"); if (execution_id !== undefined && execution_id !== null && execution_id.length > 0 && (tmpView === undefined || tmpView === null || tmpView.length === 0)) { tmpView = execution_id; } + console.log("EXECUTION ID: ", tmpView) + // Compare with currently selected item if (tmpView !== undefined && tmpView !== null && tmpView.length > 0) { // Don't clean up if it's already open @@ -1046,7 +1055,21 @@ const AngularWorkflow = (defaultprops) => { }, 5000); } } - } + } else { + const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; + var tmpView = new URLSearchParams(cursearch).get("execution_id"); + console.log("Alertnative execution id check: ", tmpView) + + if (tmpView === undefined || tmpView === null || tmpView.length === 0) { + const execution_id = tmpView; + setExecutionModalView(1); + setExecutionRequest({ + execution_id: execution_id, + }); + + start() + } + } }) .catch((error) => { //toast(error.toString()); @@ -1067,8 +1090,14 @@ const AngularWorkflow = (defaultprops) => { }) .then((response) => { if (response.status !== 200) { - console.log("Status not 200 for stream results :O!"); stop(); + setExecutionModalView(0); + toast("Failed loading the workflow run") + console.log("Status not 200 for stream results :O!"); + + //const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search; + //const newitem = removeParam("execution_id", cursearch); + //navigate(curpath + newitem) } return response.json(); @@ -1247,10 +1276,11 @@ const AngularWorkflow = (defaultprops) => { // Controls the colors and direction of execution results. // Style is in defaultCytoscapeStyle.js const handleUpdateResults = (responseJson, executionRequest) => { - if (responseJson === undefined || responseJson === null || responseJson.success === false) { - return - } - //console.log(responseJson) + if (responseJson === undefined || responseJson === null || responseJson.success === false) { + stop() + return + } +//console.log(responseJson) // Loop nodes and find results // Update on every interval? idk @@ -1263,7 +1293,8 @@ const AngularWorkflow = (defaultprops) => { //console.log("Updating data!") setExecutionData(responseJson) } else { - if (responseJson.status === "ABORTED" || responseJson.status === "STOPPED" || responseJson.status === "FAILURE" || responseJson.status === "WAITING") { + if (responseJson.status === "ABORTED" || responseJson.status === "STOPPED" || responseJson.status === "FAILURE" || responseJson.status === "WAITING" || responseJson.status === "FINISHED") { + console.log("DONE!") stop() } @@ -1321,6 +1352,7 @@ const AngularWorkflow = (defaultprops) => { }) }; + var streamDisabled2 = false const sendStreamRequest = (body) => { //console.log("Stream not activated yet.") if (!isCloud) { @@ -1356,6 +1388,9 @@ const AngularWorkflow = (defaultprops) => { .then((response) => { setSavingState(0); if (response.status !== 200) { + + setStreamDisabled(true) + streamDisabled2 = true //console.log("Status not 200 for stream :O!"); } @@ -1367,6 +1402,7 @@ const AngularWorkflow = (defaultprops) => { .catch((error) => { console.log("Stream send error: ", error.toString()) setStreamDisabled(true) + streamDisabled2 = true }) } @@ -2537,6 +2573,17 @@ const AngularWorkflow = (defaultprops) => { try { var chunkJson = JSON.parse(chunk) + if (chunkJson.success === false) { + console.log("Chunk failed: ", chunkJson) + + if (!streamDisabled) { + setStreamDisabled(true) + streamDisabled2 = true + } + return + } + + if (chunkJson.item !== undefined && chunkJson.item !== null && chunkJson.item !== "") { if (chunkJson.item === "node") { if (chunkJson.type === "move") { @@ -2561,6 +2608,13 @@ const AngularWorkflow = (defaultprops) => { } } catch (e) { console.log("Chunk JSON error: ", e) + + if (!streamDisabled) { + setStreamDisabled(true) + streamDisabled2 = true + } + + return } //data.push(chunk) @@ -2619,7 +2673,7 @@ const AngularWorkflow = (defaultprops) => { const streamUrl = "https://stream.shuffler.io" const url = `${streamUrl}/api/v1/workflows/${workflowId}/stream` while (true) { - if (streamDisabled) { + if (streamDisabled === true || streamDisabled2 === true) { break } @@ -3021,7 +3075,7 @@ const AngularWorkflow = (defaultprops) => { sendStreamRequest({ "item": "node", "type": "unselect", - "userid": userdata.id, + "id": workflow.id, }) //}, 150) }; @@ -4071,7 +4125,6 @@ const AngularWorkflow = (defaultprops) => { "item": "node", "type": "select", "id": data.id, - "userid": userdata.id, "location": { "x": event.target.position("x"), "y": event.target.position("y"), @@ -5985,10 +6038,10 @@ const AngularWorkflow = (defaultprops) => { const xParsed = destinationnodePosition.x - sourcenodePosition.x const yParsed = destinationnodePosition.y - sourcenodePosition.y - const z = Math.sqrt(xParsed * xParsed + yParsed * yParsed); - const costheta = xParsed / z; - const alpha = 0.25; - var controlPointDistance = [-alpha * yParsed * costheta, alpha * yParsed * costheta]; + const z = Math.sqrt(xParsed * xParsed + yParsed * yParsed) + const costheta = xParsed / z + const alpha = 0.3 + var controlPointDistance = [-alpha * yParsed * costheta, alpha * yParsed * costheta] var controlPointWeight = [alpha, 1 - alpha] //'control-point-weight': ['0.33', '0.66'], @@ -6151,8 +6204,18 @@ const AngularWorkflow = (defaultprops) => { const foundtriggers = inputworkflow.triggers.map((trigger) => { const node = {}; node.position = trigger.position; - node.data = trigger; + if (trigger.large_image === undefined || trigger.large_image === null || trigger.large_image.length === 0) { + + // Search triggers array for it where the name is matching and set image + var foundTrigger = triggers.find((t) => t.name === trigger.name) + if (foundTrigger !== undefined && foundTrigger !== null) { + console.log("Autofilled missing trigger image") + trigger.large_image = foundTrigger.large_image + } + } + + node.data = trigger; node.data._id = trigger["id"]; node.data.id = trigger["id"]; node.data.type = "TRIGGER"; @@ -6394,13 +6457,13 @@ const AngularWorkflow = (defaultprops) => { sendStreamRequest({ "item": "workflow", "type": "enter", - "userid": userdata.id, + "id": workflow.id, }) } const fetchRecommendations = (inputWorkflow) => { - //console.log("Disabled recommendations") - //return + console.log("Disabled recommendations as they were too inaccurate") + return const parsedWorkflow = JSON.parse(JSON.stringify(inputWorkflow)) @@ -6500,6 +6563,11 @@ const AngularWorkflow = (defaultprops) => { return response.json(); }) .then((responseJson) => { + if (responseJson === null) { + console.log("No revisions found") + return + } + if (responseJson.success === false) { console.log("Error getting workflow revisions: ", responseJson) return @@ -6553,13 +6621,16 @@ const AngularWorkflow = (defaultprops) => { } // App length necessary cus of cy initialization - if (elements.length === 0 && workflow.actions !== undefined && !graphSetup && Object.getOwnPropertyNames(workflow).length > 0 && workflowRecommendations !== undefined) { + // Not using recommendations, so skipping this for now + //if (elements.length === 0 && workflow.actions !== undefined && !graphSetup && Object.getOwnPropertyNames(workflow).length > 0 && workflowRecommendations !== undefined) { + if (elements.length === 0 && workflow.actions !== undefined && !graphSetup && Object.getOwnPropertyNames(workflow).length > 0) { setGraphSetup(true); setupGraph(workflow); console.log("In graph setup") // 2nd load - configures cytoscape - } else if (!established && cy !== undefined && ((apps !== null && apps !== undefined && apps.length > 0) || workflow.public === true) && Object.getOwnPropertyNames(workflow).length > 0 && appAuthentication !== undefined && workflowRecommendations !== undefined) { + //} else if (!established && cy !== undefined && ((apps !== null && apps !== undefined && apps.length > 0) || workflow.public === true) && Object.getOwnPropertyNames(workflow).length > 0 && appAuthentication !== undefined && workflowRecommendations !== undefined) { + } else if (!established && cy !== undefined && ((apps !== null && apps !== undefined && apps.length > 0) || workflow.public === true) && Object.getOwnPropertyNames(workflow).length > 0 && appAuthentication !== undefined) { console.log("In POST graph setup!") @@ -9086,6 +9157,33 @@ const AngularWorkflow = (defaultprops) => { backgroundColor: theme.palette.inputColor, }; + const aiQueryModal = + { + }} + > + + Condition + + + + const conditionsModal = ( { }} onChange={(event, newValue) => { // Workaround with event lol - console.log(event, newValue) + console.log("CHANGE: ", event, newValue) if (newValue !== undefined && newValue !== null) { var parsedvalue = JSON.parse(JSON.stringify(newValue)) parsedvalue.actions = [] @@ -11772,6 +11870,7 @@ const AngularWorkflow = (defaultprops) => { > { + console.log("CLICK: ", app) const newValue = app if (newValue !== undefined && newValue !== null) { @@ -13856,6 +13955,7 @@ const AngularWorkflow = (defaultprops) => { expansionModalOpen={codeEditorModalOpen} setExpansionModalOpen={setCodeEditorModalOpen} setEditorData={setEditorData} + setAiQueryModalOpen={setAiQueryModalOpen} /> } else if (Object.getOwnPropertyNames(selectedComment).length > 0) { @@ -14403,6 +14503,13 @@ const AngularWorkflow = (defaultprops) => { style={{ width: size, height: size }} /> ); + } else if (execution.execution_source === "ShuffleGPT") { + return ( + + ); } if ( @@ -15484,19 +15591,23 @@ const AngularWorkflow = (defaultprops) => { width: 30, }} onClick={() => { - const oldstartnode = cy.getElementById(data.action.id); - //console.log("FOUND NODe: ", oldstartnode) - if (oldstartnode !== undefined && oldstartnode !== null) { - const foundname = oldstartnode.data("label") - if (foundname !== undefined && foundname !== null) { - data.action.label = foundname - } - } + if (cy !== undefined) { + const oldstartnode = cy.getElementById(data.action.id); + //console.log("FOUND NODe: ", oldstartnode) + if (oldstartnode !== undefined && oldstartnode !== null) { + const foundname = oldstartnode.data("label") + if (foundname !== undefined && foundname !== null) { + data.action.label = foundname + } + } - //console.log("Click data: ", data) - //data.action.label = "" - setSelectedResult(data); - setCodeModalOpen(true); + //console.log("Click data: ", data) + //data.action.label = "" + setSelectedResult(data); + setCodeModalOpen(true); + } else { + toast("Please wait until the workflow is loaded and try again") + } }} > { {newView} + {aiQueryModal} {conditionsModal} {authenticationModal} {codePopoutModal} From ff7ca90936d35e0a602071e503ead7fee26fc17c Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 30 Nov 2023 11:14:55 +0100 Subject: [PATCH 23/33] Minor sdk update --- .env | 4 +- backend/app_sdk/app_base.py | 1 + frontend/src/components/Header.jsx | 72 +++---- frontend/src/components/NewHeader.jsx | 12 +- frontend/src/components/Oauth2Auth.jsx | 11 +- frontend/src/components/Priorities.jsx | 202 +++++++++++++++++++- frontend/src/components/RuntimeDebugger.jsx | 4 +- frontend/src/views/Admin.jsx | 4 +- frontend/src/views/AngularWorkflow.jsx | 2 +- frontend/src/views/AppCreator.jsx | 10 +- frontend/src/views/Apps.jsx | 2 +- functions/onprem/orborus/orborus.go | 4 +- 12 files changed, 274 insertions(+), 54 deletions(-) diff --git a/.env b/.env index f8be5532..f2e20b1f 100755 --- a/.env +++ b/.env @@ -70,7 +70,7 @@ SHUFFLE_SWARM_BRIDGE_DEFAULT_MTU=1500 # 1500 by default # Used for auto-cleanup of containers. REALLY important at scale. Set to false to see all container info. SHUFFLE_MEMCACHED= SHUFFLE_CONTAINER_AUTO_CLEANUP=true -SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY=3 # The amount of concurrent executions Orborus can handle. This is a soft limit, but it's recommended to keep it low. +SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY=7 # The amount of concurrent executions Orborus can handle. This is a soft limit, but it's recommended to keep it low. SHUFFLE_HEALTHCHECK_DISABLED=false SHUFFLE_ELASTIC=true SHUFFLE_LOGS_DISABLED=false @@ -93,4 +93,4 @@ SHUFFLE_OPENSEARCH_PROXY= SHUFFLE_OPENSEARCH_INDEX_PREFIX= SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY=true -DEBUG_MODE=false \ No newline at end of file +DEBUG_MODE=false diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 536fff21..156b2239 100755 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -3619,6 +3619,7 @@ class AppBase: timeout_env = os.getenv("SHUFFLE_APP_SDK_TIMEOUT", timeout) try: timeout = int(timeout_env) + self.logger.info(f"[DEBUG] Timeout set to {timeout} seconds") except Exception as e: self.logger.info(f"[WARNING] Failed parsing timeout to int: {e}") diff --git a/frontend/src/components/Header.jsx b/frontend/src/components/Header.jsx index 2acc229f..dbf68fe2 100644 --- a/frontend/src/components/Header.jsx +++ b/frontend/src/components/Header.jsx @@ -216,44 +216,44 @@ const { globalUrl, setNotifications, notifications, isLoggedIn, removeCookie, ho const NotificationItem = (props) => { const {data} = props - var image = ""; - var orgName = ""; - var orgId = ""; - if (userdata.orgs !== undefined) { - const foundOrg = userdata.orgs.find((org) => org.id === data["org_id"]); - if (foundOrg !== undefined && foundOrg !== null) { - //position: "absolute", bottom: 5, right: -5, - const imageStyle = { - width: imagesize, - height: imagesize, - pointerEvents: "none", - marginLeft: data.creator_org !== undefined && data.creator_org.length > 0 ? 20 : 0, - borderRadius: 10, - border: foundOrg.id === userdata.active_org.id ? `3px solid ${boxColor}` : null, - cursor: "pointer", - marginRight: 10, - }; + var image = ""; + var orgName = ""; + var orgId = ""; + if (userdata.orgs !== undefined) { + const foundOrg = userdata.orgs.find((org) => org.id === data["org_id"]); + if (foundOrg !== undefined && foundOrg !== null) { + //position: "absolute", bottom: 5, right: -5, + const imageStyle = { + width: imagesize, + height: imagesize, + pointerEvents: "none", + marginLeft: data.creator_org !== undefined && data.creator_org.length > 0 ? 20 : 0, + borderRadius: 10, + border: foundOrg.id === userdata.active_org.id ? `3px solid ${boxColor}` : null, + cursor: "pointer", + marginRight: 10, + }; - image = - foundOrg.image === "" ? ( - {foundOrg.name} - ) : ( - {foundOrg.name} {}} - /> - ); + image = + foundOrg.image === "" ? ( + {foundOrg.name} + ) : ( + {foundOrg.name} {}} + /> + ); - orgName = foundOrg.name; - orgId = foundOrg.id; - } - } + orgName = foundOrg.name; + orgId = foundOrg.id; + } + } return ( diff --git a/frontend/src/components/NewHeader.jsx b/frontend/src/components/NewHeader.jsx index f6f663b3..42d79ebd 100644 --- a/frontend/src/components/NewHeader.jsx +++ b/frontend/src/components/NewHeader.jsx @@ -106,6 +106,8 @@ const Header = (props) => { const clearNotifications = () => { // Don't really care about the logout + + toast("Clearing notifications") fetch(`${globalUrl}/api/v1/notifications/clear`, { credentials: "include", method: "GET", @@ -351,7 +353,7 @@ const Header = (props) => { setAnchorEl(event.currentTarget); }} > - + n.read === false).length} color="primary"> { >
- Your Notifications ({notifications.length}) + Your Notifications ({notifications.filter((data) => !data.read).length}) {notifications.length > 1 ? ( + {data.read === false ? ( + + ) : null} + + + + First seen: {new Date(data.created_at * 1000).toISOString().slice(0, 19)} + + + Last seen: {new Date(data.updated_at * 1000).toISOString().slice(0, 19)} + + + Times seen: {data.amount} + +
+ + ); + } + return (

Suggestions

@@ -125,6 +307,20 @@ const Priorities = (props) => { setShowRead(!showRead); }} />  Show read + {notifications === null || notifications === undefined || notifications.length === 0 ? null : +
+ {notifications.map((notification, index) => { + if (showRead === false && notification.read === true) { + return null + } + + return ( + + ) + })} +
+ } +
) } diff --git a/frontend/src/components/RuntimeDebugger.jsx b/frontend/src/components/RuntimeDebugger.jsx index ff2706f6..1b007021 100644 --- a/frontend/src/components/RuntimeDebugger.jsx +++ b/frontend/src/components/RuntimeDebugger.jsx @@ -181,6 +181,7 @@ const RuntimeDebugger = (props) => { }, []) const imageSize = 30 + const timenowUnix = Math.floor(Date.now() / 1000) const columns: GridColDef[] = [ { field: 'execution_source', @@ -319,7 +320,8 @@ const RuntimeDebugger = (props) => { }, { field: 'startTimestamp', headerName: 'Start time (UTC)', width: 160, renderCell: (params) => { - const hasError = params.row.completed_at-params.row.started_at > 300 + const comparisonTimestamp = params.row.completed_at === 0 ? timenowUnix : params.row.completed_at + const hasError = comparisonTimestamp-params.row.started_at > 300 return ( diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index ab9560fd..9a7884b0 100755 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -134,7 +134,7 @@ const FileCategoryInput = (props) => { const Admin = (props) => { - const { globalUrl, userdata, serverside, checkLogin } = props; + const { globalUrl, userdata, serverside, checkLogin, notifications, setNotifications, } = props; var to_be_copied = ""; const classes = useStyles(); @@ -2836,6 +2836,8 @@ If you're interested, please let me know a time that works for you, or set up a checkLogin={checkLogin} setAdminTab={setAdminTab} setCurTab={setCurTab} + notifications={notifications} + setNotifications={setNotifications} /> : adminTab === 3 ? { marginBottom: "auto", }} > - {data.action.label} + {data.action.label.replaceAll("_", " ")}
diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index bf82d4f6..15c2c094 100755 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -2511,10 +2511,16 @@ const AppCreator = (defaultprops) => { scheme: "basic", }; } else if (authenticationOption === "Oauth2") { - console.log("oauth2: ", parameterName) + console.log("oauth2: ", parameterName) var newparamName = parameterName.replaceAll('"', ""); newparamName = newparamName.replaceAll("'", ""); + // FIXME - this is a hack to get around the fact that the oauth2 + // flow is not properly defined + if (oauth2Type === "application") { + newparamName = "" + } + //parameterName, parameterValue, revocationUrl data.components.securitySchemes["Oauth2"] = { type: "oauth2", @@ -2945,7 +2951,7 @@ const AppCreator = (defaultprops) => { color="textSecondary" style={{ marginTop: 10 }} > - Base Authorization URL for Oauth2 + Authorization URL for Oauth2 { return response.json(); }) .then((responseJson) => { - //console.log("Apps: ", responseJson) + console.log("Apps: ", responseJson) //responseJson = sortByKey(responseJson, "large_image") //responseJson = sortByKey(responseJson, "is_valid") //setFilteredApps(responseJson.filter(app => !internalIds.includes(app.name) && !(!app.activated && app.generated))) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 92ddc27f..36523b4f 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -805,7 +805,7 @@ func deployWorker(image string, identifier string, env []string, executionReques log.Printf("[ERROR] Failed to start worker container in environment %s: %s", environment, err) return err } else { - log.Printf("[INFO] Worker Container %s was created under environment %s for execution %s: docker logs %s", cont.ID, environment, executionRequest.ExecutionId, cont.ID) + log.Printf("[INFO][%s] Worker Container created. Environment %s: docker logs %s", executionRequest.ExecutionId, environment, cont.ID) } //stats, err := cli.ContainerInspect(context.Background(), containerName) @@ -830,7 +830,7 @@ func deployWorker(image string, identifier string, env []string, executionReques // } //} } else { - log.Printf("[INFO] Worker Container %s was created under environment %s: docker logs %s", cont.ID, environment, cont.ID) + log.Printf("[INFO][%s] New Worker created. Environment %s: docker logs %s", executionRequest.ExecutionId, environment, cont.ID) } return nil From 706a8ee233b80a4f53ed5ccf25310c615c0e8e2a Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 30 Nov 2023 21:11:16 +0100 Subject: [PATCH 24/33] Changed apps to use milliseconds instead of seconds for timestamps to be more accurate --- .env | 2 +- backend/app_sdk/app_base.py | 20 +++++++++++--------- backend/go-app/main.go | 8 ++++++-- docker-compose.yml | 1 + functions/onprem/orborus/orborus.go | 2 +- 5 files changed, 20 insertions(+), 13 deletions(-) diff --git a/.env b/.env index f2e20b1f..73fbdf63 100755 --- a/.env +++ b/.env @@ -70,7 +70,7 @@ SHUFFLE_SWARM_BRIDGE_DEFAULT_MTU=1500 # 1500 by default # Used for auto-cleanup of containers. REALLY important at scale. Set to false to see all container info. SHUFFLE_MEMCACHED= SHUFFLE_CONTAINER_AUTO_CLEANUP=true -SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY=7 # The amount of concurrent executions Orborus can handle. This is a soft limit, but it's recommended to keep it low. +SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY=5 # The amount of concurrent executions Orborus can handle. This is a soft limit, but it's recommended to keep it low. SHUFFLE_HEALTHCHECK_DISABLED=false SHUFFLE_ELASTIC=true SHUFFLE_LOGS_DISABLED=false diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 156b2239..753849be 100755 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -302,9 +302,11 @@ class AppBase: self.authorization = os.getenv("AUTHORIZATION", "") self.current_execution_id = os.getenv("EXECUTIONID", "") self.full_execution = os.getenv("FULL_EXECUTION", "") - self.start_time = int(time.time()) self.result_wrapper_count = 0 + # Make start time with milliseconds + self.start_time = int(time.time_ns()) + self.action_result = { "action": self.action, "authorization": self.authorization, @@ -312,7 +314,7 @@ class AppBase: "result": f"", "started_at": self.start_time, "status": "", - "completed_at": int(time.time()), + "completed_at": int(time.time_ns()), } if isinstance(self.action, str): @@ -468,7 +470,7 @@ class AppBase: # Try it with some magic - action_result["completed_at"] = int(time.time()) + action_result["completed_at"] = int(time.time_ns()) self.logger.info(f"""[DEBUG] Inside Send result with status {action_result["status"]}""") #if isinstance(action_result, @@ -1011,7 +1013,7 @@ class AppBase: "result": f"All {len(param_multiplier)} values were non-unique", "started_at": self.start_time, "status": "SKIPPED", - "completed_at": int(time.time()), + "completed_at": int(time.time_ns()), } self.send_result(self.action_result, {"Content-Type": "application/json", "Authorization": "Bearer %s" % self.authorization}, "/api/v1/streams") @@ -1457,7 +1459,7 @@ class AppBase: "authorization": self.authorization, "execution_id": self.current_execution_id, "result": "", - "started_at": int(time.time()), + "started_at": int(time.time_ns()), "status": "EXECUTING" } @@ -2469,7 +2471,7 @@ class AppBase: self.action_result["result"] = f"Failed to parse LiquidPy: {error_msg}" print("[WARNING] Failed to set LiquidPy result") - self.action_result["completed_at"] = int(time.time()) + self.action_result["completed_at"] = int(time.time_ns()) self.send_result(self.action_result, headers, stream_path) self.logger.info(f"[ERROR] Sent FAILURE response to backend due to : {e}") @@ -3071,7 +3073,7 @@ class AppBase: self.logger.info("Failed one or more branch conditions.") self.action_result["result"] = tmpresult self.action_result["status"] = "SKIPPED" - self.action_result["completed_at"] = int(time.time()) + self.action_result["completed_at"] = int(time.time_ns()) self.send_result(self.action_result, headers, stream_path) return @@ -3557,7 +3559,7 @@ class AppBase: self.logger.info("[WARNING] SHOULD STOP EXECUTION BECAUSE FIELDS AREN'T UNIQUE") self.action_result["status"] = "SKIPPED" self.action_result["result"] = f"A non-unique value was found" - self.action_result["completed_at"] = int(time.time()) + self.action_result["completed_at"] = int(time.time_ns()) self.send_result(self.action_result, headers, stream_path) return @@ -3885,7 +3887,7 @@ class AppBase: }) # Send the result :) - self.action_result["completed_at"] = int(time.time()) + self.action_result["completed_at"] = int(time.time_ns()) self.send_result(self.action_result, headers, stream_path) #try: diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 7f561852..9717d6d3 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3706,6 +3706,10 @@ func runInitEs(ctx context.Context) { continue } + // FIXME: Add a randomized timer to avoid all schedules running at the same time + // Many are at 5 minutes / 1 hour. The point is to spread these out + // a bit instead of all of them starting at the exact same time + //log.Printf("Schedule: %#v", schedule) //log.Printf("Schedule time: every %d seconds", schedule.Seconds) jobret, err := newscheduler.Every(schedule.Seconds).Seconds().NotImmediately().Run(job(schedule)) @@ -3984,7 +3988,7 @@ func runInitEs(ctx context.Context) { r, err := git.Clone(storer, fs, cloneOptions) if err != nil { - log.Printf("[WARNING] Failed loading repo into memory (init): %s", err) + log.Printf("[ERROR] Failed loading repo into memory (init): %s", err) } dir, err := fs.ReadDir("") @@ -4021,7 +4025,7 @@ func runInitEs(ctx context.Context) { } _, err = git.Clone(storer, fs, cloneOptions) if err != nil { - log.Printf("[WARNING] Failed loading repo %s into memory: %s", apis, err) + log.Printf("[ERROR] Failed loading repo %s into memory: %s", apis, err) } else if err == nil && len(workflowapps) < 10 { log.Printf("[INFO] Finished git clone. Looking for updates to the repo.") dir, err := fs.ReadDir("") diff --git a/docker-compose.yml b/docker-compose.yml index ba3f4353..3ca6d2f0 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,6 +43,7 @@ services: - /var/run/docker.sock:/var/run/docker.sock environment: - SHUFFLE_APP_SDK_TIMEOUT=300 + - SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY=5 # The amount of concurrent executions Orborus can handle. #- DOCKER_HOST=tcp://docker-socket-proxy:2375 - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - BASE_URL=http://${OUTER_HOSTNAME}:5001 diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 36523b4f..b47d11e9 100755 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -63,7 +63,7 @@ var sleepTime = 2 // Making it work on low-end machines even during busy times :) // May cause some things to run slowly -var maxConcurrency = 3 +var maxConcurrency = 5 // Timeout if something rashes var workerTimeoutEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_TIMEOUT") From 63529e104fa3aca54cf0e0efe103e2e19dcf3ab1 Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 1 Dec 2023 14:30:28 +0100 Subject: [PATCH 25/33] Changes to runtime debugger --- backend/go-app/go.mod | 2 +- frontend/src/components/RuntimeDebugger.jsx | 261 ++++++++++++++++++-- frontend/src/views/AngularWorkflow.jsx | 5 +- 3 files changed, 251 insertions(+), 17 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 8e6d42ad..11da76e2 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -1,6 +1,6 @@ module shuffle-shared -//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared +replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared go 1.19 diff --git a/frontend/src/components/RuntimeDebugger.jsx b/frontend/src/components/RuntimeDebugger.jsx index 1b007021..d5df7d4e 100644 --- a/frontend/src/components/RuntimeDebugger.jsx +++ b/frontend/src/components/RuntimeDebugger.jsx @@ -4,6 +4,7 @@ import { TextField, Link, Button, + ButtonGroup, CircularProgress, Select, MenuList, @@ -13,6 +14,7 @@ import { Autocomplete, Tooltip, Typography, + IconButton, } from '@mui/material'; import { toast } from "react-toastify" @@ -31,6 +33,8 @@ import { import { OpenInNew as OpenInNewIcon, PlayArrow as PlayArrowIcon, + Insights as InsightsIcon, + Replay as ReplayIcon, } from '@mui/icons-material'; import { DataGrid, GridColDef, GridValueGetterParams } from '@mui/x-data-grid' @@ -56,6 +60,7 @@ const RuntimeDebugger = (props) => { const [rowCursor, setCursor] = useState("") const [rowsPerPage, setRowsPerPage] = useState(10) const [resultRows, setResultRows] = useState([]) + const [selectedWorkflowExecutions, setSelectedWorkflowExecutions] = useState([]) const [workflows, setWorkflows] = useState([ {"id": "", "name": "All Workflows",} ]) @@ -198,6 +203,9 @@ const RuntimeDebugger = (props) => { } else if (source === "subflow" || source.length === 36) { foundSource = subflow source = "subflow" + } else if (source === "rerun" || source.length === 36) { + foundSource = + source = "rerun of a previous run" } else { source = "manual" } @@ -278,20 +286,31 @@ const RuntimeDebugger = (props) => { width: 75, renderCell: (params) => { var foundItems = 0 - var extraItems = 0 + var foundSkipped = 0 if (params.row.results !== null && params.row.results !== undefined) { for (let key in params.row.results) { if (params.row.results[key].status === "SUCCESS") { foundItems += 1 } + + if (params.row.results[key].status === "SKIPPED") { + foundSkipped += 1 + } } } + var foundError = "" + if (foundItems + foundSkipped < params.row.workflow.actions.length && params.row.status === "FINISHED") { + foundError = "Workflow is done, but all nodes are not finished. This most likely indicates a problem with the workflow" + } + return ( - { - }}> - {foundItems} - + + 0 ? "rgba(244,0,0,0.6)" : "inherit"}} onClick={() => { + }}> + {foundItems} + + ) }, }, @@ -301,7 +320,6 @@ const RuntimeDebugger = (props) => { width: 75, renderCell: (params) => { var foundItems = 0 - var extraItems = 0 if (params.row.results !== null && params.row.results !== undefined) { for (let key in params.row.results) { if (params.row.results[key].status === "SKIPPED") { @@ -353,14 +371,81 @@ const RuntimeDebugger = (props) => { { field: 'id', headerName: 'Explore', - width: 65, + width: 100, renderCell: (params) => { - const parsedResult = params.row.result === null || params.row.result === undefined || params.row.result === "" ? null : params.row.result - const hasError = parsedResult !== null && parsedResult !== undefined && parsedResult !== "" ? parsedResult.includes("{%") && parsedResult.includes("%}") : false + const parsedResult = params.row.result === null || params.row.result === undefined || params.row.result === "" ? "" : params.row.result + + var errorReason = "" + var hasError = parsedResult !== null && parsedResult !== undefined && parsedResult !== "" ? parsedResult.includes("{%") && parsedResult.includes("%}") : false + + if (hasError) { + errorReason = "Liquid parsing error" + } + + // if success: false + // if node == FAILURE or ABORTED + if (parsedResult.includes(`\"success\": false`)) { + errorReason = "success: false in last result" + hasError = true + } + + if (!hasError && parsedResult.includes(`\"status\":`)) { + // Look for any status that is 300 or higher + const statusSplit = parsedResult.split(`\"status\":`) + if (statusSplit.length > 1) { + var foundStatus = statusSplit[1].trim() + // Check if pattern is \d, + if (foundStatus.includes(",")) { + const foundStatusSplit = foundStatus.split(",") + + if (foundStatusSplit.length > 1) { + foundStatus = foundStatusSplit[0].trim() + // Check if it's a number + } + } else { + foundStatus = "" + } + + if (!isNaN(foundStatus) && foundStatus >= 300) { + errorReason = "Status code: "+foundStatus + hasError = true + } + } + } + + if (!hasError) { + // Find last node that isn't skipped and check status + var lastresult = {} + for (var key in params.row.results) { + const result = params.row.results[key] + if (result.status === "SKIPPED") { + continue + } + + if (result.completed_at === undefined || result.completed_at === null) { + continue + } + + if (result.completed_at >= lastresult.completed_at) { + lastresult = result + } + } + + if (lastresult.id !== undefined && lastresult.status !== "SUCCESS" && lastresult.status !== "SKIPPED") { + errorReason = "Bad status for last node: "+lastresult.status + hasError = true + } + } + + if (!hasError && params.row.notifications_created !== null && params.row.notifications_created !== undefined && params.row.notifications_created !== 0) { + hasError = true + errorReason = "Generated notifications: "+params.row.notifications_created + } return ( + + Workflow result: {errorReason}

{params.row.result !== null && params.row.result !== undefined && params.row.result !== "" ? params.row.result : @@ -368,10 +453,19 @@ const RuntimeDebugger = (props) => { } }> - + - + + { + window.open(`${globalUrl}/api/v1/workflows/search/${params.row.id}`, "_blank") + }} + disabled={!userdata.support} + > + + ) @@ -383,7 +477,6 @@ const RuntimeDebugger = (props) => { // Check if the user is currently focusing a texxtfield or not // If they are, don't submit the search if (document.activeElement.tagName === "INPUT") { - console.log("User is focusing a textfield, not submitting search") return } @@ -403,6 +496,29 @@ const RuntimeDebugger = (props) => { setEndTime(date) } + const abortExecution = (workflowId, executionId) => { + fetch(`${globalUrl}/api/v1/workflows/${workflowId}/executions/${executionId}/abort`, + { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + } + ) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for ABORT EXECUTION :O!"); + } + + return response.json(); + }) + .catch((error) => { + toast.error("Error aborting execution: "+error.toString()) + }); + }; + const handleWorkflowSelectionUpdate = (e, isUserinput) => { if (e.target.value === undefined || e.target.value === null || e.target.value.id === undefined) { console.log("Returning as there's no id") @@ -417,9 +533,106 @@ const RuntimeDebugger = (props) => { submitSearch(e.target.value.id, status, startTime, endTime, rowCursor, rowsPerPage) } + const executeWorkflow = (execution) => { + const data = { + execution_argument: execution.execution_argument, + start: execution.start, + execution_source: "rerun", + }; + + fetch(`${globalUrl}/api/v1/workflows/${execution.workflow.id}/execute?start=${execution.start}`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + body: JSON.stringify(data), + } + ) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for WORKFLOW EXECUTION :O!"); + } + + return response.json(); + }) + .then((responseJson) => { + if (!responseJson.success) { + toast("Error executing workflow: "+responseJson.error) + } else { + console.log("Executed workflow: ", responseJson) + } + }) + .catch((error) => { + toast("Failed to execute workflow: "+error.toString()) + }); + } + return (
-

Workflow Run Debugger

+ +
+

Workflow Run Debugger

+ {selectedWorkflowExecutions.length > 0 ? + + + + + + + + + + : null} +
{ submitSearch(workflowId, status, startTime, endTime, rowCursor, rowsPerPage) }} style={{display: "flex", }}> @@ -614,6 +827,26 @@ const RuntimeDebugger = (props) => { onPageChange={(params) => { console.log("params: ", params) }} + onSelectionModelChange={(newSelection) => { + //console.log("newSelection: ", newSelection) + //setSelectedWorkflowExecutionsIndexes(newSelection) + var found = [] + for (var i = 0; i < newSelection.length; i++) { + // Find the workflow in the resultRows + var selected = resultRows.find((workflow) => { + return workflow.id === newSelection[i] + }) + + if (selected === undefined || selected === null) { + continue + } + + found.push(selected) + } + + setSelectedWorkflowExecutions(found) + }} + // Track which items are selected />
diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index b7cfea17..6bde408c 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -12074,7 +12074,7 @@ const AngularWorkflow = (defaultprops) => { workflow.triggers[selectedTriggerIndex].parameters[0].value } color="primary" - placeholder="defaultValue" + placeholder="10" onBlur={(e) => { setTriggerCronWrapper(e.target.value); }} @@ -15628,7 +15628,8 @@ const AngularWorkflow = (defaultprops) => { marginBottom: "auto", }} > - {data.action.label.replaceAll("_", " ")} + {data.action.label === undefined || data.action.label === null || data.action.label === "" ? data.action.label : data.action.label.replaceAll("_", " ")} +
From dee1ee48f84fac5dcc608cadf3884829e58c2810 Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 1 Dec 2023 14:30:44 +0100 Subject: [PATCH 26/33] Gomod fix for backend --- backend/go-app/go.mod | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 11da76e2..6343a95d 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -1,6 +1,6 @@ module shuffle-shared -replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared +//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared go 1.19 @@ -18,7 +18,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.4.98 + github.com/shuffle/shuffle-shared v0.5.11 golang.org/x/crypto v0.14.0 google.golang.org/api v0.125.0 google.golang.org/grpc v1.55.0 From ec726fe073499eafbbd64ce8a281a5aaea08e32e Mon Sep 17 00:00:00 2001 From: Frikky Date: Sun, 3 Dec 2023 01:42:03 +0100 Subject: [PATCH 27/33] Even more runtime debugger and docs fixes --- backend/app_sdk/app_base.py | 1 + backend/go-app/go.mod | 2 +- backend/go-app/go.sum | 4 + frontend/src/components/RuntimeDebugger.jsx | 117 +++++++++++++++----- frontend/src/views/AngularWorkflow.jsx | 15 ++- frontend/src/views/Docs.jsx | 14 ++- 6 files changed, 115 insertions(+), 38 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 753849be..a5467f76 100755 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -522,6 +522,7 @@ class AppBase: break else: self.logger.info(f"[ERROR] Bad resp {ret.status_code}: {ret.text}") + time.sleep(sleeptime) except requests.exceptions.RequestException as e: self.logger.info(f"[DEBUG] Request problem: {e}") diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 6343a95d..c93d1f5f 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -18,7 +18,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.5.11 + github.com/shuffle/shuffle-shared v0.5.14 golang.org/x/crypto v0.14.0 google.golang.org/api v0.125.0 google.golang.org/grpc v1.55.0 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index f31286be..4812fee4 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -416,6 +416,10 @@ github.com/shuffle/shuffle-shared v0.4.97 h1:1c8LdNteMykKNEV97vwP63oSP2tV/Uso3O4 github.com/shuffle/shuffle-shared v0.4.97/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= github.com/shuffle/shuffle-shared v0.4.98 h1:pgsLdWUpxZ/q+eHpAjOCH9icOsmuO5u2olmirOldy5A= github.com/shuffle/shuffle-shared v0.4.98/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= +github.com/shuffle/shuffle-shared v0.5.11 h1:Eqbs9o8E49QAL5/6aV6BfFtWSjLIvgET7AL3fa4OQTg= +github.com/shuffle/shuffle-shared v0.5.11/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= +github.com/shuffle/shuffle-shared v0.5.14 h1:d14u1e4k+qKgnf4Insq4x2S+0MMKlDqdyTTyVP3puRA= +github.com/shuffle/shuffle-shared v0.5.14/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= diff --git a/frontend/src/components/RuntimeDebugger.jsx b/frontend/src/components/RuntimeDebugger.jsx index d5df7d4e..1b4f431f 100644 --- a/frontend/src/components/RuntimeDebugger.jsx +++ b/frontend/src/components/RuntimeDebugger.jsx @@ -56,6 +56,7 @@ const RuntimeDebugger = (props) => { const [startTime, setStartTime] = useState("") const [workflow, setWorkflow] = useState({}) + const [ignoreOrg, setIgnoreOrg] = useState(false) const [searchLoading, setSearchLoading] = useState(false) const [rowCursor, setCursor] = useState("") const [rowsPerPage, setRowsPerPage] = useState(10) @@ -85,6 +86,7 @@ const RuntimeDebugger = (props) => { status: status.toUpperCase(), start_time: startTime, end_time: endTime, + ignore_org: ignoreOrg, } fetch(`${globalUrl}/api/v1/workflows/search`, { @@ -185,6 +187,38 @@ const RuntimeDebugger = (props) => { } }, []) + const forceContinue = (execution) => { + console.log(`FORCE CONTINUE execution ${execution.execution_id} for workflow ${execution.workflow.id}`) + + fetch(`${globalUrl}/api/v1/workflows/${execution.workflow.id}/executions/${execution.execution_id}/rerun`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + credentials: "include", + }) + .then((response) => response.json()) + .then((data) => { + if (data.success) { + if (data.reason !== undefined && data.reason !== null && data.reason !== "") { + toast("Successful response: " + data.reason) + } else { + toast("Successfully forced continue") + } + } else { + if (data.reason !== undefined && data.reason !== null && data.reason !== "") { + toast(`Failed to force continue: ${data.reason}`) + } else { + toast("Failed to force continue") + } + } + }) + .catch((error) => { + console.error("Error:", error); + toast(`Failed to force continue: ${error}`) + }) + } + const imageSize = 30 const timenowUnix = Math.floor(Date.now() / 1000) const columns: GridColDef[] = [ @@ -371,7 +405,7 @@ const RuntimeDebugger = (props) => { { field: 'id', headerName: 'Explore', - width: 100, + width: 120, renderCell: (params) => { const parsedResult = params.row.result === null || params.row.result === undefined || params.row.result === "" ? "" : params.row.result @@ -443,31 +477,47 @@ const RuntimeDebugger = (props) => { } return ( - - Workflow result: {errorReason}

- {params.row.result !== null && params.row.result !== undefined && params.row.result !== "" ? - params.row.result - : - null - } - - }> - - - - - { - window.open(`${globalUrl}/api/v1/workflows/search/${params.row.id}`, "_blank") - }} - disabled={!userdata.support} - > - - - - +
+ + Workflow result: {errorReason}

+ {params.row.result !== null && params.row.result !== undefined && params.row.result !== "" ? + params.row.result + : + null + } + + }> + + + + + + + + { + forceContinue(params.row) + + }} + > + + + + + { + window.open(`${globalUrl}/api/v1/workflows/search/${params.row.id}`, "_blank") + }} + disabled={!userdata.support} + > + + + +
) } }, @@ -632,6 +682,19 @@ const RuntimeDebugger = (props) => { : null} + + {userdata.support === true ? + + : null}
{ submitSearch(workflowId, status, startTime, endTime, rowCursor, rowsPerPage) @@ -799,7 +862,7 @@ const RuntimeDebugger = (props) => { + + + ) : null} {executionData.workflow !== undefined && executionData.workflow !== null && executionData.workflow.actions !== undefined && executionData.workflow.actions !== null && executionData.workflow.actions.length > 0 && executionData.workflow.actions[0].environment !== "Cloud" ?
diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 981bc7a0..73f02cc2 100755 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -1344,7 +1344,7 @@ const Workflows = (props) => { }, i * 200); } - toast(`exporting and keeping original for all ${allWorkflows.length} workflows`); + toast(`Exporting and keeping original for all ${allWorkflows.length} workflows`); }; const deduplicateIds = (data, skip_sanitize) => { diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index fef487a1..4d2d9450 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -9,7 +9,7 @@ require ( github.com/mackerelio/go-osstat v0.2.3 github.com/satori/go.uuid v1.2.0 github.com/shirou/gopsutil v3.21.11+incompatible - github.com/shuffle/shuffle-shared v0.4.96 + github.com/shuffle/shuffle-shared v0.5.29 k8s.io/api v0.28.1 k8s.io/apimachinery v0.28.1 k8s.io/client-go v0.28.1 diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index c9eea384..fa8b721c 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -278,6 +278,8 @@ github.com/shuffle/shuffle-shared v0.4.95 h1:xr92/03/uQeJiDme9S8/vgF1KWyQgJ1KQXV github.com/shuffle/shuffle-shared v0.4.95/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= github.com/shuffle/shuffle-shared v0.4.96 h1:iaIB/HP9eKpw9DMMJZhSLDbKdHJt075kFYLHg9AaiiM= github.com/shuffle/shuffle-shared v0.4.96/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= +github.com/shuffle/shuffle-shared v0.5.29 h1:n4vThl7v3mFVXbrIW71XREFdmZZo7mOBAWxnsdiNjDk= +github.com/shuffle/shuffle-shared v0.5.29/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index e4cf2262..9e0f8e1e 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -11,7 +11,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.4.57 + github.com/shuffle/shuffle-shared v0.5.29 k8s.io/api v0.28.3 k8s.io/apimachinery v0.28.3 k8s.io/client-go v0.28.3 diff --git a/functions/onprem/worker/go.sum b/functions/onprem/worker/go.sum index f26caf60..5afdce76 100644 --- a/functions/onprem/worker/go.sum +++ b/functions/onprem/worker/go.sum @@ -284,6 +284,8 @@ github.com/shuffle/shuffle-shared v0.4.50 h1:fJLfhWIJ5mYap4JwHnD/B5aaLyIULwylFSl github.com/shuffle/shuffle-shared v0.4.50/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= github.com/shuffle/shuffle-shared v0.4.57 h1:o+mMPRY4ourkE3R0qdi80jg6RlCtvAJ/VVrPk4y75Hk= github.com/shuffle/shuffle-shared v0.4.57/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= +github.com/shuffle/shuffle-shared v0.5.29 h1:n4vThl7v3mFVXbrIW71XREFdmZZo7mOBAWxnsdiNjDk= +github.com/shuffle/shuffle-shared v0.5.29/go.mod h1:X613gbo0dT3fnYvXDRwjQZyLC+T49T2nSQOrCV5QMlI= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index fb913d16..0c8fd3a5 100755 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -3,7 +3,7 @@ package main import ( "github.com/shuffle/shuffle-shared" - //"bufio" + "bytes" "context" "encoding/json" @@ -14,29 +14,23 @@ import ( "log" "net" "net/http" + "net/http/pprof" "net/url" "os" + "strconv" "strings" "time" "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/container" - //"github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/mount" dockerclient "github.com/docker/docker/client" - //"github.com/go-git/go-billy/v5/memfs" - - //newdockerclient "github.com/fsouza/go-dockerclient" - //"github.com/satori/go.uuid" + // This is for automatic removal of certain code :) "github.com/gorilla/mux" - "github.com/patrickmn/go-cache" "github.com/satori/go.uuid" - // No necessary outside shared - "cloud.google.com/go/datastore" - "cloud.google.com/go/storage" - //k8s deps corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -45,24 +39,24 @@ import ( "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/util/homedir" "path/filepath" - // "k8s.io/client-go/util/retry" ) // This is getting out of hand :) -var environment = os.Getenv("ENVIRONMENT_NAME") +var timezone = os.Getenv("TZ") var baseUrl = os.Getenv("BASE_URL") var appCallbackUrl = os.Getenv("BASE_URL") +var isKubernetes = os.Getenv("IS_KUBERNETES") +var environment = os.Getenv("ENVIRONMENT_NAME") +var logsDisabled = os.Getenv("SHUFFLE_LOGS_DISABLED") var cleanupEnv = strings.ToLower(os.Getenv("CLEANUP")) -var dockerApiVersion = strings.ToLower(os.Getenv("DOCKER_API_VERSION")) var swarmNetworkName = os.Getenv("SHUFFLE_SWARM_NETWORK_NAME") -var timezone = os.Getenv("TZ") +var dockerApiVersion = strings.ToLower(os.Getenv("DOCKER_API_VERSION")) var baseimagename = "frikky/shuffle" // var baseimagename = "registry.hub.docker.com/frikky/shuffle" var registryName = "registry.hub.docker.com" var sleepTime = 2 -var requestCache *cache.Cache var topClient *http.Client var data string var requestsSent = 0 @@ -84,11 +78,21 @@ var startAction string //var allLogs map[string]string //var containerIds []string var downloadedImages []string +type ImageDownloadBody struct { + Image string `json:"image"` +} + +type ImageRequest struct { + Image string `json:"image"` +} + +var finishedExecutions []string + // Images to be autodeployed in the latest version of Shuffle. var autoDeploy = map[string]string{ - "http:1.3.0": "frikky/shuffle:http_1.3.0", "http:1.4.0": "frikky/shuffle:http_1.4.0", + "http:1.3.0": "frikky/shuffle:http_1.3.0", "shuffle-tools:1.2.0": "frikky/shuffle:shuffle-tools_1.2.0", "shuffle-subflow:1.0.0": "frikky/shuffle:shuffle-subflow_1.0.0", "shuffle-subflow:1.1.0": "frikky/shuffle:shuffle-subflow_1.1.0", @@ -108,6 +112,166 @@ type UserInputSubflow struct { CancelUrl string `json:"cancel_url"` } +// Not using shuffle.SetWorkflowExecution as we only want to use cache in reality +func setWorkflowExecution(ctx context.Context, workflowExecution shuffle.WorkflowExecution, dbSave bool) error { + if len(workflowExecution.ExecutionId) == 0 { + log.Printf("[DEBUG] Workflowexecution executionId can't be empty.") + return errors.New("ExecutionId can't be empty.") + } + + //log.Printf("[DEBUG][%s] Setting with %d results (pre)", workflowExecution.ExecutionId, len(workflowExecution.Results)) + workflowExecution = shuffle.Fixexecution(ctx, workflowExecution) + cacheKey := fmt.Sprintf("workflowexecution_%s", workflowExecution.ExecutionId) + + execData, err := json.Marshal(workflowExecution) + if err != nil { + log.Printf("[ERROR] Failed marshalling execution during set: %s", err) + return err + } + + err = shuffle.SetCache(ctx, cacheKey, execData, 30) + if err != nil { + log.Printf("[ERROR][%s] Failed adding to cache during setexecution", workflowExecution) + return err + } + + + handleExecutionResult(workflowExecution) + validated := shuffle.ValidateFinished(ctx, -1, workflowExecution) + if validated { + shutdownData, err := json.Marshal(workflowExecution) + if err != nil { + log.Printf("[ERROR] Failed marshalling shutdowndata during set: %s", err) + } + + log.Printf("[DEBUG][%s] Sending result (set)", workflowExecution.ExecutionId) + sendResult(workflowExecution, shutdownData) + return nil + } + + // FIXME: Should this shutdown OR send the result? + // The worker may not be running the backend hmm + if dbSave { + if workflowExecution.ExecutionSource == "default" { + log.Printf("[DEBUG][%s] Shutting down (25)", workflowExecution.ExecutionId) + shutdown(workflowExecution, "", "", true) + //return + } else { + log.Printf("[DEBUG][%s] NOT shutting down with dbSave (%s). Instead sending result to backend and start polling until subflow is updated", workflowExecution.ExecutionId, workflowExecution.ExecutionSource) + + shutdownData, err := json.Marshal(workflowExecution) + if err != nil { + log.Printf("[ERROR] Failed marshalling shutdowndata during dbSave handler: %s", err) + } + + sendResult(workflowExecution, shutdownData) + + // Poll for 1 minute max if there is a "wait for results" subflow + subflowId := "" + for _, result := range workflowExecution.Results { + if result.Status == "WAITING" { + //log.Printf("[DEBUG][%s] Found waiting result", workflowExecution.ExecutionId) + subflowId = result.Action.ID + } + } + + if len(subflowId) == 0 { + log.Printf("[DEBUG][%s] No waiting result found. Not polling", workflowExecution.ExecutionId) + + for _, action := range workflowExecution.Workflow.Actions { + if action.AppName == "User Input" || action.AppName == "Shuffle Workflow" || action.AppName == "shuffle-subflow" { + workflowExecution.Workflow.Triggers = append(workflowExecution.Workflow.Triggers, shuffle.Trigger{ + AppName: action.AppName, + Parameters: action.Parameters, + ID: action.ID, + }) + } + } + + + for _, trigger := range workflowExecution.Workflow.Triggers { + //log.Printf("[DEBUG] Found trigger %s", trigger.AppName) + if trigger.AppName != "User Input" && trigger.AppName != "Shuffle Workflow" && trigger.AppName != "shuffle-subflow" { + continue + } + + + // check if it has wait for results in params + wait := false + for _, param := range trigger.Parameters { + //log.Printf("[DEBUG] Found param %s with value %s", param.Name, param.Value) + if param.Name == "check_result" && strings.ToLower(param.Value) == "true" { + //log.Printf("[DEBUG][%s] Found check result param!", workflowExecution.ExecutionId) + wait = true + break + } + } + + if wait { + // Check if it has a result or not + found := false + for _, result := range workflowExecution.Results { + //log.Printf("[DEBUG][%s] Found result %s", workflowExecution.ExecutionId, result.Action.ID) + if result.Action.ID == trigger.ID && result.Status != "SUCCESS" && result.Status != "FAILURE" { + //log.Printf("[DEBUG][%s] Found subflow result that is not handled. Waiting for results", workflowExecution.ExecutionId) + + subflowId = result.Action.ID + found = true + break + } + } + + if !found { + log.Printf("[DEBUG][%s] No result found for subflow. Setting subflowId to %s", workflowExecution.ExecutionId, trigger.ID) + subflowId = trigger.ID + } + } + + if len(subflowId) > 0 { + break + } + } + } + + if len(subflowId) > 0 { + // Under rerun period timeout + timeComparison := 120 + log.Printf("[DEBUG][%s] Starting polling for %d seconds to see if new subflow updates are found on the backend that are not handled. Subflow ID: %s", workflowExecution.ExecutionId, timeComparison, subflowId) + timestart := time.Now() + streamResultUrl := fmt.Sprintf("%s/api/v1/streams/results", baseUrl) + for { + err = handleSubflowPoller(ctx, workflowExecution, streamResultUrl, subflowId) + if err == nil { + log.Printf("[DEBUG] Subflow is finished and we are breaking the thingy") + + if os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "swarm" && workflowExecution.ExecutionSource != "default" { + log.Printf("[DEBUG] Force shutdown of worker due to optimized run with webserver. Expecting reruns to take care of this") + os.Exit(0) + } + + + break + } + + timepassed := time.Since(timestart) + if timepassed.Seconds() > float64(timeComparison) { + log.Printf("[DEBUG][%s] Max poll time reached to look for updates. Stopping poll. This poll is here to send personal results back to itself to be handled, then to stop this thread.", workflowExecution.ExecutionId) + break + } + + // Sleep for 1 second + time.Sleep(1 * time.Second) + } + } else { + log.Printf("[DEBUG][%s] No need to poll for results. Not polling", workflowExecution.ExecutionId) + } + } + } + + return nil +} + + // removes every container except itself (worker) func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason string, handleResultSend bool) { log.Printf("[DEBUG][%s] Shutdown (%s) started with reason %#v. Result amount: %d. ResultsSent: %d, Send result: %#v, Parent: %#v", workflowExecution.ExecutionId, workflowExecution.Status, reason, len(workflowExecution.Results), requestsSent, handleResultSend, workflowExecution.ExecutionParent) @@ -126,6 +290,30 @@ func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason time.Sleep(time.Duration(sleepDuration) * time.Second) } + // Might not be necessary because of cleanupEnv hostconfig autoremoval + if cleanupEnv == "true" && (os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "swarm") { + /* + ctx := context.Background() + dockercli, err := dockerclient.NewEnvClient() + if err == nil { + log.Printf("[INFO] Cleaning up %d containers", len(containerIds)) + removeOptions := types.ContainerRemoveOptions{ + RemoveVolumes: true, + Force: true, + } + + for _, containername := range containerIds { + log.Printf("[INFO] Should stop and and remove container %s (deprecated)", containername) + //dockercli.ContainerStop(ctx, containername, nil) + //dockercli.ContainerRemove(ctx, containername, removeOptions) + //removeContainers = append(removeContainers, containername) + } + } + */ + } else { + + } + if len(reason) > 0 && len(nodeId) > 0 { //log.Printf("[INFO] Running abort of workflow because it should be finished") @@ -138,7 +326,6 @@ func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason path += fmt.Sprintf("&env=%s", url.QueryEscape(environment)) } - //fmt.Printf(url.QueryEscape(query)) abortUrl += path log.Printf("[DEBUG][%s] Abort URL: %s", workflowExecution.ExecutionId, abortUrl) @@ -152,19 +339,22 @@ func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason log.Printf("[WARNING][%s] Failed building request: %s", workflowExecution.ExecutionId, err) } - authorization := os.Getenv("AUTHORIZATION") - if len(authorization) > 0 { - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", authorization)) + // FIXME: Add an API call to the backend + if os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "swarm" { + authorization := os.Getenv("AUTHORIZATION") + if len(authorization) > 0 { + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", authorization)) + } else { + log.Printf("[ERROR][%s] No authorization specified for abort", workflowExecution.ExecutionId) + } } else { - log.Printf("[ERROR][%s] No authorization specified for abort", workflowExecution.ExecutionId) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", workflowExecution.Authorization)) } req.Header.Add("Content-Type", "application/json") - client := shuffle.GetExternalClient(baseUrl) - //log.Printf("[DEBUG][%s] All App Logs: %#v", workflowExecution.ExecutionId, allLogs) - newresp, err := client.Do(req) + newresp, err := topClient.Do(req) if err != nil { log.Printf("[WARNING][%s] Failed abort request: %s", workflowExecution.ExecutionId, err) } else { @@ -178,63 +368,17 @@ func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason //Finished shutdown (after %d seconds). ", sleepDuration) // Allows everything to finish in subprocesses (apps) - time.Sleep(time.Duration(sleepDuration) * time.Second) - os.Exit(3) -} - -// } - -func isRunningInCluster() bool { - _, existsHost := os.LookupEnv("KUBERNETES_SERVICE_HOST") - _, existsPort := os.LookupEnv("KUBERNETES_SERVICE_PORT") - return existsHost && existsPort -} - -func buildEnvVars(envMap map[string]string) []corev1.EnvVar { - var envVars []corev1.EnvVar - for key, value := range envMap { - envVars = append(envVars, corev1.EnvVar{Name: key, Value: value}) - } - return envVars -} - -func getKubernetesClient() (*kubernetes.Clientset, error) { - if isRunningInCluster() { - config, err := rest.InClusterConfig() - if err != nil { - return nil, err - } - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - return nil, err - } - return clientset, nil + if os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "swarm" { + time.Sleep(time.Duration(sleepDuration) * time.Second) + os.Exit(3) } else { - home := homedir.HomeDir() - kubeconfigPath := filepath.Join(home, ".kube", "config") - config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) - if err != nil { - return nil, err - } - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - return nil, err - } - return clientset, nil + log.Printf("[DEBUG][%s] Sending result and resetting values (K8s & Swarm).", workflowExecution.ExecutionId) } } // Deploys the internal worker whenever something happens func deployApp(cli *dockerclient.Client, image string, identifier string, env []string, workflowExecution shuffle.WorkflowExecution, action shuffle.Action) error { - // log.Printf("################################### new call to deployApp ###################################") - // log.Printf("image: %s", image) - // log.Printf("identifier: %s", identifier) - // log.Printf("execution: %+v", workflowExecution) - log.Printf("[DEBUG] Adding SHUFFLE_APP_SDK_TIMEOUT=%s", os.Getenv("SHUFFLE_APP_SDK_TIMEOUT")) - env = append(env, fmt.Sprintf("SHUFFLE_APP_SDK_TIMEOUT=%s", os.Getenv("SHUFFLE_APP_SDK_TIMEOUT"))) - - if os.Getenv("IS_KUBERNETES") == "true" { - + if isKubernetes == "true" { namespace := "shuffle" localRegistry := os.Getenv("REGISTRY_URL") @@ -248,8 +392,9 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] clientset, err := getKubernetesClient() if err != nil { - fmt.Println("[ERROR]Error getting kubernetes client:", err) - // os.Exit(1) + log.Printf("[ERROR] Failed getting kubernetes: %s [INFO] Setting kubernetes to false to enable running Shuffle with Docker for the next iterations.", err) + isKubernetes = "false" + return err } log.Printf("[DEBUG] Got kubernetes client") @@ -264,8 +409,6 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] appName := strings.Join(appDetailsSplit[:len(appDetailsSplit)-1], "_") appVersion := appDetailsSplit[len(appDetailsSplit)-1] - // log.Printf("APP VERSION IS: %s", appVersion) - for _, app := range workflowExecution.Workflow.Actions { // log.Printf("[DEBUG] App: %s, Version: %s", appName, appVersion) // log.Printf("[DEBUG] Checking app %s with version %s", app.AppName, app.AppVersion) @@ -308,115 +451,140 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] createdPod, err := clientset.CoreV1().Pods(namespace).Create(context.Background(), pod, metav1.CreateOptions{}) if err != nil { - fmt.Fprintf(os.Stderr, "Error creating pod: %v\n", err) + fmt.Fprintf(os.Stderr, "Error creating pod: %v", err) // os.Exit(1) } - fmt.Printf("[DEBUG] Created pod %q in namespace %q\n", createdPod.Name, createdPod.Namespace) - } else { - // form basic hostConfig - ctx := context.Background() - - if action.AppName == "shuffle-subflow" { - // Automatic replacement of URL - for paramIndex, param := range action.Parameters { - if param.Name != "backend_url" { - continue - } - - if strings.Contains(param.Value, "shuffle-backend") { - // Automatic replacement as this is default - action.Parameters[paramIndex].Value = os.Getenv("BASE_URL") - log.Printf("[DEBUG][%s] Replaced backend_url with %s", workflowExecution.ExecutionId, os.Getenv("BASE_URL")) - } - } - } - - // Max 10% CPU every second - //CPUShares: 128, - //CPUQuota: 10000, - //CPUPeriod: 100000, - hostConfig := &container.HostConfig{ - LogConfig: container.LogConfig{ - Type: "json-file", - Config: map[string]string{ - "max-size": "10m", - }, - }, - Resources: container.Resources{}, - } - - hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:worker-%s", workflowExecution.ExecutionId)) - - // Removing because log extraction should happen first - if cleanupEnv == "true" { - hostConfig.AutoRemove = true - } - - // FIXME: Add proper foldermounts here - //log.Printf("\n\nPRE FOLDERMOUNT\n\n") - //volumeBinds := []string{"/tmp/shuffle-mount:/rules"} - //volumeBinds := []string{"/tmp/shuffle-mount:/rules"} - volumeBinds := []string{} - if len(volumeBinds) > 0 { - log.Printf("[DEBUG] Setting up binds for container!") - hostConfig.Binds = volumeBinds - hostConfig.Mounts = []mount.Mount{} - for _, bind := range volumeBinds { - if !strings.Contains(bind, ":") || strings.Contains(bind, "..") || strings.HasPrefix(bind, "~") { - log.Printf("[WARNING] Bind %s is invalid.", bind) - continue - } - - log.Printf("[DEBUG] Appending bind %s", bind) - bindSplit := strings.Split(bind, ":") - sourceFolder := bindSplit[0] - destinationFolder := bindSplit[0] - hostConfig.Mounts = append(hostConfig.Mounts, mount.Mount{ - Type: mount.TypeBind, - Source: sourceFolder, - Target: destinationFolder, - }) - } - } else { - //log.Printf("[WARNING] Not mounting folders") - } - - config := &container.Config{ - Image: image, - Env: env, - } - - // Checking as late as possible, just in case. - newExecId := fmt.Sprintf("%s_%s", workflowExecution.ExecutionId, action.ID) - _, err := shuffle.GetCache(ctx, newExecId) - if err == nil { - log.Printf("\n\n[DEBUG] Result for %s already found - returning\n\n", newExecId) - return nil - } - - cacheData := []byte("1") - err = shuffle.SetCache(ctx, newExecId, cacheData, 30) - if err != nil { - log.Printf("[WARNING] Failed setting cache for action %s: %s", newExecId, err) - } else { - log.Printf("[DEBUG] Adding %s to cache. Name: %s", newExecId, action.Name) - } - - if action.ExecutionDelay > 0 { - log.Printf("[DEBUG] Running app %s in docker with delay of %d", action.Name, action.ExecutionDelay) - waitTime := time.Duration(action.ExecutionDelay) * time.Second - - time.AfterFunc(waitTime, func() { - DeployContainer(ctx, cli, config, hostConfig, identifier, workflowExecution, newExecId) - }) - } else { - log.Printf("[DEBUG] Running app %s in docker NORMALLY as there is no delay set with identifier %s", action.Name, identifier) - returnvalue := DeployContainer(ctx, cli, config, hostConfig, identifier, workflowExecution, newExecId) - log.Printf("[DEBUG] Normal deploy ret: %s", returnvalue) - return returnvalue - } + log.Printf("[DEBUG] Created pod %q in namespace %q", createdPod.Name, createdPod.Namespace) return nil } + + // form basic hostConfig + ctx := context.Background() + + // Check action if subflow + // Check if url is default (shuffle-backend) + // If it doesn't exist, add it + if action.AppName == "shuffle-subflow" { + // Automatic replacement of URL + for paramIndex, param := range action.Parameters { + if param.Name != "backend_url" { + continue + } + + if strings.Contains(param.Value, "shuffle-backend") { + // Automatic replacement as this is default + if len(os.Getenv("BASE_URL")) > 0 { + action.Parameters[paramIndex].Value = os.Getenv("BASE_URL") + log.Printf("[DEBUG][%s] Replaced backend_url with base_url %s", workflowExecution.ExecutionId, os.Getenv("BASE_URL")) + } + + if len(os.Getenv("SHUFFLE_CLOUDRUN_URL")) > 0 { + action.Parameters[paramIndex].Value = os.Getenv("SHUFFLE_CLOUDRUN_URL") + log.Printf("[DEBUG][%s] Replaced backend_url with cloudrun %s", workflowExecution.ExecutionId, os.Getenv("SHUFFLE_CLOUDRUN_URL")) + } + } + } + } + + + // Max 10% CPU every second + //CPUShares: 128, + //CPUQuota: 10000, + //CPUPeriod: 100000, + hostConfig := &container.HostConfig{ + LogConfig: container.LogConfig{ + Type: "json-file", + Config: map[string]string{ + "max-size": "10m", + }, + }, + Resources: container.Resources{}, + } + + if os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "swarm" { + hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:worker-%s", workflowExecution.ExecutionId)) + //log.Printf("Environments: %#v", env) + } + + // Removing because log extraction should happen first + if cleanupEnv == "true" { + hostConfig.AutoRemove = true + } + + // Get environment for certificates + volumeBinds := []string{} + volumeBindString:= os.Getenv("SHUFFLE_VOLUME_BINDS") + if len(volumeBindString) > 0 { + volumeBindSplit := strings.Split(volumeBindString, ",") + for _, volumeBind := range volumeBindSplit { + if strings.Contains(volumeBind, ":") { + volumeBinds = append(volumeBinds, volumeBind) + } else { + log.Printf("[ERROR] Volume bind '%s' is invalid.", volumeBind) + } + } + } + + // Add more volume binds if possible + if len(volumeBinds) > 0 { + log.Printf("[DEBUG] Setting up binds for container. Got %d volume binds.", len(volumeBinds)) + + hostConfig.Binds = volumeBinds + hostConfig.Mounts = []mount.Mount{} + for _, bind := range volumeBinds { + if !strings.Contains(bind, ":") || strings.Contains(bind, "..") || strings.HasPrefix(bind, "~") { + log.Printf("[ERROR] Volume bind '%s' is invalid. Use absolute paths.", bind) + continue + } + + log.Printf("[DEBUG] Appending bind %s to app container", bind) + bindSplit := strings.Split(bind, ":") + sourceFolder := bindSplit[0] + destinationFolder := bindSplit[1] + hostConfig.Mounts = append(hostConfig.Mounts, mount.Mount{ + Type: mount.TypeBind, + Source: sourceFolder, + Target: destinationFolder, + }) + } + } + + config := &container.Config{ + Image: image, + Env: env, + } + + + // Checking as late as possible, just in case. + newExecId := fmt.Sprintf("%s_%s", workflowExecution.ExecutionId, action.ID) + _, err := shuffle.GetCache(ctx, newExecId) + if err == nil { + log.Printf("[DEBUG][%s] Result for action %s already found - returning", newExecId, action.ID) + return nil + } + + cacheData := []byte("1") + err = shuffle.SetCache(ctx, newExecId, cacheData, 30) + if err != nil { + //log.Printf("[WARNING][%s] Failed setting cache for action: %s", newExecId, err) + } else { + //log.Printf("[DEBUG][%s] Adding to cache. Name: %s", workflowExecution.ExecutionId, action.Name) + } + + if action.ExecutionDelay > 0 { + log.Printf("[DEBUG][%s] Running app '%s' with label '%s' in docker with delay of %d", workflowExecution.ExecutionId, action.AppName, action.Label, action.ExecutionDelay) + waitTime := time.Duration(action.ExecutionDelay) * time.Second + + time.AfterFunc(waitTime, func() { + DeployContainer(ctx, cli, config, hostConfig, identifier, workflowExecution, newExecId) + }) + } else { + log.Printf("[DEBUG][%s] Running app %s in docker NORMALLY as there is no delay set with identifier %s", workflowExecution.ExecutionId, action.Name, identifier) + returnvalue := DeployContainer(ctx, cli, config, hostConfig, identifier, workflowExecution, newExecId) + //log.Printf("[DEBUG][%s] Normal deploy ret: %s", workflowExecution.ExecutionId, returnvalue) + return returnvalue + } + return nil } @@ -429,7 +597,7 @@ func cleanupExecution(clientset *kubernetes.Clientset, workflowExecution shuffle LabelSelector: labelSelector, }) if err != nil { - return fmt.Errorf("[ERROR]failed to list apps with label selector %s: %v", labelSelector, err) + return fmt.Errorf("[ERROR] Failed to list apps with label selector %s: %#vv", labelSelector, err) } for _, pod := range podList.Items { @@ -437,14 +605,14 @@ func cleanupExecution(clientset *kubernetes.Clientset, workflowExecution shuffle if err != nil { return fmt.Errorf("failed to delete app %s: %v", pod.Name, err) } - fmt.Printf("App %s in namespace %s deleted.\n", pod.Name, namespace) + log.Printf("App %s in namespace %s deleted.", pod.Name, namespace) } podErr := clientset.CoreV1().Pods(namespace).Delete(context.TODO(), workerName, metav1.DeleteOptions{}) if podErr != nil { return fmt.Errorf("[ERROR] failed to delete the worker %s in namespace %s: %v", workerName, namespace, podErr) } - fmt.Printf("[DEBUG] %s in namespace %s deleted.\n", workerName, namespace) + log.Printf("[DEBUG] %s in namespace %s deleted.", workerName, namespace) return nil } @@ -458,6 +626,8 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont identifier, ) + //log.Printf("[DEBUG] config set: %#v", config) + if err != nil { //log.Printf("[ERROR] Failed creating container: %s", err) if !strings.Contains(err.Error(), "Conflict. The container name") { @@ -465,7 +635,7 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont cacheErr := shuffle.DeleteCache(ctx, newExecId) if cacheErr != nil { - log.Printf("[ERROR] FAILED Deleting cache for %s: %s", newExecId, cacheErr) + log.Printf("[ERROR] FAILURE Deleting cache for %s: %s", newExecId, cacheErr) } return err @@ -489,7 +659,7 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont cacheErr := shuffle.DeleteCache(ctx, newExecId) if cacheErr != nil { - log.Printf("[ERROR] FAILED Deleting cache for %s: %s", newExecId, cacheErr) + log.Printf("[ERROR] FAILURE Deleting cache for %s: %s", newExecId, cacheErr) } return err @@ -528,7 +698,7 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont cacheErr := shuffle.DeleteCache(ctx, newExecId) if cacheErr != nil { - log.Printf("[ERROR] FAILED Deleting cache for %s: %s", newExecId, cacheErr) + log.Printf("[ERROR] FAILURE Deleting cache for %s: %s", newExecId, cacheErr) } return err @@ -543,7 +713,7 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont cacheErr := shuffle.DeleteCache(ctx, newExecId) if cacheErr != nil { - log.Printf("[ERROR] FAILED Deleting cache for %s: %s", newExecId, cacheErr) + log.Printf("[ERROR] FAILURE Deleting cache for %s: %s", newExecId, cacheErr) } //shutdown(workflowExecution, workflowExecution.Workflow.ID, true) @@ -551,16 +721,16 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont } } - log.Printf("[DEBUG] Container %s was created for %s", cont.ID, identifier) + log.Printf("[DEBUG][%s] Container %s was created for %s", workflowExecution.ExecutionId, cont.ID, identifier) // Waiting to see if it exits.. Stupid, but stable(r) if workflowExecution.ExecutionSource != "default" { - log.Printf("[INFO] Handling NON-default execution source %s - NOT waiting or validating!", workflowExecution.ExecutionSource) + log.Printf("[INFO][%s] Handling NON-default execution source %s - NOT waiting or validating!", workflowExecution.ExecutionId, workflowExecution.ExecutionSource) } else if workflowExecution.ExecutionSource == "default" { - log.Printf("[INFO] Handling DEFAULT execution source %s - SKIPPING wait anyway due to exited issues!", workflowExecution.ExecutionSource) + log.Printf("[INFO][%s] Handling DEFAULT execution source %s - SKIPPING wait anyway due to exited issues!", workflowExecution.ExecutionId, workflowExecution.ExecutionSource) } - log.Printf("[DEBUG] Deployed container ID %s", cont.ID) + //log.Printf("[DEBUG] Deployed container ID %s", cont.ID) //containerIds = append(containerIds, cont.ID) return nil @@ -620,11 +790,100 @@ func removeIndex(s []string, i int) []string { return s[:len(s)-1] } +func getWorkerURLs() ([]string, error) { + workerUrls := []string{} + + // Create a new Docker client + cli, err := dockerclient.NewEnvClient() + if err != nil { + log.Println("[ERROR] Failed to create Docker client:", err) + return workerUrls, err + } + + // Specify the name of the service for which you want to list tasks + serviceName := "shuffle-workers" + + // Get the list of tasks for the service + tasks, err := cli.TaskList(context.Background(), types.TaskListOptions{ + Filters: filters.NewArgs(filters.Arg("service", serviceName)), + }) + + if err != nil { + log.Println("[ERROR] Failed to list tasks for service:", err) + return workerUrls, err + } + + // Print task information + for _, task := range tasks { + url := fmt.Sprintf("http://%s.%d.%s:33333", serviceName, task.Slot, task.ID) + workerUrls = append(workerUrls, url) + } + + return workerUrls, nil +} + +func askOtherWorkersToDownloadImage(image string) { + if os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "swarm" { + return + } + + urls, err := getWorkerURLs() + if err != nil { + log.Printf("[ERROR] Error in listing worker urls: %s", err) + return + } + + for _, url := range urls { + log.Printf("[DEBUG] Trying to speak to: %s", url) + imagesRequest := ImageRequest{ + Image: image, + } + + url = fmt.Sprintf("%s/api/v1/download", url) + + imageJSON, err := json.Marshal(imagesRequest) + + log.Printf("[INFO] Making a request to %s to download images", url) + req, err := http.NewRequest( + "POST", + url, + bytes.NewBuffer(imageJSON), + ) + + if err != nil { + log.Printf("[ERROR] Error in making request to %s : %s", url, err) + continue + } + + httpClient := &http.Client{} + resp, err := httpClient.Do(req) + if err != nil { + log.Printf("[ERROR] Error in making request to %s : %s", url, err) + continue + } + + defer resp.Body.Close() + respBody, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Printf("[ERROR] Error in reading response body : %s", err) + continue + } + + log.Printf("[INFO] Response body when tried sending images for nodes to download: %s", respBody) + } +} + func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { ctx := context.Background() - //log.Printf("[DEBUG][%s] Pre DecideExecution", workflowExecution.ExecutionId) workflowExecution, relevantActions := shuffle.DecideExecution(ctx, workflowExecution, environment) + if workflowExecution.Status == "FINISHED" || workflowExecution.Status == "FAILURE" || workflowExecution.Status == "ABORTED" { + log.Printf("[DEBUG][%s] Shutting down because status is %s", workflowExecution.ExecutionId, workflowExecution.Status) + shutdown(workflowExecution, "", "Workflow run is already finished", true) + return + } + + startAction, extra, children, parents, visited, executed, nextActions, environments := shuffle.GetExecutionVariables(ctx, workflowExecution.ExecutionId) dockercli, err := dockerclient.NewEnvClient() @@ -633,7 +892,6 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { return } - // log.Printf("\n\n[DEBUG] Got %d relevant action(s) to run!\n\n", len(relevantActions)) for _, action := range relevantActions { appname := action.AppName appversion := action.AppVersion @@ -645,6 +903,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { if strings.Contains(image, " ") { image = strings.ReplaceAll(image, " ", "-") } + askOtherWorkersToDownloadImage(image) // Added UUID to identifier just in case //identifier := fmt.Sprintf("%s_%s_%s_%s_%s", appname, appversion, action.ID, workflowExecution.ExecutionId, uuid.NewV4()) @@ -688,7 +947,9 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { } // marshal action and put it in there rofl - log.Printf("[INFO][%s] Time to execute %s (%s) with app %s:%s, function %s, env %s with %d parameters.", workflowExecution.ExecutionId, action.ID, action.Label, action.AppName, action.AppVersion, action.Name, action.Environment, len(action.Parameters)) + //log.Printf("[INFO][%s] Time to execute %s (%s) with app %s:%s, function %s, env %s with %d parameters.", workflowExecution.ExecutionId, action.ID, action.Label, action.AppName, action.AppVersion, action.Name, action.Environment, len(action.Parameters)) + + log.Printf("[DEBUG][%s] Action: Send, Label: '%s', Action: '%s', Run status: %s, Extra=", workflowExecution.ExecutionId, action.Label, action.AppName, workflowExecution.Status) actionData, err := json.Marshal(action) if err != nil { @@ -697,7 +958,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { } if action.AppID == "0ca8887e-b4af-4e3e-887c-87e9d3bc3d3e" { - log.Printf("[DEBUG] Should run filter: %#v\n\n", action) + log.Printf("[DEBUG] Should run filter: %#v", action) runFilter(workflowExecution, action) continue } @@ -718,7 +979,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { fmt.Sprintf("CALLBACK_URL=%s", baseUrl), fmt.Sprintf("BASE_URL=%s", appCallbackUrl), fmt.Sprintf("TZ=%s", timezone), - fmt.Sprintf("SHUFFLE_LOGS_DISABLED=%s", os.Getenv("SHUFFLE_LOGS_DISABLED")), + fmt.Sprintf("SHUFFLE_LOGS_DISABLED=%s", logsDisabled), } if len(actionData) >= 100000 { @@ -762,6 +1023,21 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { env = append(env, fmt.Sprintf("NO_PROXY=%s", os.Getenv("NO_PROXY"))) } + overrideHttpProxy := os.Getenv("SHUFFLE_INTERNAL_HTTP_PROXY") + overrideHttpsProxy := os.Getenv("SHUFFLE_INTERNAL_HTTPS_PROXY") + if overrideHttpProxy != "" { + env = append(env, fmt.Sprintf("SHUFFLE_INTERNAL_HTTP_PROXY=%s", overrideHttpProxy)) + } + + if overrideHttpsProxy != "" { + env = append(env, fmt.Sprintf("SHUFFLE_INTERNAL_HTTPS_PROXY=%s", overrideHttpsProxy)) + } + + if len(os.Getenv("SHUFFLE_APP_SDK_TIMEOUT")) > 0 { + env = append(env, fmt.Sprintf("SHUFFLE_APP_SDK_TIMEOUT=%s", os.Getenv("SHUFFLE_APP_SDK_TIMEOUT"))) + } + + // Fixes issue: // standard_go init_linux.go:185: exec user process caused "argument list too long" // https://devblogs.microsoft.com/oldnewthing/20100203-00/?p=15083 @@ -785,10 +1061,12 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { // 3. Add remote repo location images := []string{ image, - fmt.Sprintf("%s:%s_%s", baseimagename, parsedAppname, action.AppVersion), fmt.Sprintf("%s/%s:%s_%s", registryName, baseimagename, parsedAppname, action.AppVersion), + fmt.Sprintf("%s:%s_%s", baseimagename, parsedAppname, action.AppVersion), } + + // If cleanup is set, it should run for efficiency pullOptions := types.ImagePullOptions{} if cleanupEnv == "true" { @@ -859,7 +1137,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { return } else { if strings.Contains(buildBuf.String(), "errorDetail") { - log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), image) + log.Printf("[ERROR] Docker build:%sERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), image) log.Printf("[DEBUG] Shutting down (6)") shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) return @@ -969,7 +1247,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { return } else { if strings.Contains(buildBuf.String(), "errorDetail") { - log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), image) + log.Printf("[ERROR] Docker build:%sERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), image) log.Printf("[DEBUG] Shutting down (14)") shutdown(workflowExecution, action.ID, fmt.Sprintf("Error deploying container: %s", buildBuf.String()), true) return @@ -1018,7 +1296,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { // FIXME - clean up stopped (remove) containers with this execution id err = shuffle.UpdateExecutionVariables(ctx, workflowExecution.ExecutionId, startAction, children, parents, visited, executed, nextActions, environments, extra) if err != nil { - log.Printf("\n\n[ERROR] Failed to update exec variables for execution %s: %s (2)\n\n", workflowExecution.ExecutionId, err) + log.Printf("[ERROR] Failed to update exec variables for execution %s: %s (2)", workflowExecution.ExecutionId, err) } if len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions)+extra { @@ -1036,13 +1314,22 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { if shutdownCheck { log.Printf("[INFO][%s] BREAKING BECAUSE RESULTS IS SAME LENGTH AS ACTIONS. SHOULD CHECK ALL RESULTS FOR WHETHER THEY'RE DONE", workflowExecution.ExecutionId) - validateFinished(workflowExecution) + validated := shuffle.ValidateFinished(ctx, -1, workflowExecution) + if validated { + shutdownData, err := json.Marshal(workflowExecution) + if err != nil { + log.Printf("[ERROR] Failed marshalling shutdowndata during set: %s", err) + } + + sendResult(workflowExecution, shutdownData) + } + log.Printf("[DEBUG][%s] Shutting down (17)", workflowExecution.ExecutionId) - if os.Getenv("IS_KUBERNETES") == "true" { + if isKubernetes == "true" { // log.Printf("workflow execution: %#v", workflowExecution) clientset, err := getKubernetesClient() if err != nil { - fmt.Println("[ERROR]Error getting kubernetes client:", err) + log.Println("[ERROR] Error getting kubernetes client (1):", err) os.Exit(1) } cleanupExecution(clientset, workflowExecution, "shuffle") @@ -1094,7 +1381,7 @@ func executionInit(workflowExecution shuffle.WorkflowExecution) error { } for _, trigger := range workflowExecution.Workflow.Triggers { - //log.Printf("Appname trigger (0): %s", trigger.AppName) + //log.Printf("Appname trigger (0): %s (%s)", trigger.AppName, trigger.ID) if trigger.AppName == "User Input" || trigger.AppName == "Shuffle Workflow" { if trigger.ID == branch.SourceID { sourceFound = true @@ -1107,22 +1394,16 @@ func executionInit(workflowExecution shuffle.WorkflowExecution) error { if sourceFound { parents[branch.DestinationID] = append(parents[branch.DestinationID], branch.SourceID) } else { - log.Printf("[DEBUG] ID %s was not found in actions! Skipping parent. (TRIGGER?)", branch.SourceID) + log.Printf("[DEBUG] Parent ID %s was not found in actions! Skipping parent. (TRIGGER?)", branch.SourceID) } if destinationFound { children[branch.SourceID] = append(children[branch.SourceID], branch.DestinationID) } else { - log.Printf("[DEBUG] ID %s was not found in actions! Skipping child. (TRIGGER?)", branch.SourceID) + log.Printf("[DEBUG] Child ID %s was not found in actions! Skipping child. (TRIGGER?)", branch.SourceID) } } - /* - log.Printf("\n\n\n[INFO] CHILDREN FOUND: %#v", children) - log.Printf("[INFO] PARENTS FOUND: %#v", parents) - log.Printf("[INFO] NEXT ACTIONS: %#v\n\n", nextActions) - */ - log.Printf("[INFO][%s] shuffle.Actions: %d + Special shuffle.Triggers: %d", workflowExecution.ExecutionId, len(workflowExecution.Workflow.Actions), extra) onpremApps := []string{} toExecuteOnprem := []string{} @@ -1190,16 +1471,185 @@ func executionInit(workflowExecution shuffle.WorkflowExecution) error { environments = append(environments, action.Environment) } } - //var visited []string - //var executed []string + err := shuffle.UpdateExecutionVariables(ctx, workflowExecution.ExecutionId, startAction, children, parents, visited, executed, nextActions, environments, extra) if err != nil { - log.Printf("\n\n[ERROR] Failed to update exec variables for execution %s: %s\n\n", workflowExecution.ExecutionId, err) + log.Printf("[ERROR] Failed to update exec variables for execution %s: %s", workflowExecution.ExecutionId, err) } return nil } +func handleSubflowPoller(ctx context.Context, workflowExecution shuffle.WorkflowExecution, streamResultUrl, subflowId string) error { + extra := 0 + for _, trigger := range workflowExecution.Workflow.Triggers { + if trigger.AppName == "User Input" || trigger.AppName == "Shuffle Workflow" { + extra += 1 + } + } + + req, err := http.NewRequest( + "POST", + streamResultUrl, + bytes.NewBuffer([]byte(data)), + ) + + newresp, err := topClient.Do(req) + if err != nil { + log.Printf("[ERROR] Failed making request (1): %s", err) + time.Sleep(time.Duration(sleepTime) * time.Second) + return err + } + + defer newresp.Body.Close() + body, err := ioutil.ReadAll(newresp.Body) + if err != nil { + log.Printf("[ERROR] Failed reading body (1): %s", err) + time.Sleep(time.Duration(sleepTime) * time.Second) + return err + } + + if newresp.StatusCode != 200 { + log.Printf("[ERROR] Bad statuscode: %d, %s", newresp.StatusCode, string(body)) + + if strings.Contains(string(body), "Workflowexecution is already finished") { + log.Printf("[DEBUG] Shutting down (19)") + shutdown(workflowExecution, "", "", true) + } + + time.Sleep(time.Duration(sleepTime) * time.Second) + return errors.New(fmt.Sprintf("Bad statuscode: %d", newresp.StatusCode)) + } + + err = json.Unmarshal(body, &workflowExecution) + if err != nil { + log.Printf("[ERROR] Failed workflowExecution unmarshal: %s", err) + time.Sleep(time.Duration(sleepTime) * time.Second) + return err + } + + if workflowExecution.Status == "FINISHED" || workflowExecution.Status == "SUCCESS" { + log.Printf("[INFO][%s] Workflow execution is finished. Exiting worker.", workflowExecution.ExecutionId) + log.Printf("[DEBUG] Shutting down (20)") + if isKubernetes == "true" { + // log.Printf("workflow execution: %#v", workflowExecution) + clientset, err := getKubernetesClient() + if err != nil { + log.Println("[ERROR] Error getting kubernetes client (2):", err) + os.Exit(1) + } + + cleanupExecution(clientset, workflowExecution, "shuffle") + } else { + shutdown(workflowExecution, "", "", true) + } + } + + for _, result := range workflowExecution.Results { + if result.Action.ID != subflowId { + continue + } + + log.Printf("[DEBUG][%s] Found subflow to handle: %s (%s)", workflowExecution.ExecutionId, result.Action.Label, result.Status) + if result.Status == "SUCCESS" || result.Status == "FINISHED" || result.Status == "FAILURE" || result.Status == "ABORTED" { + // Check for results + + setWorkflowExecution(ctx, workflowExecution, false) + return nil + } + } + + log.Printf("[INFO][%s] Status: %s, Results: %d, actions: %d", workflowExecution.ExecutionId, workflowExecution.Status, len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)+extra) + return errors.New("Subflow status not found yet") +} + +func handleDefaultExecutionWrapper(ctx context.Context, workflowExecution shuffle.WorkflowExecution, streamResultUrl string, extra int) error { + if extra == -1 { + extra = 0 + for _, trigger := range workflowExecution.Workflow.Triggers { + if trigger.AppName == "User Input" || trigger.AppName == "Shuffle Workflow" { + extra += 1 + } + } + } + + req, err := http.NewRequest( + "POST", + streamResultUrl, + bytes.NewBuffer([]byte(data)), + ) + + newresp, err := topClient.Do(req) + if err != nil { + log.Printf("[ERROR] Failed making request (1): %s", err) + time.Sleep(time.Duration(sleepTime) * time.Second) + return err + } + + defer newresp.Body.Close() + body, err := ioutil.ReadAll(newresp.Body) + if err != nil { + log.Printf("[ERROR] Failed reading body (1): %s", err) + time.Sleep(time.Duration(sleepTime) * time.Second) + return err + } + + if newresp.StatusCode != 200 { + log.Printf("[ERROR] Bad statuscode: %d, %s", newresp.StatusCode, string(body)) + + if strings.Contains(string(body), "Workflowexecution is already finished") { + log.Printf("[DEBUG] Shutting down (19)") + shutdown(workflowExecution, "", "", true) + } + + time.Sleep(time.Duration(sleepTime) * time.Second) + return errors.New(fmt.Sprintf("Bad statuscode: %d", newresp.StatusCode)) + } + + err = json.Unmarshal(body, &workflowExecution) + if err != nil { + log.Printf("[ERROR] Failed workflowExecution unmarshal: %s", err) + time.Sleep(time.Duration(sleepTime) * time.Second) + return err + } + + if workflowExecution.Status == "FINISHED" || workflowExecution.Status == "SUCCESS" { + log.Printf("[INFO][%s] Workflow execution is finished. Exiting worker.", workflowExecution.ExecutionId) + log.Printf("[DEBUG] Shutting down (20)") + if isKubernetes == "true" { + // log.Printf("workflow execution: %#v", workflowExecution) + clientset, err := getKubernetesClient() + if err != nil { + log.Println("[ERROR] Error getting kubernetes client (2):", err) + os.Exit(1) + } + cleanupExecution(clientset, workflowExecution, "shuffle") + } else { + shutdown(workflowExecution, "", "", true) + } + } + + log.Printf("[INFO][%s] Status: %s, Results: %d, actions: %d", workflowExecution.ExecutionId, workflowExecution.Status, len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)+extra) + if workflowExecution.Status != "EXECUTING" { + log.Printf("[WARNING][%s] Exiting as worker execution has status %s!", workflowExecution.ExecutionId, workflowExecution.Status) + log.Printf("[DEBUG] Shutting down (21)") + if isKubernetes == "true" { + // log.Printf("workflow execution: %#v", workflowExecution) + clientset, err := getKubernetesClient() + if err != nil { + log.Println("[ERROR] Error getting kubernetes client (3):", err) + os.Exit(1) + } + cleanupExecution(clientset, workflowExecution, "shuffle") + } else { + shutdown(workflowExecution, "", "", true) + } + } + + setWorkflowExecution(ctx, workflowExecution, false) + return nil +} + func handleDefaultExecution(client *http.Client, req *http.Request, workflowExecution shuffle.WorkflowExecution) error { // if no onprem runs (shouldn't happen, but extra check), exit // if there are some, load the images ASAP for the app @@ -1220,84 +1670,10 @@ func handleDefaultExecution(client *http.Client, req *http.Request, workflowExec streamResultUrl := fmt.Sprintf("%s/api/v1/streams/results", baseUrl) for { - //fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s/executions/%s/abort", baseUrl, workflowExecution.Workflow.ID, workflowExecution.ExecutionId) - //log.Printf("[INFO] URL: %s", fullUrl) - req, err := http.NewRequest( - "POST", - streamResultUrl, - bytes.NewBuffer([]byte(data)), - ) - - newresp, err := topClient.Do(req) + err = handleDefaultExecutionWrapper(ctx, workflowExecution, streamResultUrl, extra) if err != nil { - log.Printf("[ERROR] Failed making request (1): %s", err) - time.Sleep(time.Duration(sleepTime) * time.Second) - continue + log.Printf("[ERROR] Failed handling default execution: %s", err) } - - defer newresp.Body.Close() - body, err := ioutil.ReadAll(newresp.Body) - if err != nil { - log.Printf("[ERROR] Failed reading body (1): %s", err) - time.Sleep(time.Duration(sleepTime) * time.Second) - continue - } - - if newresp.StatusCode != 200 { - log.Printf("[ERROR] Bad statuscode: %d, %s", newresp.StatusCode, string(body)) - - if strings.Contains(string(body), "Workflowexecution is already finished") { - log.Printf("[DEBUG] Shutting down (19)") - shutdown(workflowExecution, "", "", true) - } - - time.Sleep(time.Duration(sleepTime) * time.Second) - continue - } - - err = json.Unmarshal(body, &workflowExecution) - if err != nil { - log.Printf("[ERROR] Failed workflowExecution unmarshal: %s", err) - time.Sleep(time.Duration(sleepTime) * time.Second) - continue - } - - if workflowExecution.Status == "FINISHED" || workflowExecution.Status == "SUCCESS" { - log.Printf("[INFO][%s] Workflow execution is finished. Exiting worker.", workflowExecution.ExecutionId) - log.Printf("[DEBUG] Shutting down (20)") - //handle workerssssssssss - if os.Getenv("IS_KUBERNETES") == "true" { - // log.Printf("workflow execution: %#v", workflowExecution) - clientset, err := getKubernetesClient() - if err != nil { - fmt.Println("[ERROR]Error getting kubernetes client:", err) - os.Exit(1) - } - cleanupExecution(clientset, workflowExecution, "shuffle") - } else { - shutdown(workflowExecution, "", "", true) - } - } - - log.Printf("[INFO][%s] Status: %s, Results: %d, actions: %d", workflowExecution.ExecutionId, workflowExecution.Status, len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)+extra) - if workflowExecution.Status != "EXECUTING" { - log.Printf("[WARNING][%s] Exiting as worker execution has status %s!", workflowExecution.ExecutionId, workflowExecution.Status) - log.Printf("[DEBUG] Shutting down (21)") - if os.Getenv("IS_KUBERNETES") == "true" { - // log.Printf("workflow execution: %#v", workflowExecution) - clientset, err := getKubernetesClient() - if err != nil { - fmt.Println("[ERROR]Error getting kubernetes client:", err) - os.Exit(1) - } - cleanupExecution(clientset, workflowExecution, "shuffle") - } else { - shutdown(workflowExecution, "", "", true) - } - } - - setWorkflowExecution(ctx, workflowExecution, false) - //handleExecutionResult(workflowExecution) } return nil @@ -1377,7 +1753,7 @@ func runSkipAction(client *http.Client, action shuffle.Action, workflowId, workf return err } - newresp, err := client.Do(req) + newresp, err := topClient.Do(req) if err != nil { log.Printf("[WARNING] Error running skip request (0): %s", err) return err @@ -1394,167 +1770,6 @@ func runSkipAction(client *http.Client, action shuffle.Action, workflowId, workf return nil } -// Sends request back to backend to handle the node -func runUserInput(client *http.Client, action shuffle.Action, workflowId string, workflowExecution shuffle.WorkflowExecution, authorization string, configuration string, dockercli *dockerclient.Client) error { - timeNow := time.Now().Unix() - result := shuffle.ActionResult{ - Action: action, - ExecutionId: workflowExecution.ExecutionId, - Authorization: authorization, - Result: configuration, - StartedAt: timeNow, - CompletedAt: 0, - Status: "WAITING", - } - - // Checking for userinput to deploy subflow for it - subflow := false - subflowId := "" - argument := "" - continueUrl := "testing continue" - cancelUrl := "testing cancel" - for _, item := range action.Parameters { - if item.Name == "subflow" { - subflow = true - subflowId = item.Value - } else if item.Name == "alertinfo" { - argument = item.Value - } - } - - if subflow { - log.Printf("[DEBUG] Should run action with subflow app with argument %#v", argument) - newAction := shuffle.Action{ - AppName: "shuffle-subflow", - Name: "run_subflow", - AppVersion: "1.0.0", - Label: "User Input Subflow Execution", - } - - identifier := fmt.Sprintf("%s_%s_%s_%s", newAction.AppName, newAction.AppVersion, action.ID, workflowExecution.ExecutionId) - if strings.Contains(identifier, " ") { - identifier = strings.ReplaceAll(identifier, " ", "-") - } - - inputValue := UserInputSubflow{ - Argument: argument, - ContinueUrl: continueUrl, - CancelUrl: cancelUrl, - } - - parsedArgument, err := json.Marshal(inputValue) - if err != nil { - log.Printf("[ERROR] Failed to parse arguments: %s", err) - parsedArgument = []byte(argument) - } - - newAction.Parameters = []shuffle.WorkflowAppActionParameter{ - shuffle.WorkflowAppActionParameter{ - Name: "user_apikey", - Value: workflowExecution.Authorization, - }, - shuffle.WorkflowAppActionParameter{ - Name: "workflow", - Value: subflowId, - }, - shuffle.WorkflowAppActionParameter{ - Name: "argument", - Value: string(parsedArgument), - }, - } - - newAction.Parameters = append(newAction.Parameters, shuffle.WorkflowAppActionParameter{ - Name: "source_workflow", - Value: workflowExecution.Workflow.ID, - }) - - newAction.Parameters = append(newAction.Parameters, shuffle.WorkflowAppActionParameter{ - Name: "source_execution", - Value: workflowExecution.ExecutionId, - }) - - newAction.Parameters = append(newAction.Parameters, shuffle.WorkflowAppActionParameter{ - Name: "source_node", - Value: action.ID, - }) - - newAction.Parameters = append(newAction.Parameters, shuffle.WorkflowAppActionParameter{ - Name: "source_auth", - Value: workflowExecution.Authorization, - }) - - newAction.Parameters = append(newAction.Parameters, shuffle.WorkflowAppActionParameter{ - Name: "startnode", - Value: "", - }) - - // If cleanup is set, it should run for efficiency - //appName := strings.Replace(identifier, fmt.Sprintf("_%s", action.ID), "", -1) - //appName = strings.Replace(appName, fmt.Sprintf("_%s", workflowExecution.ExecutionId), "", -1) - actionData, err := json.Marshal(newAction) - if err != nil { - return err - } - - env := []string{ - fmt.Sprintf("ACTION=%s", string(actionData)), - fmt.Sprintf("EXECUTIONID=%s", workflowExecution.ExecutionId), - fmt.Sprintf("AUTHORIZATION=%s", workflowExecution.Authorization), - fmt.Sprintf("CALLBACK_URL=%s", baseUrl), - fmt.Sprintf("BASE_URL=%s", appCallbackUrl), - fmt.Sprintf("TZ=%s", timezone), - fmt.Sprintf("SHUFFLE_LOGS_DISABLED=%s", os.Getenv("SHUFFLE_LOGS_DISABLED")), - } - - if strings.ToLower(os.Getenv("SHUFFLE_PASS_APP_PROXY")) == "true" { - //log.Printf("APPENDING PROXY TO THE APP!") - env = append(env, fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY"))) - env = append(env, fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY"))) - env = append(env, fmt.Sprintf("NO_PROXY=%s", os.Getenv("NO_PROXY"))) - } - - err = deployApp(dockercli, "frikky/shuffle:shuffle-subflow_1.0.0", identifier, env, workflowExecution, newAction) - if err != nil { - log.Printf("[ERROR] Failed to deploy subflow for user input trigger %s: %s", action.ID, err) - } - } else { - log.Printf("[DEBUG] Running user input WITHOUT subflow") - } - - resultData, err := json.Marshal(result) - if err != nil { - return err - } - - streamUrl := fmt.Sprintf("%s/api/v1/streams", baseUrl) - req, err := http.NewRequest( - "POST", - streamUrl, - bytes.NewBuffer([]byte(resultData)), - ) - - if err != nil { - log.Printf("[WARNING] Error building test request (2): %s", err) - return err - } - - newresp, err := client.Do(req) - if err != nil { - log.Printf("[WARNING] Error running test request (2): %s", err) - return err - } - - defer newresp.Body.Close() - body, err := ioutil.ReadAll(newresp.Body) - if err != nil { - log.Printf("Failed reading body when waiting: %s", err) - return err - } - - log.Printf("[INFO] User Input Body: %s", string(body)) - return nil -} - func runTestExecution(client *http.Client, workflowId, apikey string) (string, string) { executeUrl := fmt.Sprintf("%s/api/v1/workflows/%s/execute", baseUrl, workflowId) req, err := http.NewRequest( @@ -1569,7 +1784,7 @@ func runTestExecution(client *http.Client, workflowId, apikey string) (string, s } req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", apikey)) - newresp, err := client.Do(req) + newresp, err := topClient.Do(req) if err != nil { log.Printf("[WARNING] Error running test request (3): %s", err) return "", "" @@ -1593,12 +1808,53 @@ func runTestExecution(client *http.Client, workflowId, apikey string) (string, s return workflowExecution.Authorization, workflowExecution.ExecutionId } +func isRunningInCluster() bool { + _, existsHost := os.LookupEnv("KUBERNETES_SERVICE_HOST") + _, existsPort := os.LookupEnv("KUBERNETES_SERVICE_PORT") + return existsHost && existsPort +} + +func buildEnvVars(envMap map[string]string) []corev1.EnvVar { + var envVars []corev1.EnvVar + for key, value := range envMap { + envVars = append(envVars, corev1.EnvVar{Name: key, Value: value}) + } + return envVars +} + +func getKubernetesClient() (*kubernetes.Clientset, error) { + if isRunningInCluster() { + config, err := rest.InClusterConfig() + if err != nil { + return nil, err + } + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, err + } + return clientset, nil + } else { + home := homedir.HomeDir() + kubeconfigPath := filepath.Join(home, ".kube", "config") + config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) + if err != nil { + return nil, err + } + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, err + } + return clientset, nil + } +} + func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { if request.Body == nil { resp.WriteHeader(http.StatusBadRequest) return } + defer request.Body.Close() body, err := ioutil.ReadAll(request.Body) if err != nil { log.Printf("[WARNING] (3) Failed reading body for workflowqueue") @@ -1607,8 +1863,6 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { return } - defer request.Body.Close() - var actionResult shuffle.ActionResult err = json.Unmarshal(body, &actionResult) if err != nil { @@ -1619,7 +1873,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { } if len(actionResult.ExecutionId) == 0 { - log.Printf("[WARNING] No workflow execution id in action result. Data: %s", string(body)) + log.Printf("[ERROR] No workflow execution id in action result. Data: %s", string(body)) resp.WriteHeader(400) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No workflow execution id in action result"}`))) return @@ -1635,44 +1889,47 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { workflowExecution, err := shuffle.GetWorkflowExecution(ctx, actionResult.ExecutionId) if err != nil { log.Printf("[ERROR][%s] Failed getting execution (workflowqueue) %s: %s", actionResult.ExecutionId, actionResult.ExecutionId, err) - resp.WriteHeader(401) + resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting execution ID %s because it doesn't exist locally."}`, actionResult.ExecutionId))) return } if workflowExecution.Authorization != actionResult.Authorization { - log.Printf("[INFO] Bad authorization key when updating node (workflowQueue) %s. Want: %s, Have: %s", actionResult.ExecutionId, workflowExecution.Authorization, actionResult.Authorization) - resp.WriteHeader(401) + log.Printf("[ERROR][%s] Bad authorization key when updating node (workflowQueue). Want: %s, Have: %s", actionResult.ExecutionId, workflowExecution.Authorization, actionResult.Authorization) + resp.WriteHeader(403) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad authorization key"}`))) return } if workflowExecution.Status == "FINISHED" { - log.Printf("[DEBUG] Workflowexecution is already FINISHED. No further action can be taken") - resp.WriteHeader(401) + log.Printf("[DEBUG][%s] Workflowexecution is already FINISHED. No further action can be taken", workflowExecution.ExecutionId) + resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Workflowexecution is already finished because it has status %s. Lastnode: %s"}`, workflowExecution.Status, workflowExecution.LastNode))) return } if workflowExecution.Status == "ABORTED" || workflowExecution.Status == "FAILURE" { + log.Printf("[WARNING][%s] Workflowexecution already has status %s. No further action can be taken", workflowExecution.ExecutionId, workflowExecution.Status) + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Workflowexecution is aborted because of %s with result %s and status %s"}`, workflowExecution.LastNode, workflowExecution.Result, workflowExecution.Status))) + return + } - if workflowExecution.Workflow.Configuration.ExitOnError { - log.Printf("[WARNING] Workflowexecution already has status %s. No further action can be taken", workflowExecution.Status) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Workflowexecution is aborted because of %s with result %s and status %s"}`, workflowExecution.LastNode, workflowExecution.Result, workflowExecution.Status))) - return - } else { - log.Printf("Continuing even though it's aborted.") + retries := 0 + retry, retriesok := request.URL.Query()["retries"] + if retriesok && len(retry) > 0 { + val, err := strconv.Atoi(retry[0]) + if err == nil { + retries = val } } - log.Printf("[INFO][%s] Got result '%s' from '%s' with app '%s':'%s'", actionResult.ExecutionId, actionResult.Status, actionResult.Action.Label, actionResult.Action.AppName, actionResult.Action.AppVersion) + log.Printf("[DEBUG][%s] Action: Received, Label: '%s', Action: '%s', Status: %s, Run status: %s, Extra=Retry:%d", workflowExecution.ExecutionId, actionResult.Action.Label, actionResult.Action.AppName, actionResult.Status, workflowExecution.Status, retries) //results = append(results, actionResult) //log.Printf("[INFO][%s] Time to execute %s (%s) with app %s:%s, function %s, env %s with %d parameters.", workflowExecution.ExecutionId, action.ID, action.Label, action.AppName, action.AppVersion, action.Name, action.Environment, len(action.Parameters)) //log.Printf("[DEBUG][%s] In workflowQueue with transaction", workflowExecution.ExecutionId) runWorkflowExecutionTransaction(ctx, 0, workflowExecution.ExecutionId, actionResult, resp) - } // Will make sure transactions are always ran for an execution. This is recursive if it fails. Allowed to fail up to 5 times @@ -1690,12 +1947,38 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl setExecution := true workflowExecution, dbSave, err := shuffle.ParsedExecutionResult(ctx, *workflowExecution, actionResult, true, 0) - if err != nil { + if err == nil { + if workflowExecution.Status != "EXECUTING" && workflowExecution.Status != "WAITING" { + log.Printf("[WARNING][%s] Execution is not executing, but %s. Stopping Transaction update.", workflowExecution.ExecutionId, workflowExecution.Status) + if resp != nil { + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Execution is not executing, but %s"}`, workflowExecution.Status))) + } + + + log.Printf("[DEBUG][%s] Shutting down (35)", workflowExecution.ExecutionId) + + // Force sending result + shutdownData, err := json.Marshal(workflowExecution) + if err != nil { + log.Printf("[ERROR][%s] Failed marshalling execution (35): %s", workflowExecution.ExecutionId, err) + } + + sendResult(*workflowExecution, shutdownData) + shutdown(*workflowExecution, "", "", false) + return + } + } else { + if strings.Contains(strings.ToLower(fmt.Sprintf("%s", err)), "already been ran") || strings.Contains(strings.ToLower(fmt.Sprintf("%s", err)), "already finished") { + log.Printf("[ERROR][%s] Skipping rerun of action result as it's already been ran: %s", workflowExecution.ExecutionId) + return + } + log.Printf("[DEBUG] Rerunning transaction? %s", err) if strings.Contains(fmt.Sprintf("%s", err), "Rerun this transaction") { workflowExecution, err := shuffle.GetWorkflowExecution(ctx, workflowExecutionId) if err != nil { - log.Printf("[ERROR] Failed getting execution cache (2): %s", err) + log.Printf("[ERROR][%s] Failed getting execution cache (2): %s", workflowExecution.ExecutionId, err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting execution (2)"}`))) return @@ -1706,15 +1989,15 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl workflowExecution, dbSave, err = shuffle.ParsedExecutionResult(ctx, *workflowExecution, actionResult, false, 0) if err != nil { - log.Printf("[ERROR] Failed execution of parsedexecution (2): %s", err) + log.Printf("[ERROR][%s] Failed execution of parsedexecution (2): %s", workflowExecution.ExecutionId, err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting execution (2)"}`))) return } else { - log.Printf("[DEBUG] Successfully got ParsedExecution with %d results!", len(workflowExecution.Results)) + log.Printf("[DEBUG][%s] Successfully got ParsedExecution with %d results!", workflowExecution.ExecutionId, len(workflowExecution.Results)) } } else { - log.Printf("[ERROR] Failed execution of parsedexecution: %s", err) + log.Printf("[ERROR][%s] Failed execution of parsedexecution: %s", workflowExecution.ExecutionId, err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting execution"}`))) return @@ -1739,43 +2022,28 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl cacheData := []byte(cache.([]uint8)) err = json.Unmarshal(cacheData, &workflowExecution) if err != nil { - log.Printf("[ERROR] Failed unmarshalling workflowexecution: %s", err) + log.Printf("[ERROR][%s] Failed unmarshalling workflowexecution: %s", workflowExecution.ExecutionId, err) } if len(parsedValue.Results) > 0 && len(parsedValue.Results) != resultLength { setExecution = false if attempts > 5 { - //log.Printf("\n\nSkipping execution input - %d vs %d. Attempts: (%d)\n\n", len(parsedValue.Results), resultLength, attempts) } attempts += 1 + log.Printf("[DEBUG][%s] Rerunning transaction as results has changed. %d vs %d", workflowExecution.ExecutionId, len(parsedValue.Results), resultLength) + /* if len(workflowExecution.Results) <= len(workflowExecution.Workflow.Actions) { + log.Printf("[DEBUG][%s] Rerunning transaction as results has changed. %d vs %d", workflowExecution.ExecutionId, len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)) runWorkflowExecutionTransaction(ctx, attempts, workflowExecutionId, actionResult, resp) return } + */ } } - /* - if value, found := requestCache.Get(cacheKey); found { - parsedValue := value.(*shuffle.WorkflowExecution) - if len(parsedValue.Results) > 0 && len(parsedValue.Results) != resultLength { - setExecution = false - if attempts > 5 { - //log.Printf("\n\nSkipping execution input - %d vs %d. Attempts: (%d)\n\n", len(parsedValue.Results), resultLength, attempts) - } - - attempts += 1 - if len(workflowExecution.Results) <= len(workflowExecution.Workflow.Actions) { - runWorkflowExecutionTransaction(ctx, attempts, workflowExecutionId, actionResult, resp) - return - } - } - } - */ - if setExecution || workflowExecution.Status == "FINISHED" || workflowExecution.Status == "ABORTED" || workflowExecution.Status == "FAILURE" { - log.Printf("[DEBUG][%s] Running setexec with status %s and %d results", workflowExecution.ExecutionId, workflowExecution.Status, len(workflowExecution.Results)) + log.Printf("[DEBUG][%s] Running setexec with status %s and %d result(s)", workflowExecution.ExecutionId, workflowExecution.Status, len(workflowExecution.Results)) err = setWorkflowExecution(ctx, *workflowExecution, dbSave) if err != nil { resp.WriteHeader(401) @@ -1789,7 +2057,6 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // Just in case. Should MAYBE validate finishing another time as well. // This fixes issues with e.g. shuffle.Action -> shuffle.Trigger -> shuffle.Action. handleExecutionResult(*workflowExecution) - //validateFinished(workflowExecution) } //if newExecutions && len(nextActions) > 0 { @@ -1802,8 +2069,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } func sendSelfRequest(actionResult shuffle.ActionResult) { - log.Printf("[INFO][%s] Not sending backend info since source is default (not swarm)", actionResult.ExecutionId) - return + data, err := json.Marshal(actionResult) if err != nil { @@ -1837,7 +2103,7 @@ func sendSelfRequest(actionResult shuffle.ActionResult) { newresp, err := topClient.Do(req) if err != nil { - log.Printf("[ERROR][%s] Error running self request (2): %s", actionResult.ExecutionId, err) + log.Printf("[ERROR][%s] Error running finishing request (2): %s", actionResult.ExecutionId, err) return } @@ -1846,9 +2112,9 @@ func sendSelfRequest(actionResult shuffle.ActionResult) { body, err := ioutil.ReadAll(newresp.Body) //log.Printf("[INFO] BACKEND STATUS: %d", newresp.StatusCode) if err != nil { - log.Printf("[ERROR][%s] Failed reading self request body: %s", actionResult.ExecutionId, err) + log.Printf("[ERROR][%s] Failed reading body: %s", actionResult.ExecutionId, err) } else { - log.Printf("[DEBUG][%s] NEWRESP (from self - 1): %s", actionResult.ExecutionId, string(body)) + log.Printf("[DEBUG][%s] NEWRESP (from backend): %s", actionResult.ExecutionId, string(body)) } } } @@ -1857,8 +2123,27 @@ func sendResult(workflowExecution shuffle.WorkflowExecution, data []byte) { if workflowExecution.ExecutionSource == "default" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "swarm" { //log.Printf("[INFO][%s] Not sending backend info since source is default (not swarm)", workflowExecution.ExecutionId) //return + } else { } + // Basically to reduce backend strain + /* + if shuffle.ArrayContains(finishedExecutions, workflowExecution.ExecutionId) { + log.Printf("[INFO][%s] NOT sending backend info since it's already been sent before.", workflowExecution.ExecutionId) + return + } + */ + + // Take it down again + /* + if len(finishedExecutions) > 100 { + log.Printf("[DEBUG][%s] Removing old execution from finishedExecutions: %s", workflowExecution.ExecutionId, finishedExecutions[0]) + finishedExecutions = finishedExecutions[99:] + } + + finishedExecutions = append(finishedExecutions, workflowExecution.ExecutionId) + */ + streamUrl := fmt.Sprintf("%s/api/v1/streams", baseUrl) req, err := http.NewRequest( "POST", @@ -1907,10 +2192,9 @@ func validateFinished(workflowExecution shuffle.WorkflowExecution) bool { workflowExecution = shuffle.Fixexecution(ctx, workflowExecution) _, extra, _, _, _, _, _, environments := shuffle.GetExecutionVariables(ctx, workflowExecution.ExecutionId) - log.Printf("[INFO][%s] VALIDATION. Status: %s, shuffle.Actions: %d, Extra: %d, Results: %d. Parent: %#v\n", workflowExecution.ExecutionId, workflowExecution.Status, len(workflowExecution.Workflow.Actions), extra, len(workflowExecution.Results), workflowExecution.ExecutionParent) + log.Printf("[INFO][%s] VALIDATION. Status: %s, shuffle.Actions: %d, Extra: %d, Results: %d. Parent: %#v", workflowExecution.ExecutionId, workflowExecution.Status, len(workflowExecution.Workflow.Actions), extra, len(workflowExecution.Results), workflowExecution.ExecutionParent) - //if len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions)+extra { - if (len(environments) == 1 && requestsSent == 0 && len(workflowExecution.Results) >= 1 && os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "swarm") || (len(workflowExecution.Results) >= len(workflowExecution.Workflow.Actions)+extra && len(workflowExecution.Workflow.Actions) > 0) { + if workflowExecution.Status == "FINISHED" || workflowExecution.Status == "ABORTED" || (len(environments) == 1 && requestsSent == 0 && len(workflowExecution.Results) >= 1 && os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "swarm") || (len(workflowExecution.Results) >= len(workflowExecution.Workflow.Actions)+extra && len(workflowExecution.Workflow.Actions) > 0) { if workflowExecution.Status == "FINISHED" { for _, result := range workflowExecution.Results { @@ -1921,7 +2205,6 @@ func validateFinished(workflowExecution shuffle.WorkflowExecution) bool { } } - requestsSent += 1 log.Printf("[DEBUG][%s] Should send full result to %s", workflowExecution.ExecutionId, baseUrl) @@ -1933,9 +2216,11 @@ func validateFinished(workflowExecution shuffle.WorkflowExecution) bool { } cacheKey := fmt.Sprintf("workflowexecution_%s", workflowExecution.ExecutionId) - err = shuffle.SetCache(ctx, cacheKey, shutdownData, 30) - if err != nil { - log.Printf("[ERROR][%s] Failed adding to cache during validateFinished", workflowExecution) + if len(workflowExecution.Authorization) > 0 { + err = shuffle.SetCache(ctx, cacheKey, shutdownData, 31) + if err != nil { + log.Printf("[ERROR][%s] Failed adding to cache during ValidateFinished", workflowExecution) + } } shuffle.RunCacheCleanup(ctx, workflowExecution) @@ -1947,6 +2232,7 @@ func validateFinished(workflowExecution shuffle.WorkflowExecution) bool { } func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { + defer request.Body.Close() body, err := ioutil.ReadAll(request.Body) if err != nil { log.Printf("[WARNING] Failed reading body for stream result queue") @@ -1955,9 +2241,6 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { return } - defer request.Body.Close() - //log.Printf("[DEBUG] In get stream results with body length %d: %s", len(body), string(body)) - var actionResult shuffle.ActionResult err = json.Unmarshal(body, &actionResult) if err != nil { @@ -1985,7 +2268,7 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { // Authorization is done here if workflowExecution.Authorization != actionResult.Authorization { - log.Printf("Bad authorization key when getting stream results %s.", actionResult.ExecutionId) + log.Printf("[ERROR] Bad authorization key when getting stream results from cache %s.", actionResult.ExecutionId) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad authorization key or execution_id might not exist."}`))) return @@ -2003,52 +2286,10 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { } -func setWorkflowExecution(ctx context.Context, workflowExecution shuffle.WorkflowExecution, dbSave bool) error { - if len(workflowExecution.ExecutionId) == 0 { - log.Printf("[DEBUG] Workflowexecution executionId can't be empty.") - return errors.New("ExecutionId can't be empty.") - } - - //log.Printf("[DEBUG][%s] Setting with %d results (pre)", workflowExecution.ExecutionId, len(workflowExecution.Results)) - workflowExecution = shuffle.Fixexecution(ctx, workflowExecution) - //log.Printf("[DEBUG][%s] Setting with %d results (post)", workflowExecution.ExecutionId, len(workflowExecution.Results)) - - cacheKey := fmt.Sprintf("workflowexecution_%s", workflowExecution.ExecutionId) - - execData, err := json.Marshal(workflowExecution) - if err != nil { - log.Printf("[ERROR] Failed marshalling execution during set: %s", err) - return err - } - - err = shuffle.SetCache(ctx, cacheKey, execData, 30) - if err != nil { - log.Printf("[ERROR][%s] Failed adding to cache during setexecution", workflowExecution) - return err - } - //requestCache.Set(cacheKey, &workflowExecution, cache.DefaultExpiration) - - handleExecutionResult(workflowExecution) - validateFinished(workflowExecution) - - // FIXME: Should this shutdown OR send the result? - // The worker may not be running the backend hmm - if dbSave { - if workflowExecution.ExecutionSource == "default" { - log.Printf("[DEBUG][%s] Shutting down (25)", workflowExecution.ExecutionId) - shutdown(workflowExecution, "", "", true) - //return - } else { - log.Printf("[DEBUG] NOT shutting down with dbSave (%s)", workflowExecution.ExecutionSource) - } - } - - return nil -} - // GetLocalIP returns the non loopback local IP of the host func getLocalIP() string { + addrs, err := net.InterfaceAddrs() if err != nil { return "" @@ -2082,7 +2323,6 @@ func getAvailablePort() (net.Listener, error) { func webserverSetup(workflowExecution shuffle.WorkflowExecution) net.Listener { hostname = getLocalIP() - os.Setenv("WORKER_HOSTNAME", hostname) // FIXME: This MAY not work because of speed between first @@ -2094,9 +2334,11 @@ func webserverSetup(workflowExecution shuffle.WorkflowExecution) net.Listener { } log.Printf("[DEBUG] OLD HOSTNAME: %s", appCallbackUrl) + + port := listener.Addr().(*net.TCPAddr).Port - log.Printf("\n\n[DEBUG] Starting webserver (2) on port %d with hostname: %s\n\n", port, hostname) + log.Printf("[DEBUG] Starting webserver (2) on port %d with hostname: %s", port, hostname) appCallbackUrl = fmt.Sprintf("http://%s:%d", hostname, port) log.Printf("[INFO] NEW WORKER HOSTNAME: %s", appCallbackUrl) @@ -2130,7 +2372,7 @@ func downloadDockerImageBackend(client *http.Client, imageName string) error { //return } - newresp, err := client.Do(req) + newresp, err := topClient.Do(req) if err != nil { log.Printf("[ERROR] Failed download request for %s: %s", imageName, err) return err @@ -2202,25 +2444,412 @@ func downloadDockerImageBackend(client *http.Client, imageName string) error { return nil } +func findActiveSwarmNodes(dockercli *dockerclient.Client) (int64, error) { + ctx := context.Background() + nodes, err := dockercli.NodeList(ctx, types.NodeListOptions{}) + if err != nil { + return 0, err + } + + nodeCount := int64(0) + for _, node := range nodes { + //log.Printf("ID: %s - %#v", node.ID, node.Status.State) + if node.Status.State == "ready" { + nodeCount += 1 + } + } + + return nodeCount, nil + + /* + containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{ + All: true, + }) + */ +} + + +// Runs data discovery + +func sendAppRequest(ctx context.Context, incomingUrl, appName string, port int, action *shuffle.Action, workflowExecution *shuffle.WorkflowExecution) error { + parsedRequest := shuffle.OrborusExecutionRequest{ + Cleanup: cleanupEnv, + ExecutionId: workflowExecution.ExecutionId, + Authorization: workflowExecution.Authorization, + EnvironmentName: os.Getenv("ENVIRONMENT_NAME"), + Timezone: os.Getenv("TZ"), + HTTPProxy: os.Getenv("HTTP_PROXY"), + HTTPSProxy: os.Getenv("HTTPS_PROXY"), + ShufflePassProxyToApp: os.Getenv("SHUFFLE_PASS_APP_PROXY"), + Url: baseUrl, + BaseUrl: baseUrl, + Action: *action, + FullExecution: *workflowExecution, + } + // Sometimes makes it have the wrong data due to timing + + // Specific for subflow to ensure worker matches the backend correctly + + parsedBaseurl := incomingUrl + if strings.Count(baseUrl, ":") >= 2 { + baseUrlSplit := strings.Split(baseUrl, ":") + if len(baseUrlSplit) >= 3 { + parsedBaseurl = strings.Join(baseUrlSplit[0:2], ":") + //parsedRequest.BaseUrl = fmt.Sprintf("%s:33333", parsedBaseurl) + } + } + + if len(parsedRequest.Url) == 0 { + // Fixed callback url to the worker itself + if strings.Count(parsedBaseurl, ":") >= 2 { + parsedRequest.Url = parsedBaseurl + } else { + // Callback to worker + parsedRequest.Url = fmt.Sprintf("%s:%d", parsedBaseurl, baseport) + + //parsedRequest.Url + } + + //log.Printf("[DEBUG][%s] Should add a baseurl for the app to get back to: %s", workflowExecution.ExecutionId, parsedRequest.Url) + } + + // Swapping because this was confusing during dev + // No real reason, just variable names + tmp := parsedRequest.Url + parsedRequest.Url = parsedRequest.BaseUrl + parsedRequest.BaseUrl = tmp + + // Run with proper hostname, but set to shuffle-worker to avoid specific host target. + // This means running with VIP instead. + if len(hostname) > 0 { + parsedRequest.BaseUrl = fmt.Sprintf("http://%s:%d", hostname, baseport) + //parsedRequest.BaseUrl = fmt.Sprintf("http://shuffle-workers:%d", baseport) + //log.Printf("[DEBUG][%s] Changing hostname to local hostname in Docker network for WORKER URL: %s", workflowExecution.ExecutionId, parsedRequest.BaseUrl) + + if parsedRequest.Action.AppName == "shuffle-subflow" || parsedRequest.Action.AppName == "shuffle-subflow-v2" || parsedRequest.Action.AppName == "User Input" { + parsedRequest.BaseUrl = fmt.Sprintf("http://%s:%d", hostname, baseport) + //parsedRequest.Url = parsedRequest.BaseUrl + } + } + + // Making sure to get the LATEST execution data + // This is due to cache timing issues + exec, err := shuffle.GetWorkflowExecution(ctx, workflowExecution.ExecutionId) + if err == nil && len(exec.ExecutionId) > 0 { + parsedRequest.FullExecution = *exec + } + + data, err := json.Marshal(parsedRequest) + if err != nil { + log.Printf("[ERROR] Failed marshalling worker request: %s", err) + return err + } + + streamUrl := fmt.Sprintf("http://%s:%d/api/v1/run", appName, port) + log.Printf("[DEBUG][%s] Worker URL: %s, Backend URL: %s, Target App: %s", workflowExecution.ExecutionId, parsedRequest.BaseUrl, parsedRequest.Url, streamUrl) + req, err := http.NewRequest( + "POST", + streamUrl, + bytes.NewBuffer([]byte(data)), + ) + + // Checking as LATE as possible, ensuring we don't rerun what's already ran + //ctx = context.Background() + newExecId := fmt.Sprintf("%s_%s", workflowExecution.ExecutionId, action.ID) + _, err = shuffle.GetCache(ctx, newExecId) + if err == nil { + log.Printf("[DEBUG] Result for %s already found (PRE REQUEST) - returning", newExecId) + return nil + } + + cacheData := []byte("1") + err = shuffle.SetCache(ctx, newExecId, cacheData, 30) + if err != nil { + log.Printf("[WARNING] Failed setting cache for action %s: %s", newExecId, err) + } else { + log.Printf("[DEBUG][%s] Adding %s to cache (%#v)", workflowExecution.ExecutionId, newExecId, action.Name) + } + + // FIXME: Add 5 tries + + newresp, err := topClient.Do(req) + if err != nil { + // Another timeout issue here somewhere + // context deadline + if strings.Contains(fmt.Sprintf("%s", err), "context deadline exceeded") || strings.Contains(fmt.Sprintf("%s", err), "Client.Timeout exceeded") { + return nil + } + + if strings.Contains(fmt.Sprintf("%s", err), "timeout awaiting response") { + return nil + } + + newerr := fmt.Sprintf("%s", err) + if strings.Contains(newerr, "connection refused") || strings.Contains(newerr, "no such host") { + newerr = fmt.Sprintf("Failed connecting to app %s. Is the Docker image available?", appName) + } else { + // escape quotes and newlines + newerr = strings.ReplaceAll(strings.ReplaceAll(newerr, "\"", "\\\""), "\n", "\\n") + } + + log.Printf("[ERROR][%s] Error running app run request: %s", workflowExecution.ExecutionId, err) + actionResult := shuffle.ActionResult{ + Action: *action, + ExecutionId: workflowExecution.ExecutionId, + Authorization: workflowExecution.Authorization, + Result: fmt.Sprintf(`{"success": false, "reason": "Failed to connect to app %s in swarm. Restart Orborus if this is recurring, or contact support@shuffler.io.", "details": "%s"}`, streamUrl, newerr), + StartedAt: int64(time.Now().Unix()), + CompletedAt: int64(time.Now().Unix()), + Status: "FAILURE", + } + + // If this happens - send failure signal to stop the workflow? + sendSelfRequest(actionResult) + return err + } + + defer newresp.Body.Close() + body, err := ioutil.ReadAll(newresp.Body) + if err != nil { + log.Printf("[ERROR] Failed reading app request body body: %s", err) + return err + } else { + log.Printf("[DEBUG][%s] NEWRESP (from app): %s", workflowExecution.ExecutionId, string(body)) + } + + return nil +} + +// Function to auto-deploy certain apps if "run" is set +// Has some issues with loading when running multiple workers and such. +func baseDeploy() { + + cli, err := dockerclient.NewEnvClient() + if err != nil { + log.Printf("[ERROR] Unable to create docker client (3): %s", err) + return + } + + for key, value := range autoDeploy { + newNameSplit := strings.Split(key, ":") + + action := shuffle.Action{ + AppName: newNameSplit[0], + AppVersion: newNameSplit[1], + ID: "TBD", + } + + workflowExecution := shuffle.WorkflowExecution{ + ExecutionId: "TBD", + } + + appname := action.AppName + appversion := action.AppVersion + appname = strings.Replace(appname, ".", "-", -1) + appversion = strings.Replace(appversion, ".", "-", -1) + + env := []string{ + fmt.Sprintf("EXECUTIONID=%s", workflowExecution.ExecutionId), + fmt.Sprintf("AUTHORIZATION=%s", workflowExecution.Authorization), + fmt.Sprintf("CALLBACK_URL=%s", baseUrl), + fmt.Sprintf("BASE_URL=%s", appCallbackUrl), + fmt.Sprintf("TZ=%s", timezone), + fmt.Sprintf("SHUFFLE_LOGS_DISABLED=%s", logsDisabled), + } + + if strings.ToLower(os.Getenv("SHUFFLE_PASS_APP_PROXY")) == "true" { + //log.Printf("APPENDING PROXY TO THE APP!") + env = append(env, fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY"))) + env = append(env, fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY"))) + env = append(env, fmt.Sprintf("NO_PROXY=%s", os.Getenv("NO_PROXY"))) + } + + if len(os.Getenv("SHUFFLE_APP_SDK_TIMEOUT")) > 0 { + log.Printf("[DEBUG] Setting SHUFFLE_APP_SDK_TIMEOUT to %s", os.Getenv("SHUFFLE_APP_SDK_TIMEOUT")) + env = append(env, fmt.Sprintf("SHUFFLE_APP_SDK_TIMEOUT=%s", os.Getenv("SHUFFLE_APP_SDK_TIMEOUT"))) + } + + identifier := fmt.Sprintf("%s_%s_%s_%s", appname, appversion, action.ID, workflowExecution.ExecutionId) + if strings.Contains(identifier, " ") { + identifier = strings.ReplaceAll(identifier, " ", "-") + } + + //deployApp(cli, value, identifier, env, workflowExecution, action) + log.Printf("[DEBUG] Deploying app with identifier %s to ensure basic apps are available from the get-go", identifier) + err = deployApp(cli, value, identifier, env, workflowExecution, action) + _ = err + //err := deployApp(cli, value, identifier, env, workflowExecution, action) + //if err != nil { + // log.Printf("[DEBUG] Failed deploying app %s: %s", value, err) + //} + } + + appsInitialized = true +} + +func getStreamResultsWrapper(client *http.Client, req *http.Request, workflowExecution shuffle.WorkflowExecution, firstRequest bool, environments []string) ([]string, error) { + // Because of this, it always has updated data. + // Removed request requirement from app_sdk + newresp, err := topClient.Do(req) + if err != nil { + log.Printf("[ERROR] Failed request: %s", err) + time.Sleep(time.Duration(sleepTime) * time.Second) + return environments, err + } + + defer newresp.Body.Close() + body, err := ioutil.ReadAll(newresp.Body) + if err != nil { + log.Printf("[ERROR] Failed reading body: %s", err) + time.Sleep(time.Duration(sleepTime) * time.Second) + return environments, err + } + + if newresp.StatusCode != 200 { + log.Printf("[ERROR] %sStatusCode (1): %d", string(body), newresp.StatusCode) + time.Sleep(time.Duration(sleepTime) * time.Second) + return environments, errors.New(fmt.Sprintf("Bad status code: %d", newresp.StatusCode) ) + } + + err = json.Unmarshal(body, &workflowExecution) + if err != nil { + log.Printf("[ERROR] Failed workflowExecution unmarshal: %s", err) + time.Sleep(time.Duration(sleepTime) * time.Second) + return environments, err + } + + if firstRequest { + firstRequest = false + + ctx := context.Background() + cacheKey := fmt.Sprintf("workflowexecution_%s", workflowExecution.ExecutionId) + execData, err := json.Marshal(workflowExecution) + if err != nil { + log.Printf("[ERROR][%s] Failed marshalling execution during set (3): %s", workflowExecution.ExecutionId, err) + } else { + err = shuffle.SetCache(ctx, cacheKey, execData, 30) + if err != nil { + log.Printf("[ERROR][%s] Failed adding to cache during setexecution (3): %s", workflowExecution.ExecutionId, err) + } + } + + for _, action := range workflowExecution.Workflow.Actions { + found := false + for _, environment := range environments { + if action.Environment == environment { + found = true + break + } + } + + if !found { + environments = append(environments, action.Environment) + } + } + + // Checks if a subflow is child of the startnode, as sub-subflows aren't working properly yet + childNodes := shuffle.FindChildNodes(workflowExecution, workflowExecution.Start, []string{}, []string{}) + log.Printf("[DEBUG] Looking for subflow in %#v to check execution pattern as child of %s", childNodes, workflowExecution.Start) + subflowFound := false + for _, childNode := range childNodes { + for _, trigger := range workflowExecution.Workflow.Triggers { + if trigger.ID != childNode { + continue + } + + if trigger.AppName == "Shuffle Workflow" { + subflowFound = true + break + } + } + + if subflowFound { + break + } + } + + log.Printf("[DEBUG] Environments: %s. Source: %s. 1 env = webserver, 0 or >1 = default. Subflow exists: %#v", environments, workflowExecution.ExecutionSource, subflowFound) + if len(environments) == 1 && workflowExecution.ExecutionSource != "default" && !subflowFound { + log.Printf("[DEBUG] Running OPTIMIZED execution (not manual)") + listener := webserverSetup(workflowExecution) + err := executionInit(workflowExecution) + if err != nil { + log.Printf("[DEBUG] Workflow setup failed: %s", workflowExecution.ExecutionId, err) + log.Printf("[DEBUG] Shutting down (30)") + shutdown(workflowExecution, "", "", true) + } + + go func() { + time.Sleep(time.Duration(1)) + handleExecutionResult(workflowExecution) + }() + + runWebserver(listener) + //log.Printf("Before wait") + //wg := sync.WaitGroup{} + //wg.Add(1) + //wg.Wait() + } else { + log.Printf("[DEBUG] Running NON-OPTIMIZED execution for type %s with %d environment(s). This only happens when ran manually OR when running with subflows. Status: %s", workflowExecution.ExecutionSource, len(environments), workflowExecution.Status) + err := executionInit(workflowExecution) + if err != nil { + log.Printf("[DEBUG] Workflow setup failed: %s", workflowExecution.ExecutionId, err) + shutdown(workflowExecution, "", "", true) + } + + // Trying to make worker into microservice~ :) + } + } + + if workflowExecution.Status == "FINISHED" || workflowExecution.Status == "SUCCESS" { + log.Printf("[DEBUG] Workflow %s is finished. Exiting worker.", workflowExecution.ExecutionId) + log.Printf("[DEBUG] Shutting down (31)") + shutdown(workflowExecution, "", "", true) + } + + if workflowExecution.Status == "EXECUTING" || workflowExecution.Status == "RUNNING" { + //log.Printf("Status: %s", workflowExecution.Status) + err = handleDefaultExecution(client, req, workflowExecution) + if err != nil { + log.Printf("[DEBUG] Workflow %s is finished: %s", workflowExecution.ExecutionId, err) + log.Printf("[DEBUG] Shutting down (32)") + shutdown(workflowExecution, "", "", true) + } + } else { + log.Printf("[DEBUG] Workflow %s has status %s. Exiting worker (if WAITING, rerun will happen).", workflowExecution.ExecutionId, workflowExecution.Status) + log.Printf("[DEBUG] Shutting down (33)") + shutdown(workflowExecution, workflowExecution.Workflow.ID, "", true) + } + + time.Sleep(time.Duration(sleepTime) * time.Second) + return environments, nil +} + // Initial loop etc func main() { // Elasticsearch necessary to ensure we'ren ot running with Datastore configurations for minimal/maximal data sizes - _, err := shuffle.RunInit(datastore.Client{}, storage.Client{}, "", "worker", true, "elasticsearch") + // Recursive import kind of :) + _, err := shuffle.RunInit(*shuffle.GetDatastore(), *shuffle.GetStorage(), "", "worker", true, "elasticsearch") if err != nil { - log.Printf("[ERROR] Failed to run worker init: %s", err) + if !strings.Contains(fmt.Sprintf("%s", err), "no such host") { + log.Printf("[ERROR] Failed to run worker init: %s", err) + } } else { log.Printf("[DEBUG] Ran init for worker to set up cache system. Docker version: %s", dockerApiVersion) } log.Printf("[INFO] Setting up worker environment") - sleepTime := 5 + sleepTime = 5 client := shuffle.GetExternalClient(baseUrl) if timezone == "" { timezone = "Europe/Amsterdam" } - log.Printf("[INFO] Running with timezone %s and swarm config %#v", timezone, os.Getenv("SHUFFLE_SWARM_CONFIG")) + topClient = client + swarmConfig := os.Getenv("SHUFFLE_SWARM_CONFIG") + log.Printf("[INFO] Running with timezone %s and swarm config %#v", timezone, swarmConfig) + authorization := "" executionId := "" @@ -2269,148 +2898,13 @@ func main() { } topClient = client - firstRequest := true environments := []string{} for { - // Because of this, it always has updated data. - // Removed request requirement from app_sdk - newresp, err := client.Do(req) + environments, err = getStreamResultsWrapper(client, req, workflowExecution, firstRequest, environments) if err != nil { - log.Printf("[ERROR] Failed request: %s", err) - time.Sleep(time.Duration(sleepTime) * time.Second) - continue + log.Printf("[ERROR] Failed getting stream results: %s", err) } - - defer newresp.Body.Close() - body, err := ioutil.ReadAll(newresp.Body) - if err != nil { - log.Printf("[ERROR] Failed reading body: %s", err) - time.Sleep(time.Duration(sleepTime) * time.Second) - continue - } - - if newresp.StatusCode != 200 { - log.Printf("[ERROR] %s\nStatusCode (1): %d", string(body), newresp.StatusCode) - time.Sleep(time.Duration(sleepTime) * time.Second) - continue - } - - err = json.Unmarshal(body, &workflowExecution) - if err != nil { - log.Printf("[ERROR] Failed workflowExecution unmarshal: %s", err) - time.Sleep(time.Duration(sleepTime) * time.Second) - continue - } - - if firstRequest { - firstRequest = false - //workflowExecution.StartedAt = int64(time.Now().Unix()) - - ctx := context.Background() - cacheKey := fmt.Sprintf("workflowexecution_%s", workflowExecution.ExecutionId) - execData, err := json.Marshal(workflowExecution) - if err != nil { - log.Printf("[ERROR][%s] Failed marshalling execution during set (3): %s", workflowExecution.ExecutionId, err) - } else { - err = shuffle.SetCache(ctx, cacheKey, execData, 30) - if err != nil { - log.Printf("[ERROR][%s] Failed adding to cache during setexecution (3): %s", workflowExecution.ExecutionId, err) - } - } - - //requestCache = cache.New(60*time.Minute, 120*time.Minute) - //requestCache.Set(cacheKey, &workflowExecution, cache.DefaultExpiration) - - for _, action := range workflowExecution.Workflow.Actions { - found := false - for _, environment := range environments { - if action.Environment == environment { - found = true - break - } - } - - if !found { - environments = append(environments, action.Environment) - } - } - - // Checks if a subflow is child of the startnode, as sub-subflows aren't working properly yet - childNodes := shuffle.FindChildNodes(workflowExecution, workflowExecution.Start, []string{}, []string{}) - log.Printf("[DEBUG] Looking for subflow in %#v to check execution pattern as child of %s", childNodes, workflowExecution.Start) - subflowFound := false - for _, childNode := range childNodes { - for _, trigger := range workflowExecution.Workflow.Triggers { - if trigger.ID != childNode { - continue - } - - if trigger.AppName == "Shuffle Workflow" { - subflowFound = true - break - } - } - - if subflowFound { - break - } - } - - log.Printf("\n\nEnvironments: %s. Source: %s. 1 env = webserver, 0 or >1 = default. Subflow exists: %#v\n\n", environments, workflowExecution.ExecutionSource, subflowFound) - if len(environments) == 1 && workflowExecution.ExecutionSource != "default" && !subflowFound { - log.Printf("\n\n[DEBUG] Running OPTIMIZED execution (not manual)\n\n") - listener := webserverSetup(workflowExecution) - err := executionInit(workflowExecution) - if err != nil { - log.Printf("[DEBUG] Workflow setup failed: %s", workflowExecution.ExecutionId, err) - log.Printf("[DEBUG] Shutting down (30)") - shutdown(workflowExecution, "", "", true) - } - - go func() { - time.Sleep(time.Duration(1)) - handleExecutionResult(workflowExecution) - }() - - runWebserver(listener) - //log.Printf("Before wait") - //wg := sync.WaitGroup{} - //wg.Add(1) - //wg.Wait() - } else { - log.Printf("\n\n[DEBUG] Running NON-OPTIMIZED execution for type %s with %d environment(s). This only happens when ran manually OR when running with subflows. Status: %s\n\n", workflowExecution.ExecutionSource, len(environments), workflowExecution.Status) - err := executionInit(workflowExecution) - if err != nil { - log.Printf("[DEBUG] Workflow setup failed: %s", workflowExecution.ExecutionId, err) - shutdown(workflowExecution, "", "", true) - } - - // Trying to make worker into microservice~ :) - } - } - - if workflowExecution.Status == "FINISHED" || workflowExecution.Status == "SUCCESS" { - log.Printf("[DEBUG] Workflow %s is finished. Exiting worker.", workflowExecution.ExecutionId) - log.Printf("[DEBUG] Shutting down (31)") - shutdown(workflowExecution, "", "", true) - } - - if workflowExecution.Status == "EXECUTING" || workflowExecution.Status == "RUNNING" { - //log.Printf("Status: %s", workflowExecution.Status) - err = handleDefaultExecution(client, req, workflowExecution) - if err != nil { - log.Printf("[DEBUG] Workflow %s is finished: %s", workflowExecution.ExecutionId, err) - log.Printf("[DEBUG] Shutting down (32)") - shutdown(workflowExecution, "", "", true) - } - } else { - log.Printf("[DEBUG] Workflow %s has status %s. Exiting worker.", workflowExecution.ExecutionId, workflowExecution.Status) - log.Printf("[DEBUG] Shutting down (33)") - shutdown(workflowExecution, workflowExecution.Workflow.ID, "", true) - } - - time.Sleep(time.Duration(sleepTime) * time.Second) } } @@ -2447,6 +2941,7 @@ func checkUnfinished(resp http.ResponseWriter, request *http.Request, execReques } func handleRunExecution(resp http.ResponseWriter, request *http.Request) { + defer request.Body.Close() body, err := ioutil.ReadAll(request.Body) if err != nil { log.Printf("[WARNING] Failed reading body for stream result queue") @@ -2455,8 +2950,6 @@ func handleRunExecution(resp http.ResponseWriter, request *http.Request) { return } - defer request.Body.Close() - //log.Printf("[DEBUG] In run execution with body length %d", len(body)) var execRequest shuffle.OrborusExecutionRequest err = json.Unmarshal(body, &execRequest) @@ -2511,10 +3004,10 @@ func handleRunExecution(resp http.ResponseWriter, request *http.Request) { os.Setenv("AUTHORIZATION", execRequest.Authorization) } - topClient = &http.Client{} var workflowExecution shuffle.WorkflowExecution data = fmt.Sprintf(`{"execution_id": "%s", "authorization": "%s"}`, execRequest.ExecutionId, execRequest.Authorization) streamResultUrl := fmt.Sprintf("%s/api/v1/streams/results", baseUrl) + topClient = shuffle.GetExternalClient(streamResultUrl) req, err := http.NewRequest( "POST", @@ -2530,6 +3023,7 @@ func handleRunExecution(resp http.ResponseWriter, request *http.Request) { return } + defer newresp.Body.Close() body, err = ioutil.ReadAll(newresp.Body) if err != nil { log.Printf("[ERROR] Failed reading body (2): %s", err) @@ -2560,6 +3054,7 @@ func handleRunExecution(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() + //err = shuffle.SetWorkflowExecution(ctx, workflowExecution, true) err = setWorkflowExecution(ctx, workflowExecution, true) if err != nil { log.Printf("[ERROR] Failed initializing execution saving for %s: %s", workflowExecution.ExecutionId, err) @@ -2601,14 +3096,12 @@ func handleRunExecution(resp http.ResponseWriter, request *http.Request) { if err != nil { log.Printf("[ERROR][%s] Failed marshalling execution during set (3): %s", workflowExecution.ExecutionId, err) } else { - err = shuffle.SetCache(ctx, cacheKey, execData, 30) + err = shuffle.SetCache(ctx, cacheKey, execData, 31) if err != nil { log.Printf("[ERROR][%s] Failed adding to cache during setexecution (3): %s", workflowExecution.ExecutionId, err) } } - //requestCache.Set(cacheKey, &workflowExecution, cache.DefaultExpiration) - err = executionInit(workflowExecution) if err != nil { log.Printf("[DEBUG][%s] Shutting down (30) - Workflow setup failed: %s", workflowExecution.ExecutionId, workflowExecution.ExecutionId, err) @@ -2618,16 +3111,95 @@ func handleRunExecution(resp http.ResponseWriter, request *http.Request) { //shutdown(workflowExecution, "", "", true) } - //go handleExecutionResult(workflowExecution) handleExecutionResult(workflowExecution) resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } +func handleDownloadImage(resp http.ResponseWriter, request *http.Request) { + // Read the request body + defer request.Body.Close() + bodyBytes, err := ioutil.ReadAll(request.Body) + if err != nil { + log.Printf("[ERROR] Failed reading body for stream result queue. Error: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + return + } + + // get images from request + image := &ImageDownloadBody{} + err = json.Unmarshal(bodyBytes, image) + if err != nil { + log.Printf("[ERROR] Error in unmarshalling body: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + return + } + + client, err := dockerclient.NewEnvClient() + if err != nil { + log.Printf("[ERROR] Unable to create docker client (4): %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + return + } + + // check if images are already downloaded + // Retrieve a list of Docker images + images, err := client.ImageList(context.Background(), types.ImageListOptions{}) + if err != nil { + log.Printf("[ERROR] listing images: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + return + } + + for _, img := range images { + for _, tag := range img.RepoTags { + splitTag := strings.Split(tag, ":") + baseTag := tag + if len(splitTag) > 1 { + baseTag = splitTag[1] + } + + var possibleNames []string + possibleNames = append(possibleNames, fmt.Sprintf("frikky/shuffle:%s", baseTag)) + possibleNames = append(possibleNames, fmt.Sprintf("registry.hub.docker.com/frikky/shuffle:%s", baseTag)) + + if (arrayContains(possibleNames, image.Image)) { + log.Printf("[DEBUG] Image %s already downloaded that has been requested to download", image.Image) + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "image already present"}`))) + return + } + } + } + + log.Printf("[INFO] Downloading image %s", image.Image) + downloadDockerImageBackend(&http.Client{Timeout: 60 * time.Second}, image.Image) + + // return success + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true, "status": "starting download"}`))) +} + func runWebserver(listener net.Listener) { r := mux.NewRouter() r.HandleFunc("/api/v1/streams", handleWorkflowQueue).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/streams/results", handleGetStreamResults).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/execute", handleRunExecution).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/run", handleRunExecution).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/download", handleDownloadImage).Methods("POST", "OPTIONS") + + + if strings.ToLower(os.Getenv("SHUFFLE_DEBUG_MEMORY")) == "true" { + r.HandleFunc("/debug/pprof/", pprof.Index) + r.HandleFunc("/debug/pprof/heap", pprof.Handler("heap").ServeHTTP) + r.HandleFunc("/debug/pprof/profile", pprof.Profile) + r.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + r.HandleFunc("/debug/pprof/trace", pprof.Trace) + } //log.Fatal(http.ListenAndServe(port, nil)) //srv := http.Server{ @@ -2638,7 +3210,7 @@ func runWebserver(listener net.Listener) { //log.Fatal(http.Serve(listener, nil)) - log.Printf("\n\n[DEBUG] NEW webserver setup\n\n") + log.Printf("[DEBUG] NEW webserver setup") http.Handle("/", r) srv := http.Server{ @@ -2651,7 +3223,7 @@ func runWebserver(listener net.Listener) { err := srv.Serve(listener) if err != nil { - log.Printf("serveIssue: %#v", err) + log.Printf("[ERROR] Serve issue in worker: %#v", err) } log.Printf("[DEBUG] Do we see this?") } From bb37e3e55d4d4062b3f3c98b1f3b344acad85230 Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 7 Dec 2023 21:54:52 +0100 Subject: [PATCH 33/33] Minor play button fixes for worker + backend --- backend/go-app/go.mod | 2 +- backend/go-app/main.go | 3 + backend/go-app/walkoff.go | 183 +++++++++++++++++- frontend/src/components/ShuffleCodeEditor.jsx | 31 ++- frontend/src/views/AngularWorkflow.jsx | 81 +++----- 5 files changed, 235 insertions(+), 65 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index cbbc0ea3..af4b2f8b 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -18,7 +18,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.5.29 + github.com/shuffle/shuffle-shared v0.5.30 golang.org/x/crypto v0.14.0 google.golang.org/api v0.125.0 google.golang.org/grpc v1.55.0 diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 0d0f2f0f..571e8422 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -4801,6 +4801,7 @@ func initHandlers() { // App specific // From here down isnt checked for org specific r.HandleFunc("/api/v1/apps/{key}/execute", executeSingleAction).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/apps/{key}/run", executeSingleAction).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/categories", shuffle.GetActiveCategories).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/apps/categories/run", shuffle.RunCategoryAction).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/upload", handleAppZipUpload).Methods("POST", "OPTIONS") @@ -4841,9 +4842,11 @@ func initHandlers() { r.HandleFunc("/api/v1/workflows/search", shuffle.HandleWorkflowRunSearch).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/schedules", shuffle.HandleGetSchedules).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/executions", shuffle.GetWorkflowExecutions).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/workflows/{key}/executions/{key}/rerun", checkUnfinishedExecution).Methods("GET", "POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/executions/{key}/abort", shuffle.AbortExecution).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/schedule", scheduleWorkflow).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/download_remote", loadSpecificWorkflows).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/workflows/{key}/run", executeWorkflow).Methods("GET", "POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/execute", executeWorkflow).Methods("GET", "POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/schedule/{schedule}", stopSchedule).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/stream", shuffle.HandleStreamWorkflow).Methods("GET", "OPTIONS") diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 05ced567..dafbcdc7 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -1667,7 +1667,7 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request // FIXME - tmp name based on future companyname-companyId // This leads to issues with overlaps. Should set limits and such instead for _, environment := range execInfo.Environments { - log.Printf("[INFO] Execution: %s should execute onprem with execution environment \"%s\". Workflow: %s", workflowExecution.ExecutionId, environment, workflowExecution.Workflow.ID) + log.Printf("[INFO][%s] Execution: should execute onprem with execution environment \"%s\". Workflow: %s", workflowExecution.ExecutionId, environment, workflowExecution.Workflow.ID) executionRequest := shuffle.ExecutionRequest{ ExecutionId: workflowExecution.ExecutionId, @@ -3360,11 +3360,21 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) { return } - workflowExecution.Priority = 10 + + workflowExecution.Priority = 11 environments, err := shuffle.GetEnvironments(ctx, user.ActiveOrg.Id) environment := "Shuffle" if len(environments) >= 1 { + // Find default one environment = environments[0].Name + + for _, env := range environments { + if env.Default { + environment = env.Name + break + } + } + } else { log.Printf("[ERROR] No environments found for org %s. Exiting", user.ActiveOrg.Id) resp.WriteHeader(401) @@ -3372,6 +3382,14 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) { return } + // Enforcing same env for job + run to be default + // FIXME: Should use environment that is in the source workflow if it exists + for i, _ := range workflowExecution.Workflow.Actions { + workflowExecution.Workflow.Actions[i].Environment = environment + workflowExecution.Workflow.Actions[i].Label = "TMP" + } + shuffle.SetWorkflowExecution(ctx, workflowExecution, false) + log.Printf("[INFO] Execution (single action): %s should execute onprem with execution environment \"%s\". Workflow: %s", workflowExecution.ExecutionId, environment, workflowExecution.Workflow.ID) executionRequest := shuffle.ExecutionRequest{ @@ -3379,6 +3397,7 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) { WorkflowId: workflowExecution.Workflow.ID, Authorization: workflowExecution.Authorization, Environments: []string{environment}, + Priority: 11, } executionRequest.Priority = workflowExecution.Priority @@ -3399,6 +3418,9 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) { log.Printf("[ERROR] Failed to marshal retStruct in single execution: %s", err) } + // Deleting as this is a single action and doesn't need to be stored + shuffle.DeleteKey(ctx, "workflowexecution", executionRequest.ExecutionId) + resp.WriteHeader(200) resp.Write([]byte(returnBytes)) } @@ -3995,3 +4017,160 @@ func checkWorkflowApp(workflowApp shuffle.WorkflowApp) error { return nil } + +func checkUnfinishedExecution(resp http.ResponseWriter, request *http.Request) { + cors := shuffle.HandleCors(resp, request) + if cors { + return + } + + location := strings.Split(request.URL.String(), "/") + var fileId string + if location[1] == "api" { + if len(location) <= 4 { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + fileId = location[4] + } + + if len(fileId) != 36 { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Workflow ID to abort is not valid"}`)) + return + } + + executionId := location[6] + if len(executionId) != 36 { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "ExecutionID not valid"}`)) + return + } + + ctx := shuffle.GetContext(request) + exec, err := shuffle.GetWorkflowExecution(ctx, executionId) + if err != nil { + log.Printf("[ERROR] Failed getting execution (rerun workflow - 1) %s: %s", executionId, err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting execution ID %s because it doesn't exist (abort)."}`, executionId))) + return + } + + apikey := request.Header.Get("Authorization") + parsedKey := "" + if strings.HasPrefix(apikey, "Bearer ") { + apikeyCheck := strings.Split(apikey, " ") + if len(apikeyCheck) == 2 { + parsedKey = apikeyCheck[1] + } + } + + // ONLY allowed to run automatically with the same auth (july 2022) + if exec.Authorization != parsedKey { + user, err := shuffle.HandleApiAuthentication(resp, request) + if err != nil { + log.Printf("[ERROR][%s] Bad authorization key for execution (rerun workflow - 3): %s", executionId, err) + resp.WriteHeader(403) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed because you're not authorized to see this workflow (3)."}`))) + return + } + + // Check if user is in the correct org + if user.ActiveOrg.Id == exec.ExecutionOrg && user.Role != "org-reader" { + log.Printf("[AUDIT][%s] User %s (%s) is force continuing execution from org access", executionId, user.Username, user.Id) + } else if user.SupportAccess { + log.Printf("[AUDIT][%s] User %s (%s) is force continuing execution with support access", executionId, user.Username, user.Id) + } else { + log.Printf("[ERROR][%s] Bad authorization key for continue execution (rerun workflow - 2): %s", executionId, err) + resp.WriteHeader(403) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed because you're not authorized to see this workflow (2)."}`))) + return + } + } + + // Meant as a function that periodically checks whether previous executions have finished or not. + // Should probably be based on executedIds and finishedIds + // Schedule a check in the future instead? + + // Auth vs execution check! + extraInputs := 0 + for _, trigger := range exec.Workflow.Triggers { + if trigger.Name == "User Input" && trigger.AppName == "User Input" { + extraInputs += 1 + + //exec.Workflow.Actions = append(exec.Workflow.Actions, shuffle.Action{ + // ID: trigger.ID, + // Label: trigger.Label, + // Name: trigger.Name, + //}) + } else if trigger.Name == "Shuffle Workflow" && trigger.AppName == "Shuffle Workflow" { + extraInputs += 1 + + //exec.Workflow.Actions = append(exec.Workflow.Actions, shuffle.Action{ + // ID: trigger.ID, + // Label: trigger.Label, + // Name: trigger.Name, + //}) + } + } + + if exec.Status != "ABORTED" && exec.Status != "FINISHED" && exec.Status != "FAILURE" { + log.Printf("[DEBUG][%s] Rechecking execution and its status to send to backend IF the status is EXECUTING (%s - %d/%d finished)", exec.ExecutionId, exec.Status, len(exec.Results), len(exec.Workflow.Actions)+extraInputs) + } + + // Usually caused by issue during startup + if exec.Status == "" { + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No status for the execution"}`))) + return + } + + if exec.Status != "EXECUTING" { + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Already finished"}`))) + return + } + + // Force it back in the queue to be executed + if len(exec.Workflow.Actions) == 0 { + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Not a cloud env workflow. Only rerunning cloud env."}`))) + return + } + + log.Printf("[DEBUG][%s] Workflow: %s (%s)", exec.ExecutionId, exec.Workflow.Name, exec.Workflow.ID) + if exec.Workflow.ID == "" || exec.Workflow.Name == "" { + log.Printf("[ERROR][%s] No workflow ID found for execution", exec.ExecutionId) + shuffle.DeleteKey(ctx, "workflowexecution", exec.ExecutionId) + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "No workflow name / ID found. Can't run. Contact support@shuffler.io if this persists."}`))) + return + } + + environment := exec.Workflow.Actions[0].Environment + log.Printf("[DEBUG][%s] Not a cloud env workflow. Re-adding job in queue for env %s.", exec.ExecutionId, environment) + + parsedEnv := fmt.Sprintf("%s_%s", strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(environment, " ", "-"), "_", "-")), exec.ExecutionOrg) + log.Printf("[DEBUG][%s] Adding new run job to env (2): %s", exec.ExecutionId, parsedEnv) + + executionRequest := shuffle.ExecutionRequest{ + ExecutionId: exec.ExecutionId, + WorkflowId: exec.Workflow.ID, + Authorization: exec.Authorization, + Environments: []string{environment}, + } + + // Increase priority on reruns to catch up + executionRequest.Priority = 11 + err = shuffle.SetWorkflowQueue(ctx, executionRequest, parsedEnv) + if err != nil { + log.Printf("[ERROR] Failed adding execution to db: %s", err) + } + + + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Reran workflow in %s"}`, parsedEnv))) + +} diff --git a/frontend/src/components/ShuffleCodeEditor.jsx b/frontend/src/components/ShuffleCodeEditor.jsx index 3053083c..39b1a26c 100644 --- a/frontend/src/components/ShuffleCodeEditor.jsx +++ b/frontend/src/components/ShuffleCodeEditor.jsx @@ -1597,14 +1597,29 @@ const CodeEditor = (props) => { Expected Output - { - executeSingleAction(expOutput) - }}> - - {executing ? : } - - - + + + } diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index e9e28582..53a43232 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -11151,15 +11151,7 @@ const AngularWorkflow = (defaultprops) => { style={{ marginTop: 10 }} label={
Wait for results
} /> - -
+
{ // email,sms,app ... workflow.triggers[selectedTriggerIndex].parameters[2] = { name: "type", - value: "email", + value: "subflow", }; workflow.triggers[selectedTriggerIndex].parameters[3] = { @@ -12624,16 +12616,7 @@ const AngularWorkflow = (defaultprops) => { />
*/} - -
- Parameters +
{ display: "flex", }} > -
Information + + The information you want to show the user. Supports variables. +
{ display: "flex", }} > -
- Contact options + Input options + + Use subflows to connect to any app you want, or use the default email and sms options +
{ { + setTriggerOptionsWrapper("subflow"); + }} + color="primary" + value="subflow" + /> + } + label={
Subflow
} + /> + { 0 && workflow.triggers[selectedTriggerIndex].parameters[2] !== undefined && workflow.triggers[selectedTriggerIndex].parameters[2].value !== undefined ? workflow.triggers[selectedTriggerIndex].parameters[2].value.includes("sms") : false} onChange={() => { setTriggerOptionsWrapper("sms"); @@ -12728,21 +12714,8 @@ const AngularWorkflow = (defaultprops) => { } label={
SMS
} /> - { - setTriggerOptionsWrapper("subflow"); - }} - color="primary" - value="subflow" - /> - } - label={
Subflow
} - /> - {workflow.triggers[selectedTriggerIndex].parameters[2] !== undefined && workflow.triggers[selectedTriggerIndex].parameters[2].value.includes("subflow") ? ( + {workflow.triggers[selectedTriggerIndex].parameters[2] !== undefined && workflow.triggers[selectedTriggerIndex].parameters[2].value.includes("subflow") ? (
{workflows === undefined || workflows === null || @@ -12848,7 +12821,7 @@ const AngularWorkflow = (defaultprops) => { }, }} fullWidth - label="Email" + label="Email" color="primary" required placeholder={"mail1@company.com,mail2@company.com"}