From 106ffa3174240d8516fb160d27aecb01eaf54460 Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 15 Oct 2021 15:16:35 +0200 Subject: [PATCH 01/18] Fixed frontend security issues --- backend/go-app/go.mod | 52 +++++++++---------------------- docker-compose.yml | 6 ++-- frontend/package.json | 9 +++--- functions/onprem/orborus/build.sh | 2 +- functions/onprem/worker/build.sh | 2 +- 5 files changed, 25 insertions(+), 46 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 16f22e0f..c178559b 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -1,50 +1,28 @@ -module shuffle +module main -go 1.13 - -replace github.com/shuffle/shuffle-shared => ../../../../git/shuffle-shared - -//replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi +go 1.15 require ( - cloud.google.com/go v0.75.0 - cloud.google.com/go/datastore v1.4.0 - cloud.google.com/go/pubsub v1.3.1 - cloud.google.com/go/storage v1.12.0 - github.com/Masterminds/semver v1.5.0 // indirect - github.com/RobotsAndPencils/go-saml v0.0.0-20170520135329-fb13cb52a46b // indirect - github.com/algolia/algoliasearch-client-go/v3 v3.18.1 // indirect + cloud.google.com/go/datastore v1.6.0 + cloud.google.com/go/pubsub v1.17.0 + cloud.google.com/go/storage v1.18.1 github.com/basgys/goxml2json v1.1.0 - github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 // indirect github.com/carlescere/scheduler v0.0.0-20170109141437-ee74d2f83d82 - github.com/docker/distribution v2.7.1+incompatible // indirect - github.com/docker/docker v20.10.3-0.20210216175712-646072ed6524+incompatible - github.com/docker/go-connections v0.4.0 - github.com/docker/go-units v0.4.0 // indirect - github.com/elastic/go-elasticsearch/v7 v7.13.1 // indirect - github.com/frikky/kin-openapi v0.39.0 - github.com/frikky/shuffle-shared v0.1.15 - github.com/fsouza/go-dockerclient v1.7.2 + github.com/docker/docker v20.10.9+incompatible + github.com/frikky/kin-openapi v0.40.0 + github.com/fsouza/go-dockerclient v1.7.4 github.com/ghodss/yaml v1.0.0 - github.com/go-git/go-billy/v5 v5.0.0 - github.com/go-git/go-git/v5 v5.0.0 - github.com/google/go-github/v28 v28.1.1 - github.com/gorilla/handlers v1.4.2 // indirect + github.com/go-git/go-billy/v5 v5.3.1 + github.com/go-git/go-git/v5 v5.4.2 github.com/gorilla/mux v1.8.0 - github.com/h2non/filetype v1.0.12 - github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect - github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d // indirect - github.com/patrickmn/go-cache v2.1.0+incompatible + github.com/h2non/filetype v1.1.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.1.15 + github.com/shuffle/shuffle-shared v0.1.17 go4.org v0.0.0-20201209231011-d4a079459e60 // indirect - golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 - golang.org/x/oauth2 v0.0.0-20210113160501-8b1d76fa0423 - google.golang.org/api v0.36.0 + golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 + google.golang.org/api v0.58.0 google.golang.org/appengine v1.6.7 - google.golang.org/genproto v0.0.0-20210113195801-ae06605f4595 - google.golang.org/grpc v1.34.1 + google.golang.org/grpc v1.41.0 gopkg.in/src-d/go-git.v4 v4.13.1 - gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) diff --git a/docker-compose.yml b/docker-compose.yml index e804bb4f..4205bb18 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: frontend: #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:latest + image: ghcr.io/frikky/shuffle-frontend:nightly container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -17,7 +17,7 @@ services: - backend backend: #build: ./backend - image: ghcr.io/frikky/shuffle-backend:latest + image: ghcr.io/frikky/shuffle-backend:nightly container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: @@ -40,7 +40,7 @@ services: #- database orborus: #build: ./functions/onprem/orborus - image: ghcr.io/frikky/shuffle-orborus:latest + image: ghcr.io/frikky/shuffle-orborus:nightly container_name: shuffle-orborus hostname: shuffle-orborus networks: diff --git a/frontend/package.json b/frontend/package.json index 14744ca8..aa5d6139 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,9 +1,10 @@ { "name": "shuffler", "homepage": "https://shuffler.io", - "version": "0.9.24", + "version": "0.9.25", "private": true, "dependencies": { + "@babel/core": "^7.15.8", "@material-ui/core": "^4.5.2", "@material-ui/data-grid": "^4.0.0-alpha.22", "@material-ui/icons": "^4.11.2", @@ -13,8 +14,8 @@ "@uiw/react-codemirror": "^3.2.1", "@use-it/interval": "^1.0.0", "babel-eslint": "^10.1.0", - "class-transformer": "^0.3.1", - "create-react-app": "^2.0.3", + "class-transformer": "^0.4.0", + "create-react-app": "^4.0.3", "cytoscape": "^3.11.0", "cytoscape-clipboard": "^2.2.1", "cytoscape-cxtmenu": "^3.1.1", @@ -62,7 +63,7 @@ "shellwords": "^0.1.1", "simplebar": "^4.2.3", "styled-components": "^4.4.0", - "webpack": "4.44.2", + "webpack": "^4.44.2", "websocket": "^1.0.30", "yaml": "^1.7.2", "yamljs": "^0.3.0", diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index 98bb5f27..b1117435 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-orborus -VERSION=0.9.23 +VERSION=0.9.25 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh index 779339e7..56f46f97 100644 --- a/functions/onprem/worker/build.sh +++ b/functions/onprem/worker/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-worker -VERSION=0.9.23 +VERSION=0.9.25 echo "Running docker build with $NAME:$VERSION" #CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin . From b73df06667d16694385ef7cc38456945c6a22051 Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 15 Oct 2021 17:12:23 +0200 Subject: [PATCH 02/18] Fixed workflow bug for launch --- frontend/Dockerfile | 1 + frontend/src/views/Workflows.jsx | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 12ba0b27..5c29245a 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -17,6 +17,7 @@ COPY ./src /usr/src/app/src/ COPY ./*.sh /usr/src/app/ COPY ./*.json /usr/src/app/ +RUN rm -rf /usr/src/app/node_modules/webpack RUN yarn build # Production environment diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 5805dd78..2a4cbd8b 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -1604,7 +1604,7 @@ const Workflows = (props) => { return ( - + {data.name} From 348fcaaf934e9b76643ec5f7d312f7836c5c3ccb Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 17 Oct 2021 03:26:55 +0200 Subject: [PATCH 03/18] Deprecated use of old casting system to push use of Liquid formatting (ok just in betagit status) --- backend/app_sdk/app_base.py | 56 ++++++++++++++++++-------- backend/app_sdk/build.sh | 2 +- frontend/src/defaultCytoscapeStyle.js | 2 + frontend/src/views/AngularWorkflow.jsx | 2 + 4 files changed, 44 insertions(+), 18 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index b725ae9f..73ed57e8 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -1147,6 +1147,11 @@ class AppBase: except TypeError: return data, False + # Because liquid can handle ALL of this now. + # Implemented for >0.9.25 + self.logger.info("[DEBUG] Skipping parser because use of its been deprecated >0.9.25 due to Liquid implementation") + return data, False + wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length", "lenght", "join", "replace"] if not any(wrapper in data for wrapper in wrappers): @@ -1219,7 +1224,7 @@ class AppBase: if isinstance(data, str) and len(data) > 4: if (data[0] == "{" or data[0] == "[") and (data[len(data)-1] == "]" or data[len(data)-1] == "}"): - self.logger.info("Skipping parser because use of {[ and ]}") + self.logger.info("[DEBUG] Skipping parser because use of {[ and ]}") return data newdata = [] @@ -1587,10 +1592,11 @@ class AppBase: # Sending self as it's not a normal function def parse_liquid(template, self): - - #self.logger.info("Inside liquid with glob: %s" % globals()) + + errors = False + error_msg = "" try: - if len(template) > 5000000: + if len(template) > 10000000: self.logger.info("[DEBUG] Skipping liquid - size too big (%d)" % len(template)) return template @@ -1609,21 +1615,37 @@ class AppBase: # Can't handle self yet (?) ret = run.render(**globals()) return ret - #try: - #run = Liquid(template) - #return ret - #except liquid.exceptions.LiquidSyntaxError as e: - # run = Liquid(template, {'mode': 'python'}) - # ret = run.render(**globals()) - # return ret - #except liquid.exceptions.LiquidRenderError as e: - # self.logger.info("Render error: %s" % e) except jinja2.exceptions.TemplateNotFound as e: - self.logger.info("[ERROR] Template error: %s" % e) + self.logger.info(f"[ERROR] Liquid Template error: {e}") + error = True + error_msg = e except jinja2.exceptions.TemplateSyntaxError as e: - self.logger.info("[ERROR] Syntax error: %s" % e) - except: - self.logger.info("[ERROR] General exception for liquid") + self.logger.info(f"[ERROR] Liquid Syntax error: {e}") + error = True + error_msg = e + except Exception as e: + self.logger.info(f"[ERROR] General exception for liquid: {e}") + error = True + error_msg = e + + if error == True: + self.action_result["status"] = "FAILURE" + data = { + "success": False, + "input": template, + "reason": f"Failed to parse LiquidPy: {error_msg}", + } + try: + self.action_result["result"] = json.dumps(data) + except Exception as e: + self.action_result["result"] = f"Failed to parse LiquidPy: {error_msg}" + print("[WARNING] Failed to set LiquidPy result") + + self.action_result["completed_at"] = int(time.time()) + self.send_result(self.action_result, headers, stream_path) + + self.logger.info(f"[ERROR] Sent FAILURE response to backend due to : {e}") + os.exit() return template diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index a7518354..ae5e1f56 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -3,7 +3,7 @@ ### DEFAULT NAME=shuffle-app_sdk -VERSION=0.9.23 +VERSION=0.9.25 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly diff --git a/frontend/src/defaultCytoscapeStyle.js b/frontend/src/defaultCytoscapeStyle.js index 48642d60..7f3c84cd 100644 --- a/frontend/src/defaultCytoscapeStyle.js +++ b/frontend/src/defaultCytoscapeStyle.js @@ -14,6 +14,7 @@ const data = [{ 'margin': '5px', 'border-width': '1px', 'text-margin-x': '10px', + 'cursor': 'pointer', } }, { @@ -26,6 +27,7 @@ const data = [{ 'text-margin-y': '-15px', 'width': '5px', "color": "white", + 'cursor': 'pointer', "line-fill": "linear-gradient", "line-gradient-stop-positions": ["0.0", "100"], "line-gradient-stop-colors": ["grey", "grey"], diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 78da9dee..f9a0caed 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -887,6 +887,8 @@ const AngularWorkflow = (props) => { //console.log(curworkflowTrigger) newTriggers.push(curworkflowTrigger) + } else { + alert.info("No handler for type: "+type) } } } From 6d3824cf6f002ed8db00f565b1468fc5571ff919 Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 18 Oct 2021 16:48:12 +0200 Subject: [PATCH 04/18] Added possibility of hiding/showing body for full control in in HTTP requests --- backend/app_sdk/app_base.py | 2 +- backend/app_sdk/build.sh | 2 +- backend/go-app/walkoff.go | 16 ++++- frontend/src/components/ParsedAction.jsx | 76 +++++++++++++++++++++--- frontend/src/defaultCytoscapeStyle.js | 2 +- frontend/src/views/AngularWorkflow.jsx | 4 +- 6 files changed, 86 insertions(+), 16 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 73ed57e8..d8e04d6e 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -2555,7 +2555,7 @@ class AppBase: break except TypeError as e: newres = "" - self.logger.info(f"[DEBUG] Got exec error: {errorstring}") + self.logger.info(f"[DEBUG] Got exec error: {e}") errorstring = f"{e}" if "got an unexpected keyword argument" in errorstring: fieldsplit = errorstring.split("'") diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index ae5e1f56..847c0185 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -3,7 +3,7 @@ ### DEFAULT NAME=shuffle-app_sdk -VERSION=0.9.25 +VERSION=0.9.26 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 7078110e..22b90cd1 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -4050,9 +4050,19 @@ func LoadSpecificApps(resp http.ResponseWriter, request *http.Request) { if tmpBody.ForceUpdate { dockercli, err := dockerclient.NewEnvClient() if err == nil { - _, err := dockercli.ImagePull(ctx, "frikky/shuffle:app_sdk", types.ImagePullOptions{}) - if err != nil { - log.Printf("[WARNING] Failed to download apps with the new App SDK: %s", err) + + appSdk := os.Getenv("SHUFFLE_APP_SDK_VERSION") + if len(appSdk) == 0 { + _, err := dockercli.ImagePull(ctx, "frikky/shuffle:app_sdk", types.ImagePullOptions{}) + if err != nil { + log.Printf("[WARNING] Failed to download new App SDK: %s", err) + } + } else { + _, err := dockercli.ImagePull(ctx, fmt.Sprintf("%s/%s/shuffle-app_sdk:%s", "ghcr.io", "frikky", appSdk), types.ImagePullOptions{}) + if err != nil { + log.Printf("[WARNING] Failed to download new App SDK %s: %s", err) + } + } } else { log.Printf("[WARNING] Failed to download apps with the new App SDK because of docker cli: %s", err) diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index ef635b6f..b6bcbd46 100644 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -49,6 +49,7 @@ const useStyles = makeStyles({ // }, //) +const openApiFieldDesc = "Generated by OpenAPI body example" const ParsedAction = (props) => { const {workflow, setWorkflow, setAction, setSelectedAction, setUpdate, appActionArguments, selectedApp, workflowExecutions, setSelectedResult, selectedAction, setSelectedApp, setSelectedTrigger, setSelectedEdge, setCurrentView, cy, setAuthenticationModalOpen,setVariablesModalOpen, setCodeModalOpen, selectedNameChange, rightsidebarStyle, showEnvironment, selectedActionEnvironment, environments, setNewSelectedAction, appApiViewStyle, globalUrl, setSelectedActionEnvironment, requiresAuthentication, hideExtraTypes, scrollConfig, setScrollConfig, authenticationType, appAuthentication, getAppAuthentication } = props @@ -56,6 +57,8 @@ const ParsedAction = (props) => { const classes = useStyles() const [expansionModalOpen, setExpansionModalOpen] = React.useState(false); + const [hideBody, setHideBody] = React.useState(false) + const [activateHidingBody, setActivateHidingBody] = React.useState(false) const keywords = ["len(", "lower(", "upper(", "trim(", "split(", "length(", "number(", "parse(", "join("] const getParents = (action) => { @@ -258,7 +261,8 @@ const ParsedAction = (props) => { if (actionlist.length === 0) { // FIXME: Have previous execution values in here - actionlist.push({"type": "Execution Argument", "name": "Execution Argument", "value": "$exec", "highlight": "exec", "autocomplete": "exec", "example": "hello"}) + actionlist.push({"type": "Execution Argument", "name": "Execution Argument", "value": "$exec", "highlight": "exec", "autocomplete": "exec", "example": ""}) + actionlist.push({"type": "Shuffle DB", "name": "Shuffle DB", "value": "$shuffle_cache", "highlight": "shuffle", "autocomplete": "shuffle", "example": ""}) if (workflow.workflow_variables !== null && workflow.workflow_variables !== undefined && workflow.workflow_variables.length > 0) { for (var key in workflow.workflow_variables) { const item = workflow.workflow_variables[key] @@ -788,12 +792,52 @@ const ParsedAction = (props) => { //setSelectedActionParameters(selectedActionParameters) } + var hideBodyButton = "" + const hideBodyButtonValue = +
+ + { + setHideBody(!hideBody) + + for (var key in selectedActionParameters) { + var currentItem = selectedActionParameters[key] + + if (currentItem.description === openApiFieldDesc) { + currentItem.field_active = !hideBody + console.log("Changing", currentItem) + } + } + }} + name="requires_unique" + /> + } + label={"Automatically fix body"} + /> + +
+ if (selectedApp.generated && data.name === "body") { const regex = /\${(\w+)}/g const found = placeholder.match(regex) - if (found === null) { + + hideBodyButton = hideBodyButtonValue + if (found === null || !hideBody) { //setExtraBodyFields([]) + // + if (found === null) { + setActivateHidingBody(true) + } } else { + console.log("SHOW BUTTON") + rows = "1" disabled = true openApiHelperText = "OpenAPI spec: fill the following fields." @@ -818,7 +862,7 @@ const ParsedAction = (props) => { selectedActionParameters.push({ action_field: "", configuration: false, - description: "Generated by OpenAPI body example", + description: openApiFieldDesc, example: "", id: "", multiline: false, @@ -830,6 +874,7 @@ const ParsedAction = (props) => { tags: null, value: "", variant: "STATIC_VALUE", + field_active: true, }) } @@ -837,10 +882,14 @@ const ParsedAction = (props) => { setSelectedActionParameters(selectedActionParameters) } - return + return hideBodyButton } } + if (activateHidingBody === true) { + hideBodyButton = "" + } + const clickedFieldId = "rightside_field_"+count // { } */ } else if (selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0) { - if (selectedActionParameters[count].value === "" && selectedActionParameters[count].required) { + if (selectedActionParameters[count].value === "") { + // && selectedActionParameters[count].required) { // Rofl, dirty workaround :) const e = { target: { @@ -1009,7 +1059,7 @@ const ParsedAction = (props) => { } changeActionParameter(e, count, data) - } + } datafield = { top: appBarSize+25, right: 25, height: "80vh", - width: 350, + width: 365, minWidth: 200, maxWidth: 600, maxHeight: "100vh", @@ -8570,6 +8570,7 @@ const AngularWorkflow = (props) => { maxHeight: 700, overflowY: "auto", overflowX: "hidden", + zIndex: 10012, //boxShadow: "none", }, }} @@ -9301,6 +9302,7 @@ const AngularWorkflow = (props) => { maxHeight: 700, padding: 15, overflow: "hidden", + zIndex: 10012, }, }} > From ba0dbb5354659afdc830d4b7ce28685f47debc1c Mon Sep 17 00:00:00 2001 From: frikky Date: Wed, 20 Oct 2021 13:32:01 +0200 Subject: [PATCH 05/18] Bumped Liquidpy and added other webhook protocols (e.g. GET) --- backend/app_sdk/requirements.txt | 2 +- backend/go-app/main.go | 17 +++++++++++++---- docker-compose.yml | 4 ++-- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/backend/app_sdk/requirements.txt b/backend/app_sdk/requirements.txt index 79cf5ea2..d561950b 100644 --- a/backend/app_sdk/requirements.txt +++ b/backend/app_sdk/requirements.txt @@ -1,4 +1,4 @@ urllib3==1.26.5 requests==2.25.1 MarkupSafe==2.0.1 -liquidpy==0.7.1 +liquidpy==0.7.2 diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 2b644674..2cd023b4 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -1955,9 +1955,18 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { // 1. Get callback data // 2. Load the configuration // 3. Execute the workflow - cors := shuffle.HandleCors(resp, request) - if cors { - return + //cors := shuffle.HandleCors(resp, request) + //if cors { + // return + //} + + if request.Method != "POST" { + request.Method = "POST" + } + + if request.Body == nil { + stringReader := strings.NewReader("") + request.Body = ioutil.NopCloser(stringReader) } path := strings.Split(request.URL.String(), "/") @@ -5732,7 +5741,7 @@ func initHandlers() { // Triggers r.HandleFunc("/api/v1/hooks/new", shuffle.HandleNewHook).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/hooks/{key}", handleWebhookCallback).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/hooks/{key}", handleWebhookCallback).Methods("POST", "GET", "PATCH", "PUT", "DELETE", "OPTIONS") r.HandleFunc("/api/v1/hooks/{key}/delete", shuffle.HandleDeleteHook).Methods("DELETE", "OPTIONS") // OpenAPI configuration diff --git a/docker-compose.yml b/docker-compose.yml index 4205bb18..eeefca1b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: frontend: - #build: ./frontend + build: ./frontend image: ghcr.io/frikky/shuffle-frontend:nightly container_name: shuffle-frontend hostname: shuffle-frontend @@ -16,7 +16,7 @@ services: depends_on: - backend backend: - #build: ./backend + build: ./backend image: ghcr.io/frikky/shuffle-backend:nightly container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} From a2507885279266316b6ebe4b392ada86d93d9066 Mon Sep 17 00:00:00 2001 From: frikky Date: Thu, 21 Oct 2021 00:35:47 +0200 Subject: [PATCH 06/18] #168: Fixed Oauth2 refresh token cycle --- backend/go-app/go.mod | 4 + backend/go-app/main.go | 1 + backend/go-app/walkoff.go | 707 +---------------------- frontend/src/components/Oauth2Auth.jsx | 11 +- frontend/src/defaultCytoscapeStyle.js | 11 + frontend/src/views/AngularWorkflow.jsx | 220 ++++--- frontend/src/views/Apps.jsx | 4 +- frontend/src/views/SetAuthentication.jsx | 8 + frontend/src/views/Workflows.jsx | 2 +- 9 files changed, 171 insertions(+), 797 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index c178559b..aecda8fb 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -2,6 +2,10 @@ module main go 1.15 +replace github.com/shuffle/shuffle-shared => ../../../../git/shuffle-shared +//replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi +//replace github.com/frikky/go-elasticsearch => ../../../../git/go-elasticsearch + require ( cloud.google.com/go/datastore v1.6.0 cloud.google.com/go/pubsub v1.17.0 diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 2cd023b4..7bd0f90b 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5648,6 +5648,7 @@ func initHandlers() { log.Printf("[DEBUG] Initialized Shuffle database connection. Setting up environment.") if elasticConfig == "elasticsearch" { + time.Sleep(5 * time.Second) go runInitEs(ctx) } else { go runInit(ctx) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 22b90cd1..56415757 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -1168,8 +1168,6 @@ func getWorkflowLocal(fileId string, request *http.Request) ([]byte, error) { return body, nil } -//// New execution with firestore - func handleExecution(id string, workflow shuffle.Workflow, request *http.Request) (shuffle.WorkflowExecution, string, error) { //go func() { // log.Printf("\n\nPRE TIME: %s\n\n", time.Now().Format("2006-01-02 15:04:05")) @@ -1233,725 +1231,38 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request return shuffle.WorkflowExecution{}, fmt.Sprintf(`workflow %s is invalid`, workflow.ID), errors.New("Failed getting workflow") } - workflowBytes, err := json.Marshal(workflow) + workflowExecution, execInfo, _, err := shuffle.PrepareWorkflowExecution(ctx, workflow, request) if err != nil { - log.Printf("Failed workflow unmarshal in execution: %s", err) + log.Printf("[WARNING] Failed in prepareExecution: %s", err) return shuffle.WorkflowExecution{}, "", err } - //log.Println(workflow) - var workflowExecution shuffle.WorkflowExecution - err = json.Unmarshal(workflowBytes, &workflowExecution.Workflow) + err = imageCheckBuilder(execInfo.ImageNames) if err != nil { - log.Printf("Failed execution unmarshaling: %s", err) - return shuffle.WorkflowExecution{}, "Failed unmarshal during execution", err - } - - makeNew := true - start, startok := request.URL.Query()["start"] - if request.Method == "POST" { - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Printf("[ERROR] Failed request POST read: %s", err) - return shuffle.WorkflowExecution{}, "Failed getting body", err - } - - // This one doesn't really matter. - log.Printf("[INFO] Running POST execution with body of length %d for workflow %s", len(string(body)), workflowExecution.Workflow.ID) - - if len(body) >= 4 { - if body[0] == 34 && body[len(body)-1] == 34 { - body = body[1 : len(body)-1] - } - if body[0] == 34 && body[len(body)-1] == 34 { - body = body[1 : len(body)-1] - } - } - - sourceAuth, sourceAuthOk := request.URL.Query()["source_auth"] - if sourceAuthOk { - //log.Printf("\n\n\nSETTING SOURCE WORKFLOW AUTH TO %s!!!\n\n\n", sourceAuth[0]) - workflowExecution.ExecutionSourceAuth = sourceAuth[0] - } else { - //log.Printf("Did NOT get source workflow") - } - - sourceNode, sourceNodeOk := request.URL.Query()["source_node"] - if sourceNodeOk { - //log.Printf("\n\n\nSETTING SOURCE WORKFLOW NODE TO %s!!!\n\n\n", sourceNode[0]) - workflowExecution.ExecutionSourceNode = sourceNode[0] - } else { - //log.Printf("Did NOT get source workflow") - } - - //workflowExecution.ExecutionSource = "default" - sourceWorkflow, sourceWorkflowOk := request.URL.Query()["source_workflow"] - if sourceWorkflowOk { - //log.Printf("Got source workflow %s", sourceWorkflow) - workflowExecution.ExecutionSource = sourceWorkflow[0] - } else { - //log.Printf("Did NOT get source workflow") - } - - sourceExecution, sourceExecutionOk := request.URL.Query()["source_execution"] - if sourceExecutionOk { - //log.Printf("[INFO] Got source execution%s", sourceExecution) - workflowExecution.ExecutionParent = sourceExecution[0] - } else { - //log.Printf("Did NOT get source execution") - } - - if len(string(body)) < 50 { - //log.Println(body) - // String in string - //log.Println(body) - - //if string(body)[0] == "\"" && string(body)[string(body) - log.Printf("[DEBUG] Body: %s", string(body)) - } - - var execution shuffle.ExecutionRequest - err = json.Unmarshal(body, &execution) - if err != nil { - log.Printf("[WARNING] Failed execution POST unmarshaling - continuing anyway: %s", err) - //return shuffle.WorkflowExecution{}, "", err - } - - if execution.Start == "" && len(body) > 0 { - execution.ExecutionArgument = string(body) - } - - // FIXME - this should have "execution_argument" from executeWorkflow frontend - //log.Printf("EXEC: %#v", execution) - if len(execution.ExecutionArgument) > 0 { - workflowExecution.ExecutionArgument = execution.ExecutionArgument - } - - if len(execution.ExecutionSource) > 0 { - workflowExecution.ExecutionSource = execution.ExecutionSource - } - - //log.Printf("Execution data: %#v", execution) - if len(execution.Start) == 36 && len(workflow.Actions) > 0 { - log.Printf("[INFO] Should start execution on node %s", execution.Start) - workflowExecution.Start = execution.Start - - found := false - for _, action := range workflow.Actions { - if action.ID == execution.Start { - found = true - break - } - } - - if !found { - log.Printf("[ERROR] Action %s was NOT found! Exiting execution.", execution.Start) - return shuffle.WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", workflow.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", workflow.Start)) - } - } else if len(execution.Start) > 0 { - //log.Printf("[INFO] !") - //log.Printf("[ERROR] START ACTION %s IS WRONG ID LENGTH %d!", execution.Start, len(execution.Start)) - //return shuffle.WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", execution.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", execution.Start)) - } - - if len(execution.ExecutionId) == 36 { - workflowExecution.ExecutionId = execution.ExecutionId - } else { - sessionToken := uuid.NewV4() - workflowExecution.ExecutionId = sessionToken.String() - } - } else { - // Check for parameters of start and ExecutionId - // This is mostly used for user input trigger - - answer, answerok := request.URL.Query()["answer"] - referenceId, referenceok := request.URL.Query()["reference_execution"] - if answerok && referenceok { - // If answer is false, reference execution with result - log.Printf("[INFO] Answer is OK AND reference is OK!") - if answer[0] == "false" { - log.Printf("Should update reference and return, no need for further execution!") - - // Get the reference execution - oldExecution, err := shuffle.GetWorkflowExecution(ctx, referenceId[0]) - if err != nil { - log.Printf("Failed getting execution (execution) %s: %s", referenceId[0], err) - return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed getting execution ID %s because it doesn't exist.", referenceId[0]), err - } - - if oldExecution.Workflow.ID != id { - log.Println("Wrong workflowid!") - return shuffle.WorkflowExecution{}, fmt.Sprintf("Bad ID %s", referenceId), errors.New("Bad ID") - } - - newResults := []shuffle.ActionResult{} - //log.Printf("%#v", oldExecution.Results) - for _, result := range oldExecution.Results { - log.Printf("%s - %s", result.Action.ID, start[0]) - if result.Action.ID == start[0] { - note, noteok := request.URL.Query()["note"] - if noteok { - result.Result = fmt.Sprintf("User note: %s", note[0]) - } else { - result.Result = fmt.Sprintf("User clicked %s", answer[0]) - } - - // Stopping the whole thing - result.CompletedAt = int64(time.Now().Unix()) - result.Status = "ABORTED" - oldExecution.Status = result.Status - oldExecution.Result = result.Result - oldExecution.LastNode = result.Action.ID - } - - newResults = append(newResults, result) - } - - oldExecution.Results = newResults - err = shuffle.SetWorkflowExecution(ctx, *oldExecution, true) - if err != nil { - log.Printf("Error saving workflow execution actionresult setting: %s", err) - return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed setting workflowexecution actionresult in execution: %s", err), err - } - - return shuffle.WorkflowExecution{}, "", nil - } - } - - if referenceok { - log.Printf("Handling an old execution continuation!") - // Will use the old name, but still continue with NEW ID - oldExecution, err := shuffle.GetWorkflowExecution(ctx, referenceId[0]) - if err != nil { - log.Printf("Failed getting execution (execution) %s: %s", referenceId[0], err) - return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed getting execution ID %s because it doesn't exist.", referenceId[0]), err - } - - workflowExecution = *oldExecution - } - - if len(workflowExecution.ExecutionId) == 0 { - sessionToken := uuid.NewV4() - workflowExecution.ExecutionId = sessionToken.String() - } else { - log.Printf("Using the same executionId as before: %s", workflowExecution.ExecutionId) - makeNew = false - } - - // Don't override workflow defaults - } - - if startok { - //log.Printf("\n\n[INFO] Setting start to %s based on query!\n\n", start[0]) - //workflowExecution.Workflow.Start = start[0] - workflowExecution.Start = start[0] - } - - // FIXME - regex uuid, and check if already exists? - if len(workflowExecution.ExecutionId) != 36 { - log.Printf("Invalid uuid: %s", workflowExecution.ExecutionId) - return shuffle.WorkflowExecution{}, "Invalid uuid", err - } - - // FIXME - find owner of workflow - // FIXME - get the actual workflow itself and build the request - // MAYBE: Don't send the workflow within the pubsub, as this requires more data to be sent - // Check if a worker already exists for company, else run one with: - // locations, project IDs and subscription names - - // When app is executed: - // Should update with status execution (somewhere), which will trigger the next node - // IF action.type == internal, we need the internal watcher to be running and executing - // This essentially means the WORKER has to be the responsible party for new actions in the INTERNAL landscape - // Results are ALWAYS posted back to cloud@execution_id? - if makeNew { - workflowExecution.Type = "workflow" - //workflowExecution.Stream = "tmp" - //workflowExecution.WorkflowQueue = "tmp" - //workflowExecution.SubscriptionNameNodestream = "testcompany-nodestream" - //workflowExecution.Locations = []string{"europe-west2"} - workflowExecution.ProjectId = gceProject - workflowExecution.WorkflowId = workflow.ID - workflowExecution.StartedAt = int64(time.Now().Unix()) - workflowExecution.CompletedAt = 0 - workflowExecution.Authorization = uuid.NewV4().String() - - // Status for the entire workflow. - workflowExecution.Status = "EXECUTING" - } - - if len(workflowExecution.ExecutionSource) == 0 { - log.Printf("[INFO] No execution source (trigger) specified. Setting to default") - workflowExecution.ExecutionSource = "default" - } else { - log.Printf("[INFO] Execution source is %s for execution ID %s in workflow %s", workflowExecution.ExecutionSource, workflowExecution.ExecutionId, workflowExecution.Workflow.ID) - } - - workflowExecution.ExecutionVariables = workflow.ExecutionVariables - if len(workflowExecution.Start) == 0 && len(workflowExecution.Workflow.Start) > 0 { - workflowExecution.Start = workflowExecution.Workflow.Start - } - - startnodeFound := false - newStartnode := "" - for _, item := range workflowExecution.Workflow.Actions { - if item.ID == workflowExecution.Start { - startnodeFound = true - } - - if item.IsStartNode { - newStartnode = item.ID - } - } - - if !startnodeFound { - log.Printf("[INFO] Couldn't find startnode %s. Remapping to %#v", workflowExecution.Start, newStartnode) - - if len(newStartnode) > 0 { - workflowExecution.Start = newStartnode - } else { - return shuffle.WorkflowExecution{}, fmt.Sprintf("Startnode couldn't be found"), errors.New("Startnode isn't defined in this workflow..") - } - } - - childNodes := shuffle.FindChildNodes(workflowExecution, workflowExecution.Start) - - topic := "workflows" - startFound := false - // FIXME - remove this? - newActions := []shuffle.Action{} - defaultResults := []shuffle.ActionResult{} - - allAuths := []shuffle.AppAuthenticationStorage{} - for _, action := range workflowExecution.Workflow.Actions { - //action.LargeImage = "" - if action.ID == workflowExecution.Start { - startFound = true - } - //log.Println(action.Environment) - - if action.Environment == "" { - return shuffle.WorkflowExecution{}, fmt.Sprintf("Environment is not defined for %s", action.Name), errors.New("Environment not defined!") - } - - // FIXME: Authentication parameters - if len(action.AuthenticationId) > 0 { - if len(allAuths) == 0 { - allAuths, err = shuffle.GetAllWorkflowAppAuth(ctx, workflow.ExecutingOrg.Id) - if err != nil { - log.Printf("Api authentication failed in get all app auth: %s", err) - return shuffle.WorkflowExecution{}, fmt.Sprintf("Api authentication failed in get all app auth: %s", err), err - } - } - - curAuth := shuffle.AppAuthenticationStorage{Id: ""} - for _, auth := range allAuths { - if auth.Id == action.AuthenticationId { - curAuth = auth - break - } - } - - if len(curAuth.Id) == 0 { - return shuffle.WorkflowExecution{}, fmt.Sprintf("Auth ID %s doesn't exist", action.AuthenticationId), errors.New(fmt.Sprintf("Auth ID %s doesn't exist", action.AuthenticationId)) - } - - if curAuth.Encrypted { - setField := true - newFields := []shuffle.AuthenticationStore{} - for _, field := range curAuth.Fields { - parsedKey := fmt.Sprintf("%s_%d_%s_%s", curAuth.OrgId, curAuth.Created, curAuth.Label, field.Key) - newValue, err := shuffle.HandleKeyDecryption(field.Value, parsedKey) - if err != nil { - log.Printf("[WARNING] Failed decryption for %s: %s", field.Key, err) - setField = false - break - } - - field.Value = newValue - newFields = append(newFields, field) - } - - if setField { - curAuth.Fields = newFields - } - } else { - log.Printf("[INFO] AUTH IS NOT ENCRYPTED - attempting encrypting!") - err = shuffle.SetWorkflowAppAuthDatastore(ctx, curAuth, curAuth.Id) - if err != nil { - log.Printf("[WARNING] Failed running encryption during execution: %s", err) - } - } - - newParams := []shuffle.WorkflowAppActionParameter{} - if strings.ToLower(curAuth.Type) == "oauth2" { - log.Printf("[DEBUG] Should replace auth parameters (Oauth2)") - - for _, param := range curAuth.Fields { - if param.Key == "expiration" { - continue - } - - newParams = append(newParams, shuffle.WorkflowAppActionParameter{ - Name: param.Key, - Value: param.Value, - }) - } - - for _, param := range action.Parameters { - //log.Printf("Param: %#v", param) - if param.Configuration { - continue - } - - newParams = append(newParams, param) - } - } else { - // Rebuild params with the right data. This is to prevent issues on the frontend - for _, param := range action.Parameters { - - for _, authparam := range curAuth.Fields { - if param.Name == authparam.Key { - param.Value = authparam.Value - //log.Printf("Name: %s - value: %s", param.Name, param.Value) - //log.Printf("Name: %s - value: %s\n", param.Name, param.Value) - break - } - } - - newParams = append(newParams, param) - } - } - - action.Parameters = newParams - } - - action.LargeImage = "" - if len(action.Label) == 0 { - action.Label = action.ID - } - //log.Printf("LABEL: %s", action.Label) - newActions = append(newActions, action) - - // If the node is NOT found, it's supposed to be set to SKIPPED, - // as it's not a childnode of the startnode - // This is a configuration item for the workflow itself. - if len(workflowExecution.Results) > 0 { - defaultResults = []shuffle.ActionResult{} - for _, result := range workflowExecution.Results { - if result.Status == "WAITING" { - result.Status = "FINISHED" - result.Result = "Continuing" - } - - defaultResults = append(defaultResults, result) - } - } else if len(workflowExecution.Results) == 0 && !workflowExecution.Workflow.Configuration.StartFromTop { - found := false - for _, nodeId := range childNodes { - if nodeId == action.ID { - //log.Printf("Found %s", action.ID) - found = true - } - } - - if !found { - if action.ID == workflowExecution.Start { - continue - } - - //log.Printf("[WARNING] Set %s to SKIPPED as it's NOT a childnode of the startnode.", action.ID) - curaction := shuffle.Action{ - AppName: action.AppName, - AppVersion: action.AppVersion, - Label: action.Label, - Name: action.Name, - ID: action.ID, - } - //action - //curaction.Parameters = [] - defaultResults = append(defaultResults, shuffle.ActionResult{ - Action: curaction, - ExecutionId: workflowExecution.ExecutionId, - Authorization: workflowExecution.Authorization, - Result: "Skipped because it's not under the startnode", - StartedAt: 0, - CompletedAt: 0, - Status: "SKIPPED", - }) - } - } - } - - removeTriggers := []string{} - for triggerIndex, trigger := range workflowExecution.Workflow.Triggers { - //log.Printf("[INFO] ID: %s vs %s", trigger.ID, workflowExecution.Start) - if trigger.ID == workflowExecution.Start { - if trigger.AppName == "User Input" { - startFound = true - break - } - } - - if trigger.AppName == "User Input" || trigger.AppName == "Shuffle Workflow" { - found := false - for _, node := range childNodes { - if node == trigger.ID { - found = true - break - } - } - - if !found { - //log.Printf("SHOULD SET TRIGGER %s TO BE SKIPPED", trigger.ID) - - curaction := shuffle.Action{ - AppName: "shuffle-subflow", - AppVersion: trigger.AppVersion, - Label: trigger.Label, - Name: trigger.Name, - ID: trigger.ID, - } - - defaultResults = append(defaultResults, shuffle.ActionResult{ - Action: curaction, - ExecutionId: workflowExecution.ExecutionId, - Authorization: workflowExecution.Authorization, - Result: "Skipped because it's not under the startnode", - StartedAt: 0, - CompletedAt: 0, - Status: "SKIPPED", - }) - } else { - // Replaces trigger with the subflow - //if trigger.AppName == "Shuffle Workflow" { - // replaceActions := false - // workflowAction := "" - // for _, param := range trigger.Parameters { - // if param.Name == "argument" && !strings.Contains(param.Value, ".#") { - // replaceActions = true - // } - - // if param.Name == "startnode" { - // workflowAction = param.Value - // } - // } - - // if replaceActions { - // replacementNodes, newBranches, lastnode := shuffle.GetReplacementNodes(ctx, workflowExecution, trigger, trigger.Label) - // log.Printf("REPLACEMENTS: %d, %d", len(replacementNodes), len(newBranches)) - // if len(replacementNodes) > 0 { - // for _, action := range replacementNodes { - // found := false - - // for subActionIndex, subaction := range newActions { - // if subaction.ID == action.ID { - // found = true - // //newActions[subActionIndex].Name = action.Name - // newActions[subActionIndex].Label = action.Label - // break - // } - // } - - // if !found { - // action.SubAction = true - // newActions = append(newActions, action) - // } - - // // Check if it's already set to have a value - // for resultIndex, result := range defaultResults { - // if result.Action.ID == action.ID { - // defaultResults = append(defaultResults[:resultIndex], defaultResults[resultIndex+1:]...) - // break - // } - // } - // } - - // for _, branch := range newBranches { - // workflowExecution.Workflow.Branches = append(workflowExecution.Workflow.Branches, branch) - // } - - // // Append branches: - // // parent -> new inner node (FIRST one) - // for branchIndex, branch := range workflowExecution.Workflow.Branches { - // if branch.DestinationID == trigger.ID { - // log.Printf("REPLACE DESTINATION WITH %s!!", workflowAction) - // workflowExecution.Workflow.Branches[branchIndex].DestinationID = workflowAction - // } - - // if branch.SourceID == trigger.ID { - // log.Printf("REPLACE SOURCE WITH LASTNODE %s!!", lastnode) - // workflowExecution.Workflow.Branches[branchIndex].SourceID = lastnode - // } - // } - - // // Remove the trigger - // removeTriggers = append(removeTriggers, workflowExecution.Workflow.Triggers[triggerIndex].ID) - // } - - // log.Printf("NEW ACTION LENGTH %d, RESULT: %d, Triggers: %d, BRANCHES: %d", len(newActions), len(defaultResults), len(workflowExecution.Workflow.Triggers), len(workflowExecution.Workflow.Branches)) - // } - //} - _ = triggerIndex - } - } - } - - //newTriggers := []shuffle.Trigger{} - //for _, trigger := range workflowExecution.Workflow.Triggers { - // found := false - // for _, triggerId := range removeTriggers { - // if trigger.ID == triggerId { - // found = true - // break - // } - // } - - // if found { - // log.Printf("[WARNING] Removed trigger %s during execution", trigger.ID) - // continue - // } - - // newTriggers = append(newTriggers, trigger) - //} - //workflowExecution.Workflow.Triggers = newTriggers - _ = removeTriggers - - if !startFound { - if len(workflowExecution.Start) == 0 && len(workflowExecution.Workflow.Start) > 0 { - workflowExecution.Start = workflow.Start - } else if len(workflowExecution.Workflow.Actions) > 0 { - workflowExecution.Start = workflowExecution.Workflow.Actions[0].ID - } else { - log.Printf("[ERROR] Startnode %s doesn't exist!!", workflowExecution.Start) - return shuffle.WorkflowExecution{}, fmt.Sprintf("Workflow action %s doesn't exist in workflow", workflowExecution.Start), errors.New(fmt.Sprintf(`Workflow start node "%s" doesn't exist. Exiting!`, workflowExecution.Start)) - } - } - - //log.Printf("EXECUTION START: %s", workflowExecution.Start) - - // Verification for execution environments - workflowExecution.Results = defaultResults - workflowExecution.Workflow.Actions = newActions - onpremExecution := true - environments := []string{} - - if len(workflowExecution.ExecutionOrg) == 0 && len(workflow.ExecutingOrg.Id) > 0 { - workflowExecution.ExecutionOrg = workflow.ExecutingOrg.Id - } - - var allEnvs []shuffle.Environment - if len(workflowExecution.ExecutionOrg) > 0 { - //log.Printf("[INFO] Executing ORG: %s", workflowExecution.ExecutionOrg) - - allEnvironments, err := shuffle.GetEnvironments(ctx, workflowExecution.ExecutionOrg) - if err != nil { - log.Printf("Failed finding environments: %s", err) - return shuffle.WorkflowExecution{}, fmt.Sprintf("Workflow environments not found for this org"), errors.New(fmt.Sprintf("Workflow environments not found for this org")) - } - - for _, curenv := range allEnvironments { - if curenv.Archived { - continue - } - - allEnvs = append(allEnvs, curenv) - } - } else { - log.Printf("[ERROR] No org identified for execution of %s. Returning", workflowExecution.Workflow.ID) - return shuffle.WorkflowExecution{}, "No org identified for execution", errors.New("No org identified for execution") - } - - if len(allEnvs) == 0 { - log.Printf("[ERROR] No active environments found for org: %s", workflowExecution.ExecutionOrg) - return shuffle.WorkflowExecution{}, "No active environments found", errors.New(fmt.Sprintf("No active env found for org %s", workflowExecution.ExecutionOrg)) - } - - // Check if the actions are children of the startnode? - imageNames := []string{} - cloudExec := false - for _, action := range workflowExecution.Workflow.Actions { - // Verify if the action environment exists and append - found := false - for _, env := range allEnvs { - if env.Name == action.Environment { - found = true - - if env.Type == "cloud" { - cloudExec = true - } else if env.Type == "onprem" { - onpremExecution = true - } else { - log.Printf("[ERROR] No handler for environment type %s", env.Type) - return shuffle.WorkflowExecution{}, "No active environments found", errors.New(fmt.Sprintf("No handler for environment type %s", env.Type)) - } - break - } - } - - if !found { - log.Printf("[ERROR] Couldn't find environment %s. Maybe it's inactive?", action.Environment) - return shuffle.WorkflowExecution{}, "Couldn't find the environment", errors.New(fmt.Sprintf("Couldn't find env %s in org %s", action.Environment, workflowExecution.ExecutionOrg)) - } - - found = false - for _, env := range environments { - if env == action.Environment { - - found = true - break - } - } - - // Check if the app exists? - newName := action.AppName - newName = strings.ReplaceAll(newName, " ", "-") - imageNames = append(imageNames, fmt.Sprintf("%s:%s_%s", baseDockerName, newName, action.AppVersion)) - - if !found { - environments = append(environments, action.Environment) - } - } - - err = imageCheckBuilder(imageNames) - if err != nil { - log.Printf("[ERROR] Failed building the required images from %#v: %s", imageNames, err) + log.Printf("[ERROR] Failed building the required images from %#v: %s", execInfo.ImageNames, err) return shuffle.WorkflowExecution{}, "Failed building missing Docker images", err } - //b, err := json.Marshal(workflowExecution) - //if err == nil { - // log.Printf("LEN: %d", len(string(b))) - // //workflowExecution.ExecutionOrg.SyncFeatures = Org{} - //} - - workflowExecution.Workflow.ExecutingOrg = shuffle.OrgMini{ - Id: workflowExecution.Workflow.ExecutingOrg.Id, - } - workflowExecution.Workflow.Org = []shuffle.OrgMini{ - workflowExecution.Workflow.ExecutingOrg, - } - //Org []Org `json:"org,omitempty" datastore:"org"` err = shuffle.SetWorkflowExecution(ctx, workflowExecution, true) if err != nil { - log.Printf("[WARNING] Error saving workflow execution for updates %s: %s", topic, err) + log.Printf("[WARNING] Error saving workflow execution for updates %s", err) return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed setting workflowexecution: %s", err), err } // Adds queue for onprem execution // FIXME - add specifics to executionRequest, e.g. specific environment (can run multi onprem) - if onpremExecution { + if execInfo.OnpremExecution { // FIXME - tmp name based on future companyname-companyId // This leads to issues with overlaps. Should set limits and such instead - for _, environment := range environments { + for _, environment := range execInfo.Environments { log.Printf("[INFO] Execution: %s should execute onprem with execution environment \"%s\". Workflow: %s", workflowExecution.ExecutionId, environment, workflowExecution.Workflow.ID) executionRequest := shuffle.ExecutionRequest{ ExecutionId: workflowExecution.ExecutionId, WorkflowId: workflowExecution.Workflow.ID, Authorization: workflowExecution.Authorization, - Environments: environments, + Environments: execInfo.Environments, } //executionRequestWrapper, err := getWorkflowQueue(ctx, environment) @@ -1972,7 +1283,7 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request } // Verifies and runs cloud executions - if cloudExec { + if execInfo.CloudExec { featuresList, err := handleVerifyCloudsync(workflowExecution.ExecutionOrg) if !featuresList.Workflows.Active || err != nil { log.Printf("Error: %s", err) diff --git a/frontend/src/components/Oauth2Auth.jsx b/frontend/src/components/Oauth2Auth.jsx index 4fd89ac7..6335f47c 100644 --- a/frontend/src/components/Oauth2Auth.jsx +++ b/frontend/src/components/Oauth2Auth.jsx @@ -53,10 +53,12 @@ const AuthenticationOauth2 = (props) => { var resources = "" if (scopes !== undefined && scopes !== null & scopes.length > 0) { + //scopes.push("offline_access") resources = scopes.join(",") } const authentication_url = authenticationType.token_uri + console.log("AUTH: ", authenticationType) console.log("SCOPES2: ", resources) const redirectUri = `${window.location.protocol}//${window.location.host}/set_authentication` @@ -65,7 +67,14 @@ const AuthenticationOauth2 = (props) => { state += `%26oauth_url%3d${oauth_url}` console.log("ADDING OAUTH2 URL: ", state) } - const url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&scope=${resources}&prompt=consent&state=${state}` + + if (authenticationType.refresh_uri !== undefined && authenticationType.refresh_uri !== null && authenticationType.refresh_uri.length > 0) { + state += `%26refresh_uri%3d${authenticationType.refresh_uri}` + } else { + state += `%26refresh_uri%3d${authentication_url}` + } + + const url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&scope=${resources}&prompt=consent&state=${state}&access_type=offline` //const url = `https://accounts.zoho.com/oauth/v2/auth?response_type=code&client_id=${client_id}&scope=AaaServer.profile.Read&redirect_uri=${redirectUri}&prompt=consent` console.log("Full URI: ", url) diff --git a/frontend/src/defaultCytoscapeStyle.js b/frontend/src/defaultCytoscapeStyle.js index d836604e..c0c2c163 100644 --- a/frontend/src/defaultCytoscapeStyle.js +++ b/frontend/src/defaultCytoscapeStyle.js @@ -101,6 +101,7 @@ const data = [{ selector: `node[type="TRIGGER"]`, css: { 'shape': 'octagon', + 'border-radius': '5px', 'border-color': 'orange', 'background-color': '#213243', 'background-width': '100%', @@ -203,6 +204,16 @@ const data = [{ 'transition-duration': '0.5s', }, }, + { + selector: '.hover-highlight', + css: { + 'background-color': '#5f9265', + 'border-color': '#5f9265', + 'border-width': '5px', + 'transition-property': 'background-color', + 'transition-duration': '0.5s', + }, + }, { selector: '.failure-highlight', css: { diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 8dbea6d3..72ca424a 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -11,7 +11,7 @@ import NestedMenuItem from "material-ui-nested-menu-item"; import ReactMarkdown from 'react-markdown'; -import {TextField, Drawer, Button, Paper, Grid, Tabs, InputAdornment, Tab, ButtonBase, Tooltip, Select, MenuItem, Divider, Dialog, Modal, DialogActions, DialogTitle, InputLabel, DialogContent, FormControl, IconButton, Menu, Input, FormGroup, FormControlLabel, Typography, Checkbox, Breadcrumbs, CircularProgress, Switch, Fade} from '@material-ui/core'; +import {Slide, TextField, Drawer, Button, Paper, Grid, Tabs, InputAdornment, Tab, ButtonBase, Tooltip, Select, MenuItem, Divider, Dialog, Modal, DialogActions, DialogTitle, InputLabel, DialogContent, FormControl, IconButton, Menu, Input, FormGroup, FormControlLabel, Typography, Checkbox, Breadcrumbs, CircularProgress, Switch, Fade} from '@material-ui/core'; import {OpenInNew as OpenInNewIcon,Undo as UndoIcon, FileCopy as FileCopyIcon, GetApp as GetAppIcon, Search as SearchIcon, ArrowUpward as ArrowUpwardIcon, Visibility as VisibilityIcon, Done as DoneIcon, Close as CloseIcon, Error as ErrorIcon, FindReplace as FindreplaceIcon, ArrowLeft as ArrowLeftIcon, Cached as CachedIcon, DirectionsRun as DirectionsRunIcon, Add as AddIcon, Polymer as PolymerIcon, FormatListNumbered as FormatListNumberedIcon, Create as CreateIcon, PlayArrow as PlayArrowIcon, AspectRatio as AspectRatioIcon, MoreVert as MoreVertIcon, Apps as AppsIcon, Schedule as ScheduleIcon, FavoriteBorder as FavoriteBorderIcon, Pause as PauseIcon, Delete as DeleteIcon, AddCircleOutline as AddCircleOutlineIcon, Save as SaveIcon, KeyboardArrowLeft as KeyboardArrowLeftIcon, KeyboardArrowRight as KeyboardArrowRightIcon, ArrowBack as ArrowBackIcon, Settings as SettingsIcon, LockOpen as LockOpenIcon, ExpandMore as ExpandMoreIcon, VpnKey as VpnKeyIcon} from '@material-ui/icons'; import * as cytoscape from 'cytoscape'; @@ -1260,9 +1260,9 @@ const AngularWorkflow = (props) => { selectedAction.selectedAuthentication = item for (var key in workflow.actions) { - console.log(workflow.actions[key].app_name) + //console.log(workflow.actions[key].app_name) if (workflow.actions[key].app_name == selectedApp.name) { - console.log("Setting auth at: ", workflow.actions[key], item.id) + //console.log("Setting auth at: ", workflow.actions[key], item.id) workflow.actions[key].selectedAuthentication = item workflow.actions[key].authentication_id = item.id appUpdates = true @@ -3306,6 +3306,16 @@ const AngularWorkflow = (props) => { }, { duration: animationDuration, }) + + const outgoingEdges = event.target.outgoers('edge') + const incomingEdges = event.target.incomers('edge') + if (outgoingEdges.length > 0) { + outgoingEdges.removeClass('hover-highlight') + } + + if (incomingEdges.length > 0) { + outgoingEdges.removeClass('hover-highlight') + } } const buttonColor = "rgba(255,255,255,0.9)" @@ -3481,6 +3491,16 @@ const AngularWorkflow = (props) => { }) previousnodecolor = event.target.style("border-color") + + const outgoingEdges = event.target.outgoers('edge') + const incomingEdges = event.target.incomers('edge') + if (outgoingEdges.length > 0) { + outgoingEdges.addClass('hover-highlight') + } + + if (incomingEdges.length > 0) { + outgoingEdges.addClass('hover-highlight') + } } const onEdgeHoverOut = (event) => { @@ -4668,9 +4688,9 @@ const AngularWorkflow = (props) => { :
- + - Loading apps + Loading Apps
} @@ -7800,46 +7820,46 @@ const AngularWorkflow = (props) => { } return ( -
- + -
+ setVariablesModalOpen={setVariablesModalOpen} + setLastSaved={setLastSaved} + setCodeModalOpen={setCodeModalOpen} + selectedNameChange={selectedNameChange} + rightsidebarStyle={rightsidebarStyle} + showEnvironment={showEnvironment} + selectedActionEnvironment={selectedActionEnvironment} + environments={environments} + setNewSelectedAction={setNewSelectedAction} + sortByKey={sortByKey} + + appApiViewStyle={appApiViewStyle} + globalUrl={globalUrl} + setSelectedActionEnvironment={setSelectedActionEnvironment} + requiresAuthentication={requiresAuthentication} + /> + ) } else if (Object.getOwnPropertyNames(selectedTrigger).length > 0) { @@ -8733,60 +8753,70 @@ const AngularWorkflow = (props) => {
{leftView} - { - // FIXME: There's something specific loading when - // you do the first hover of a node. Why is this different? - //console.log("CY: ", incy) - setCy(incy) - }} - /> + {workflow.id === undefined || workflow.id === null || apps.length === 0 ? +
+ + + Loading Workflow + +
+ : + { + // FIXME: There's something specific loading when + // you do the first hover of a node. Why is this different? + //console.log("CY: ", incy) + setCy(incy) + }} + /> + }
{executionModal} - */} + + setVariablesModalOpen={setVariablesModalOpen} + setLastSaved={setLastSaved} + setCodeModalOpen={setCodeModalOpen} + selectedNameChange={selectedNameChange} + rightsidebarStyle={rightsidebarStyle} + showEnvironment={showEnvironment} + selectedActionEnvironment={selectedActionEnvironment} + environments={environments} + setNewSelectedAction={setNewSelectedAction} + sortByKey={sortByKey} + + appApiViewStyle={appApiViewStyle} + globalUrl={globalUrl} + setSelectedActionEnvironment={setSelectedActionEnvironment} + requiresAuthentication={requiresAuthentication} + />
diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 03142daa..ec694c2b 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -1073,10 +1073,10 @@ const Apps = (props) => { : - + No apps have been created, uploaded or downloaded yet. Click "Load existing apps" above to get the baseline. This may take a while as its building docker images. - + If you're still not able to see any apps, please follow our troubleshooting guide for loading apps! diff --git a/frontend/src/views/SetAuthentication.jsx b/frontend/src/views/SetAuthentication.jsx index 17849a28..5d1ca617 100644 --- a/frontend/src/views/SetAuthentication.jsx +++ b/frontend/src/views/SetAuthentication.jsx @@ -95,6 +95,14 @@ const SetAuthentication = (props) => { if (query[0] === "oauth_url") { appAuthData.fields.push({"key": "oauth_url", "value": query[1]}) } + + if (query[0] === "refresh_uri") { + appAuthData.fields.push({"key": "refresh_uri", "value": query[1]}) + } + + if (query[0] === "refresh_url") { + appAuthData.fields.push({"key": "refresh_url", "value": query[1]}) + } } } diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 2a4cbd8b..6ee09494 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -1555,7 +1555,7 @@ const Workflows = (props) => { let workflowData = ""; if (workflows.length > 0) { const columns = [ - { field: 'image', headerName: 'Logo', width: 42, renderCell: (params) => { + { field: 'image', headerName: 'Logo', width: 50, sortable: false, renderCell: (params) => { const data = params.row.record var boxColor = "#FECC00" From 7144c911d735db1bab9b9fa9e9feb0dfedab4350 Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 22 Oct 2021 12:05:59 +0200 Subject: [PATCH 07/18] Added BASE_URL to .env --- .env | 1 + docker-compose.yml | 2 +- frontend/src/components/Oauth2Auth.jsx | 9 ++++----- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.env b/.env index 7e073bd9..5a78c4d8 100644 --- a/.env +++ b/.env @@ -30,6 +30,7 @@ SHUFFLE_FILE_LOCATION=./shuffle-files SHUFFLE_ENCRYPTION_MODIFIER= # Other configs +BASE_URL=http://shuffle-backend:5001 BACKEND_HOSTNAME=shuffle-backend BACKEND_PORT=5001 FRONTEND_PORT=3001 diff --git a/docker-compose.yml b/docker-compose.yml index eeefca1b..0b4bd8fa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: frontend: - build: ./frontend + #build: ./frontend image: ghcr.io/frikky/shuffle-frontend:nightly container_name: shuffle-frontend hostname: shuffle-frontend diff --git a/frontend/src/components/Oauth2Auth.jsx b/frontend/src/components/Oauth2Auth.jsx index 6335f47c..2d9afa2d 100644 --- a/frontend/src/components/Oauth2Auth.jsx +++ b/frontend/src/components/Oauth2Auth.jsx @@ -58,9 +58,8 @@ const AuthenticationOauth2 = (props) => { } const authentication_url = authenticationType.token_uri - console.log("AUTH: ", authenticationType) - - console.log("SCOPES2: ", resources) + //console.log("AUTH: ", authenticationType) + //console.log("SCOPES2: ", resources) const redirectUri = `${window.location.protocol}//${window.location.host}/set_authentication` var state = `workflow_id%3D${workflow.id}%26reference_action_id%3d${selectedAction.app_id}%26app_name%3d${selectedAction.app_name}%26app_id%3d${selectedAction.app_id}%26app_version%3d${selectedAction.app_version}%26authentication_url%3d${authentication_url}%26scope%3d${resources}%26client_id%3d${client_id}%26client_secret%3d${client_secret}` if (oauth_url !== undefined && oauth_url !== null && oauth_url.length > 0) { @@ -77,8 +76,8 @@ const AuthenticationOauth2 = (props) => { const url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&scope=${resources}&prompt=consent&state=${state}&access_type=offline` //const url = `https://accounts.zoho.com/oauth/v2/auth?response_type=code&client_id=${client_id}&scope=AaaServer.profile.Read&redirect_uri=${redirectUri}&prompt=consent` - console.log("Full URI: ", url) - console.log("Redirect Uri: ", redirectUri) + //console.log("Full URI: ", url) + //console.log("Redirect Uri: ", redirectUri) // &resource=https%3A%2F%2Fgraph.microsoft.com& // FIXME: Awful, but works for prototyping From a506b6e124ea811bb4265867316718f1edd3d433 Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 22 Oct 2021 13:59:31 +0200 Subject: [PATCH 08/18] #512: Fixed frontend mechanism for unselecting router conditions --- backend/go-app/go.mod | 4 ++-- backend/go-app/walkoff.go | 14 +++++++------- docker-compose.yml | 2 +- frontend/src/views/AngularWorkflow.jsx | 13 +++++++++++++ 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index aecda8fb..bd9760cb 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -2,7 +2,7 @@ module main go 1.15 -replace github.com/shuffle/shuffle-shared => ../../../../git/shuffle-shared +//replace github.com/shuffle/shuffle-shared => ../../../../git/shuffle-shared //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi //replace github.com/frikky/go-elasticsearch => ../../../../git/go-elasticsearch @@ -21,7 +21,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/h2non/filetype v1.1.1 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.1.17 + github.com/shuffle/shuffle-shared v0.1.18 go4.org v0.0.0-20201209231011-d4a079459e60 // indirect golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 google.golang.org/api v0.58.0 diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 56415757..b72fcce9 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -849,7 +849,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { var trigger shuffle.Trigger err = json.Unmarshal([]byte(actionResult.Result), &trigger) if err != nil { - log.Printf("Failed unmarshaling actionresult for user input: %s", err) + log.Printf("[WARNING] Failed unmarshaling actionresult for user input: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -862,21 +862,21 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { err := handleUserInput(trigger, orgId, workflowExecution.Workflow.ID, workflowExecution.ExecutionId) if err != nil { - log.Printf("Failed userinput handler: %s", err) + log.Printf("[WARNING] Failed userinput handler: %s", err) actionResult.Result = fmt.Sprintf("Cloud error: %s", err) workflowExecution.Results = append(workflowExecution.Results, actionResult) workflowExecution.Status = "ABORTED" err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, true) if err != nil { - log.Printf("Failed to set execution during wait") + log.Printf("[WARNING] Failed to set execution during wait: %s", err) } else { - log.Printf("Successfully set the execution to waiting.") + log.Printf("[INFO] Successfully set the execution %s to waiting.", workflowExecution.ExecutionId) } resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error: %s"}`, err))) } else { - log.Printf("Successful userinput handler") + log.Printf("[INFO] Successful userinput handler") resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "CLOUD IS DONE"}`))) @@ -886,7 +886,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { workflowExecution.Status = actionResult.Status err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, true) if err != nil { - log.Printf("Failed ") + log.Printf("[WARNING] Failed setting userinput: %s", err) } else { log.Printf("Successfully set the execution to waiting.") } @@ -1995,7 +1995,7 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { // Double unmarshal because of user apps newbody, err := json.Marshal(newapps) if err != nil { - log.Printf("Failed unmarshalling all newapps: %s", err) + log.Printf("[ERROR] Failed unmarshalling all newapps: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking workflow apps"}`))) return diff --git a/docker-compose.yml b/docker-compose.yml index 0b4bd8fa..4205bb18 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,7 @@ services: depends_on: - backend backend: - build: ./backend + #build: ./backend image: ghcr.io/frikky/shuffle-backend:nightly container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 72ca424a..9f3e69d3 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1595,6 +1595,19 @@ const AngularWorkflow = (props) => { if (event.target.data().decorator) { alert.info("This edge can't be edited.") } else { + console.log("DATA: ", event.target.data()) + const destinationId = event.target.data("target") + console.log("DATA: ", event.target.data()) + const curaction = workflow.actions.find(a => a.id === destinationId) + console.log("ACTION: ", curaction) + if (curaction !== undefined && curaction !== null) { + if (curaction.app_name == "Shuffle Tools" && curaction.name === "router") { + alert.info("Router action can't have incoming conditions") + event.target.unselect() + return + } + } + setSelectedEdgeIndex(workflow.branches.findIndex(data => data.id === event.target.data()["id"])) setSelectedEdge(event.target.data()) } From 42040cb6bd07737b165ec1c8f2466efc1c974a51 Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 22 Oct 2021 18:41:52 +0200 Subject: [PATCH 09/18] Fixed shuffle-shared version in worker and orborus and bumped versions --- backend/go-app/go.mod | 2 +- frontend/src/components/OrgHeader.js | 33 ++- frontend/src/views/AngularWorkflow.jsx | 12 +- frontend/src/views/Workflows.jsx | 4 +- functions/onprem/orborus/build.sh | 2 +- functions/onprem/orborus/go.mod | 37 +-- functions/onprem/orborus/go.sum | 266 +------------------ functions/onprem/orborus/orborus.go | 2 +- functions/onprem/worker/Dockerfile | 3 +- functions/onprem/worker/build.sh | 2 +- functions/onprem/worker/go.mod | 38 +-- functions/onprem/worker/go.sum | 346 +------------------------ functions/onprem/worker/worker.go | 4 +- 13 files changed, 72 insertions(+), 679 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index bd9760cb..c175d626 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -2,7 +2,7 @@ module main go 1.15 -//replace github.com/shuffle/shuffle-shared => ../../../../git/shuffle-shared +replace github.com/shuffle/shuffle-shared => ../../../../git/shuffle-shared //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi //replace github.com/frikky/go-elasticsearch => ../../../../git/go-elasticsearch diff --git a/frontend/src/components/OrgHeader.js b/frontend/src/components/OrgHeader.js index 65dd6e97..8730ed59 100644 --- a/frontend/src/components/OrgHeader.js +++ b/frontend/src/components/OrgHeader.js @@ -38,6 +38,7 @@ const OrgHeader = (props) => { const [workflowDownloadBranch, setWorkflowDownloadBranch] = React.useState(selectedOrganization.defaults === undefined ? defaultBranch : selectedOrganization.defaults.workflow_download_branch === undefined || selectedOrganization.defaults.workflow_download_branch.length === 0 ? defaultBranch : selectedOrganization.defaults.workflow_download_branch) const [ssoEntrypoint, setSsoEntrypoint] = React.useState(selectedOrganization.sso_config === undefined ? "" : selectedOrganization.sso_config.sso_entrypoint === undefined || selectedOrganization.sso_config.sso_entrypoint.length === 0 ? "" : selectedOrganization.sso_config.sso_entrypoint) const [ssoCertificate, setSsoCertificate] = React.useState(selectedOrganization.sso_config === undefined ? "" : selectedOrganization.sso_config.sso_certificate === undefined || selectedOrganization.sso_config.sso_certificate.length === 0 ? "" : selectedOrganization.sso_config.sso_certificate) + const [notificationWorkflow, setNotificationWorkflow] = React.useState(selectedOrganization.defaults === undefined ? "" : selectedOrganization.defaults.notification_workflow === undefined || selectedOrganization.defaults.notification_workflow.length === 0 ? "" : selectedOrganization.defaults.notification_workflow) const [file, setFile] = React.useState("") const [fileBase64, setFileBase64] = React.useState(selectedOrganization.image) @@ -113,7 +114,7 @@ const OrgHeader = (props) => { setFile(fileObject) } - console.log("USER: ", userdata) + //console.log("USER: ", userdata) const orgSaveButton = const outlookButton = selectedTrigger.name !== "Office365" ? null : From 9ff574623ec93f347bc3a2b06ebe8cc772af022b Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 23 Oct 2021 16:59:08 +0200 Subject: [PATCH 12/18] Added more info and control to settings page --- frontend/src/components/Header.js | 6 +- frontend/src/views/AngularWorkflow.jsx | 4 +- frontend/src/views/SettingsPage.jsx | 145 +++++++++++++------------ frontend/src/views/Workflows.jsx | 23 ++-- 4 files changed, 96 insertions(+), 82 deletions(-) diff --git a/frontend/src/components/Header.js b/frontend/src/components/Header.js index 449c057b..113ba374 100644 --- a/frontend/src/components/Header.js +++ b/frontend/src/components/Header.js @@ -276,7 +276,7 @@ const Header = props => { setAnchorEl(event.currentTarget); }}> - + { { setAnchorElAvatar(event.currentTarget); }}> - + { {notificationMenu} {userdata === undefined || userdata.admin === undefined || userdata.admin === null || !userdata.admin ? null : - diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 6d5f8a22..7267bf20 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -6091,12 +6091,12 @@ const AngularWorkflow = (props) => {
- - {/* -

Settings

-
- setUsername(e.target.value)} - /> -
-
- setFirstname(e.target.value)} - /> - setLastname(e.target.value)} - /> -
-
{ } return obj; - }); + }) workflowData = {
{view === "grid" ? - - - {filteredWorkflows.map((data, index) => { - return ( - - ) - })} - + +
+ +
+ + + {filteredWorkflows.map((data, index) => { + return ( + + ) + })} + + : } From e321079385c8012666a2fb723d2a58e83a0bd5cd Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 23 Oct 2021 20:01:31 +0200 Subject: [PATCH 13/18] Added filter by app system to workflow UI --- frontend/src/views/Workflows.jsx | 81 +++++++++++++++++++++++++------- 1 file changed, 64 insertions(+), 17 deletions(-) diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index f2a530af..7bf7df06 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -3,7 +3,7 @@ import { useInterval } from 'react-powerhooks'; import { makeStyles } from '@material-ui/core/styles'; import { useTheme } from '@material-ui/core/styles'; -import {Avatar, Grid, Paper, Tooltip, Divider, Button, TextField, FormControl, IconButton, Menu, MenuItem, FormControlLabel, Chip, Switch, Typography, Zoom, CircularProgress, Dialog, DialogTitle, DialogActions, DialogContent} from '@material-ui/core'; +import {Badge, Avatar, Grid, Paper, Tooltip, Divider, Button, TextField, FormControl, IconButton, Menu, MenuItem, FormControlLabel, Chip, Switch, Typography, Zoom, CircularProgress, Dialog, DialogTitle, DialogActions, DialogContent} from '@material-ui/core'; import {GridOn as GridOnIcon, List as ListIcon, Close as CloseIcon, Compare as CompareIcon, Maximize as MaximizeIcon, Minimize as MinimizeIcon, AddCircle as AddCircleIcon, Toc as TocIcon, Send as SendIcon, Search as SearchIcon, FileCopy as FileCopyIcon, Delete as DeleteIcon, BubbleChart as BubbleChartIcon, Restore as RestoreIcon, Cached as CachedIcon, GetApp as GetAppIcon, Apps as AppsIcon, Edit as EditIcon, MoreVert as MoreVertIcon, PlayArrow as PlayArrowIcon, Add as AddIcon, Publish as PublishIcon, CloudUpload as CloudUploadIcon, CloudDownload as CloudDownloadIcon} from '@material-ui/icons'; import NestedMenuItem from "material-ui-nested-menu-item"; //import {Search as SearchIcon, ArrowUpward as ArrowUpwardIcon, Visibility as VisibilityIcon, Done as DoneIcon, Close as CloseIcon, Error as ErrorIcon, FindReplace as FindreplaceIcon, ArrowLeft as ArrowLeftIcon, Cached as CachedIcon, DirectionsRun as DirectionsRunIcon, Add as AddIcon, Polymer as PolymerIcon, FormatListNumbered as FormatListNumberedIcon, Create as CreateIcon, PlayArrow as PlayArrowIcon, AspectRatio as AspectRatioIcon, MoreVert as MoreVertIcon, Apps as AppsIcon, Schedule as ScheduleIcon, FavoriteBorder as FavoriteBorderIcon, Pause as PauseIcon, Delete as DeleteIcon, AddCircleOutline as AddCircleOutlineIcon, Save as SaveIcon, KeyboardArrowLeft as KeyboardArrowLeftIcon, KeyboardArrowRight as KeyboardArrowRightIcon, ArrowBack as ArrowBackIcon, Settings as SettingsIcon, LockOpen as LockOpenIcon, ExpandMore as ExpandMoreIcon, VpnKey as VpnKeyIcon} from '@material-ui/icons'; @@ -333,6 +333,7 @@ const Workflows = (props) => { const theme = useTheme(); const alert = useAlert() const classes = useStyles(theme); + const imgSize = 60 const referenceUrl = globalUrl+"/api/v1/hooks/" @@ -376,6 +377,7 @@ const Workflows = (props) => { const [view, setView] = React.useState("grid") const [filters, setFilters] = React.useState([]) const [submitLoading, setSubmitLoading] = React.useState(false) + const [actionImageList, setActionImageList] = React.useState([]) const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" @@ -394,6 +396,7 @@ const Workflows = (props) => { found = filters.map(filter => curWorkflow.name.toLowerCase().includes(filter)) } else { found = filters.map(filter => { + const newfilter = filter.toLowerCase() if (filter === undefined) { return false } @@ -407,10 +410,9 @@ const Workflows = (props) => { } else if (curWorkflow.org_id === filter) { return true } else if (curWorkflow.actions !== null && curWorkflow.actions !== undefined) { - const newfilter = filter.toLowerCase() for (var key in curWorkflow.actions) { const action = curWorkflow.actions[key] - if (action.app_name.toLowerCase().includes(newfilter)) { + if (action.app_name.toLowerCase() === newfilter || action.app_name.toLowerCase().includes(newfilter)) { return true } } @@ -687,6 +689,27 @@ const Workflows = (props) => { if (responseJson !== undefined) { setWorkflows(responseJson) + + if (responseJson !== undefined) { + var actionnamelist = [] + var parsedactionlist = [] + for (var key in responseJson) { + for (var actionkey in responseJson[key].actions) { + const action = responseJson[key].actions[actionkey] + console.log("Action: ", action) + if (actionnamelist.includes(action.app_name)) { + continue + } + + actionnamelist.push(action.app_name) + parsedactionlist.push(action) + } + } + + console.log(parsedactionlist) + setActionImageList(parsedactionlist) + } + setFilteredWorkflows(responseJson) setWorkflowDone(true) } else { @@ -758,6 +781,7 @@ const Workflows = (props) => { display: "flex", flexWrap: 'wrap', alignContent: "space-between", + marginTop: 5, } const paperAppStyle = { @@ -1761,7 +1785,7 @@ const Workflows = (props) => { } ]; let rows = []; - rows = workflows.map((data, index) => { + rows = filteredWorkflows.map((data, index) => { let obj = { "id":index+1, "title":data.name, @@ -2112,20 +2136,43 @@ const Workflows = (props) => {
+ {actionImageList !== undefined && actionImageList !== null && actionImageList.length > 0 ? +
+ {actionImageList.map((data, index) => { + if (data.large_image === undefined || data.large_image === null || data.large_image.length === 0) { + return null + } + + if (data.app_name.toLowerCase() === "shuffle tools") { + data.large_image = theme.palette.defaultImage + } + + return ( + + { + console.log("FILTER: ", data) + addFilter(data.app_name) + }}> + + + {data.app_name} + + + + + ) + })} +
+ : null} {view === "grid" ? - -
- -
- - - {filteredWorkflows.map((data, index) => { - return ( - - ) - })} - - + + + {filteredWorkflows.map((data, index) => { + return ( + + ) + })} + : } From 9445e7092b0d15bbf54b2918559c8b3ccb8ac66a Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 24 Oct 2021 01:23:22 +0200 Subject: [PATCH 14/18] Fixed issues with worker --- frontend/src/views/Apps.jsx | 9 +++++++-- frontend/src/views/Workflows.jsx | 2 +- functions/onprem/worker/go.mod | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index ec694c2b..ad85db41 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -370,8 +370,13 @@ const Apps = (props) => { }} /> var newAppname = data.name - newAppname = newAppname.charAt(0).toUpperCase()+newAppname.substring(1) - newAppname = newAppname.replaceAll("_", " ") + if (newAppname === undefined) { + newAppname = "Undefined" + } else { + newAppname = newAppname.charAt(0).toUpperCase()+newAppname.substring(1) + newAppname = newAppname.replaceAll("_", " ") + } + var sharing = "public" if (!data.sharing) { sharing = "private" diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 7bf7df06..c2930ba3 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -1938,7 +1938,7 @@ const Workflows = (props) => { }} color="primary"> {submitLoading ? - + : "Submit" } diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index f60fa5a4..197e1404 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -8,6 +8,6 @@ require ( github.com/docker/go-connections v0.4.0 // indirect github.com/gorilla/mux v1.8.0 github.com/patrickmn/go-cache v2.1.0+incompatible - github.com/shuffle/shuffle-shared v0.1.19 + github.com/shuffle/shuffle-shared v0.1.20 go4.org v0.0.0-20201209231011-d4a079459e60 // indirect ) From 49cfb00a6c399600b6b6fe0458a6e9d123a43540 Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 25 Oct 2021 15:51:52 +0200 Subject: [PATCH 15/18] Added workflow priority search and orborus checkin IP --- backend/go-app/main.go | 2 +- backend/go-app/walkoff.go | 62 ++++++++++++++++++++++++-- frontend/src/views/Admin.jsx | 24 ++++++---- frontend/src/views/AngularWorkflow.jsx | 1 + frontend/src/views/Workflows.jsx | 8 ++-- 5 files changed, 81 insertions(+), 16 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 7bd0f90b..40904bc3 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3846,7 +3846,7 @@ func runInitEs(ctx context.Context) { //} } else { - log.Printf("[DEBUG] There are %d org(s).", len(activeOrgs)) + log.Printf("[DEBUG] Found %d org(s) in total.", len(activeOrgs)) if len(activeOrgs) == 1 { if len(activeOrgs[0].Users) == 0 { diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index b72fcce9..365017a0 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -569,7 +569,7 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque // FIXME: Add authentication? id := request.Header.Get("Org-Id") if len(id) == 0 { - log.Printf("No Org-Id header set - confirm") + log.Printf("[ERROR] No Org-Id header set - confirm") resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Specify the org-id header."}`))) return @@ -579,7 +579,7 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque ctx := context.Background() executionRequests, err := shuffle.GetWorkflowQueue(ctx, id) if err != nil { - log.Printf("(1) Failed reading body for workflowqueue: %s", err) + log.Printf("[WARNING] (1) Failed reading body for workflowqueue: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Entity parsing error - confirm"}`))) return @@ -685,11 +685,65 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { return } + env, err := shuffle.GetEnvironment(ctx, id, "") + timeNow := time.Now().Unix() + if err == nil && len(env.Id) > 0 && len(env.Name) > 0 { + if time.Now().Unix() > env.Edited+60 { + log.Printf("[DEBUG] Updating env with IP %s!", request.RemoteAddr) + env.RunningIp = request.RemoteAddr + env.Checkin = timeNow + err = shuffle.SetEnvironment(ctx, env) + if err != nil { + log.Printf("[WARNING] Failed updating environment: %s", err) + } + } + } + + // Checking and updating the environment related to the first execution if len(executionRequests.Data) == 0 { executionRequests.Data = []shuffle.ExecutionRequest{} } else { - //log.Printf("[INFO] Executionrequests (%s): %d", id, len(executionRequests.Data)) - //log.Printf("IDS: %#v", executionRequests.Data[0].ExecutionId) + log.Printf("In workflowqueue with %d", len(executionRequests.Data)) + + // Try again :) + if len(env.Id) == 0 && len(env.Name) == 0 { + orgId := "" + for _, requestData := range executionRequests.Data { + execution, err := shuffle.GetWorkflowExecution(ctx, requestData.ExecutionId) + if err == nil { + if len(execution.ExecutionOrg) > 0 { + orgId = execution.ExecutionOrg + break + } + } + } + + if len(orgId) > 0 { + env, err := shuffle.GetEnvironment(ctx, id, orgId) + if err != nil { + log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", id, err) + //resp.WriteHeader(401) + //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No env found matching %s"}`, id))) + //return + } else { + log.Printf("Found Env: %#v", env) + if timeNow > env.Edited+60 { + log.Printf("Updating env with IP %s!", request.RemoteAddr) + env.RunningIp = request.RemoteAddr + env.Checkin = timeNow + err = shuffle.SetEnvironment(ctx, env) + if err != nil { + log.Printf("[WARNING] Failed updating environment: %s", err) + } + } + } + } + } + + if len(executionRequests.Data) > 10 { + executionRequests.Data = executionRequests.Data[0:9] + } + log.Printf("In workflowqueue with %d (2)", len(executionRequests.Data)) } newjson, err := json.Marshal(executionRequests) diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index 24bcef4d..1888e374 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -2717,7 +2717,7 @@ const Admin = (props) => { style={{minWidth: 150, maxWidth: 150}} /> { primary="Archived" style={{minWidth: 150, maxWidth: 150}} /> + {environments === undefined || environments === null ? null : environments.map((environment, index)=> { if (!showArchived && environment.archived) { @@ -2747,19 +2751,19 @@ const Admin = (props) => { return null } - //var bgColor = "#27292d" - //if (index % 2 === 0) { - // bgColor = "#1f2023" - //} + var bgColor = "#27292d" + if (index % 2 === 0) { + bgColor = "#1f2023" + } return ( - + { style={{minWidth: 150, maxWidth: 150, overflow: "hidden"}} primary={environment.archived.toString()} /> + ) })} @@ -2907,7 +2915,7 @@ const Admin = (props) => { style={{minWidth: 150, maxWidth: 150}} /> {