From 4b2d5da2a4fd909ac5530211d6de8895c5d47cbe Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 7 Jul 2020 09:25:31 +0200 Subject: [PATCH 01/16] #35: Added PoC of node autocompleter --- frontend/src/AngularWorkflow.js | 163 ++++++++++++++++++-------- frontend/src/defaultCytoscapeStyle.js | 12 +- 2 files changed, 126 insertions(+), 49 deletions(-) diff --git a/frontend/src/AngularWorkflow.js b/frontend/src/AngularWorkflow.js index 69a5431c..5eaad198 100644 --- a/frontend/src/AngularWorkflow.js +++ b/frontend/src/AngularWorkflow.js @@ -35,6 +35,7 @@ import Switch from '@material-ui/core/Switch'; import ReactJson from 'react-json-view' import { useBeforeunload } from 'react-beforeunload'; +import ArrowUpwardIcon from '@material-ui/icons/ArrowUpward'; import CachedIcon from '@material-ui/icons/Cached'; import DirectionsRunIcon from '@material-ui/icons/DirectionsRun'; import PolymerIcon from '@material-ui/icons/Polymer'; @@ -326,6 +327,7 @@ const AngularWorkflow = (props) => { currentnode.removeClass('not-executing-highlight') currentnode.removeClass('success-highlight') currentnode.removeClass('failure-highlight') + currentnode.removeClass('shuffle-hover-highlight') currentnode.removeClass('awaiting-data-highlight') incomingEdges.addClass('success-highlight') currentnode.addClass('executing-highlight') @@ -334,6 +336,7 @@ const AngularWorkflow = (props) => { currentnode.removeClass('not-executing-highlight') currentnode.removeClass('success-highlight') currentnode.removeClass('failure-highlight') + currentnode.removeClass('shuffle-hover-highlight') currentnode.removeClass('awaiting-data-highlight') currentnode.removeClass('executing-highlight') currentnode.addClass('skipped-highlight') @@ -342,6 +345,7 @@ const AngularWorkflow = (props) => { currentnode.removeClass('not-executing-highlight') currentnode.removeClass('success-highlight') currentnode.removeClass('failure-highlight') + currentnode.removeClass('shuffle-hover-highlight') currentnode.removeClass('awaiting-data-highlight') currentnode.addClass('executing-highlight') @@ -363,6 +367,7 @@ const AngularWorkflow = (props) => { currentnode.removeClass('not-executing-highlight') currentnode.removeClass('executing-highlight') currentnode.removeClass('failure-highlight') + currentnode.removeClass('shuffle-hover-highlight') currentnode.removeClass('awaiting-data-highlight') currentnode.addClass('success-highlight') @@ -384,6 +389,7 @@ const AngularWorkflow = (props) => { if (targetnode !== undefined && !targetnode.classes().includes("success-highlight") && !targetnode.classes().includes("failure-highlight")) { targetnode.removeClass('not-executing-highlight') targetnode.removeClass('success-highlight') + targetnode.removeClass('shuffle-hover-highlight') targetnode.removeClass('failure-highlight') targetnode.removeClass('awaiting-data-highlight') targetnode.addClass('executing-highlight') @@ -398,6 +404,7 @@ const AngularWorkflow = (props) => { currentnode.removeClass('executing-highlight') currentnode.removeClass('success-highlight') currentnode.removeClass('awaiting-data-highlight') + currentnode.removeClass('shuffle-hover-highlight') currentnode.addClass('failure-highlight') if (!visited.includes(item.action.label)) { @@ -411,6 +418,7 @@ const AngularWorkflow = (props) => { currentnode.removeClass('executing-highlight') currentnode.removeClass('success-highlight') currentnode.removeClass('failure-highlight') + currentnode.removeClass('shuffle-hover-highlight') currentnode.addClass('awaiting-data-highlight') break default: @@ -1486,7 +1494,7 @@ const AngularWorkflow = (props) => { {workflow.workflow_variables === null ? null : workflow.workflow_variables.map(variable=> { return ( -
+
{ }}>
@@ -2197,7 +2205,6 @@ const AngularWorkflow = (props) => { var allkeys = [action.id] var handled = [] var results = [] - console.log("BEFORE PARENTS!") while(true) { for (var key in allkeys) { @@ -2247,6 +2254,7 @@ const AngularWorkflow = (props) => { const [selectedActionParameters, setSelectedActionParameters] = React.useState([]) const [selectedVariableParameter, setSelectedVariableParameter] = React.useState() const [showDropdown, setShowDropdown] = React.useState(false) + const [showDropdownNumber, setShowDropdownNumber] = React.useState(0) const [actionlist, setActionlist] = React.useState([]) useEffect(() => { @@ -2266,7 +2274,7 @@ const AngularWorkflow = (props) => { } if (actionlist.length === 0) { - actionlist.push({"type": "Execution Argument", "name": "Execution Argument", "value": "$exec", "highlight": "exec", "autocomplete": "$exec"}) + actionlist.push({"type": "Execution Argument", "name": "Execution Argument", "value": "$exec", "highlight": "exec", "autocomplete": "exec"}) if (workflow.workflow_variables !== null && workflow.workflow_variables !== undefined && workflow.workflow_variables.length > 0) { for (var key in workflow.workflow_variables) { const item = workflow.workflow_variables[key] @@ -2298,11 +2306,10 @@ const AngularWorkflow = (props) => { }) const changeActionParameter = (event, count) => { - console.log("EVENT: ", event.target.value) if (event.target.value[event.target.value.length-1] === "$") { - console.log("LAST IS $ - SHOULD SHOW DROPDOWN") if (!showDropdown) { setShowDropdown(true) + setShowDropdownNumber(count) } } else { if (showDropdown) { @@ -2393,39 +2400,7 @@ const AngularWorkflow = (props) => { // FIXME: Issue #40 - selectedActionParameters not reset if (Object.getOwnPropertyNames(selectedAction).length > 0 && selectedActionParameters.length > 0) { return ( -
- - {showDropdown ? - - : null} - - +
Arguments {selectedActionParameters.map((data, count) => { if (data.variant === "") { @@ -2473,14 +2448,14 @@ const AngularWorkflow = (props) => { }} onBlur={(event) => { // Super basic check - if (event.target.value.startsWith("{")) { - console.log("VALIDATING JSON") - try { - JSON.parse(event.target.value) - } catch (e) { - alert.error("Failed to parse json: ", e) - } - } + //if (event.target.value.startsWith("{")) { + // console.log("VALIDATING JSON") + // try { + // JSON.parse(event.target.value) + // } catch (e) { + // alert.error("Failed to parse json: ", e) + // } + //} }} /> @@ -2596,7 +2571,7 @@ const AngularWorkflow = (props) => { itemColor = "#ffeb3b" } return ( -
+
@@ -2630,6 +2605,98 @@ const AngularWorkflow = (props) => {
{datafield} + {showDropdown && showDropdownNumber === count && data.variant === "STATIC_VALUE" ? + + Autocomplete + + + : null} + +
)})}
@@ -2783,7 +2850,6 @@ const AngularWorkflow = (props) => { selectedAction.execution_variable = {"name": "No selection"} } else { const value = workflow.execution_variables.find(a => a.name === e.target.value) - console.log("FOUND: ", value) selectedAction.execution_variable = value } setSelectedAction(selectedAction) @@ -4431,6 +4497,7 @@ const AngularWorkflow = (props) => {
node', css: { From a696e01289ac59b842148876f4acf9a64b457ec9 Mon Sep 17 00:00:00 2001 From: Harduino Date: Tue, 7 Jul 2020 11:17:30 +0300 Subject: [PATCH 02/16] siemonster :: fix wrong user id on import workflows --- backend/go-app/walkoff.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 660db8fd..7f440276 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -3712,7 +3712,7 @@ func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) { _ = r log.Printf("Starting workflow folder iteration") - iterateWorkflowGithubFolders(fs, dir, "", "") + iterateWorkflowGithubFolders(fs, dir, "", "", user.Id) } else if strings.Contains(tmpBody.URL, "s3") { //https://docs.aws.amazon.com/sdk-for-go/api/service/s3/ @@ -4012,7 +4012,7 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, } // Onlyname is used to -func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error { +func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, userId string) error { var err error for _, file := range dir { @@ -4031,7 +4031,7 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra } // Go routine? Hmm, this can be super quick I guess - err = iterateWorkflowGithubFolders(fs, dir, tmpExtra, "") + err = iterateWorkflowGithubFolders(fs, dir, tmpExtra, "", userId) if err != nil { continue } @@ -4058,6 +4058,11 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra continue } + // rewrite owner to user who imports now + if userId != "" { + workflow.Owner = userId + } + ctx := context.Background() err = setWorkflow(ctx, workflow, workflow.ID) if err != nil { From 76468eb363294cfaf097237e24fbac781f784e6a Mon Sep 17 00:00:00 2001 From: Harduino Date: Tue, 7 Jul 2020 11:32:55 +0300 Subject: [PATCH 03/16] siemonster :: generate nginx.conf --- docker-compose.yml | 2 + frontend/Dockerfile | 21 +++++- frontend/confd/conf.d/nginx.conf.toml | 9 +++ frontend/confd/templates/nginx.conf | 103 ++++++++++++++++++++++++++ frontend/entrypoint.sh | 7 ++ frontend/nginx.conf | 89 ---------------------- 6 files changed, 138 insertions(+), 93 deletions(-) create mode 100644 frontend/confd/conf.d/nginx.conf.toml create mode 100644 frontend/confd/templates/nginx.conf create mode 100755 frontend/entrypoint.sh delete mode 100644 frontend/nginx.conf diff --git a/docker-compose.yml b/docker-compose.yml index 508567fa..8faba6f3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,8 @@ services: - "${FRONTEND_PORT_HTTPS}:443" networks: - shuffle + environment: + - BACKEND_HOSTNAME=${BACKEND_HOSTNAME} restart: unless-stopped depends_on: - backend diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 2c937de4..c0d93924 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -8,12 +8,17 @@ COPY package.json /usr/src/app/package.json RUN npm install --verbose -COPY . /usr/src/app +# copy only required files to not trigger rebuilding every time +COPY ./certs /usr/src/app/certs/ +COPY ./public /usr/src/app/public/ +COPY ./src /usr/src/app/src/ +COPY ./*.sh /usr/src/app/ +COPY ./*.json /usr/src/app/ RUN npm run-script build # Production environment -from nginx:latest +FROM nginx:latest RUN mkdir -p /usr/share/nginx/html/build RUN mkdir -p /usr/share/nginx/html/css @@ -26,8 +31,16 @@ COPY --from=builder /usr/src/app/build /usr/share/nginx/html COPY --from=builder /usr/src/app/certs/fullchain.pem /etc/nginx/fullchain.cert.pem COPY --from=builder /usr/src/app/certs/privkey.pem /etc/nginx/privkey.pem -# Prod -COPY --from=builder /usr/src/app/nginx.conf /etc/nginx/nginx.conf +# install CONFD +ENV CONFD_VERSION 0.16.0 +RUN curl -sSL https://github.com/kelseyhightower/confd/releases/download/v${CONFD_VERSION}/confd-${CONFD_VERSION}-linux-amd64 -o /usr/local/bin/confd && \ + chmod +x /usr/local/bin/confd +COPY ./confd /etc/confd + +# rewrite command & entrypoint with ours +COPY ./entrypoint.sh / +ENTRYPOINT [ "/entrypoint.sh" ] +CMD ["nginx", "-g", "daemon off;"] EXPOSE 80 EXPOSE 443 diff --git a/frontend/confd/conf.d/nginx.conf.toml b/frontend/confd/conf.d/nginx.conf.toml new file mode 100644 index 00000000..0c6498de --- /dev/null +++ b/frontend/confd/conf.d/nginx.conf.toml @@ -0,0 +1,9 @@ +[template] +src = "nginx.conf" +dest = "/etc/nginx/nginx.conf" +uid = 0 +gid = 0 +mode = "0644" +keys = [ + "/", +] diff --git a/frontend/confd/templates/nginx.conf b/frontend/confd/templates/nginx.conf new file mode 100644 index 00000000..6b61364a --- /dev/null +++ b/frontend/confd/templates/nginx.conf @@ -0,0 +1,103 @@ +user nobody nogroup; +worker_processes auto; # auto-detect number of logical CPU cores + +events { + worker_connections 512; # set the max number of simultaneous connections (per worker process) +} + +http { + client_max_body_size 250M; + + include mime.types; + + # thanks stackoverflow http://stackoverflow.com/a/5132440/2406040 + gzip on; + gzip_http_version 1.1; + gzip_vary on; + gzip_comp_level 6; + gzip_proxied any; + gzip_types text/plain text/css application/json application/javascript application/x-javascript text/javascript text/xml application/xml application/rss+xml application/atom+xml application/rdf+xml; + + # make sure gzip does not lose large gzipped js or css files + # see http://blog.leetsoft.com/2007/07/25/nginx-gzip-ssl.html + gzip_buffers 16 8k; + + # Disable gzip for certain browsers. + gzip_disable "MSIE [1-6].(?!.*SV1)"; + + server { + listen 80; + server_name "localhost"; + location / { + # avoid clickjacking + add_header X-Frame-Options DENY; + # block MIME sniffing + add_header X-Content-Type-Options nosniff; + + # security headers + add_header X-XSS-Protection "1; mode=block"; + # add_header Content-Security-Policy "default-src 'self'"; + add_header Referrer-Policy "no-referrer"; + server_tokens off; + + root /usr/share/nginx/html; + gzip_static on; + expires 1y; + add_header Cache-Control public; + add_header ETag ""; + try_files $uri /index.html; + } + + location /api/v1 { + proxy_pass http://{{ getenv "BACKEND_HOSTNAME" "shuffle-backend" }}:5001; + proxy_buffering off; + proxy_http_version 1.1; + + proxy_connect_timeout 900; + proxy_send_timeout 900; + proxy_read_timeout 900; + send_timeout 900; + } + } + + server { + listen 443 ssl; + server_name "localhost"; + ssl_certificate fullchain.cert.pem; + ssl_certificate_key privkey.pem; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_ciphers HIGH:!aNULL:!MD5; + + location / { + # avoid clickjacking + add_header X-Frame-Options DENY; + # block MIME sniffing + add_header X-Content-Type-Options nosniff; + + # security headers + add_header X-XSS-Protection "1; mode=block"; + # add_header Content-Security-Policy "default-src 'self'"; + add_header Referrer-Policy "no-referrer"; + server_tokens off; + + root /usr/share/nginx/html; + gzip_static on; + expires 1y; + add_header Cache-Control public; + add_header ETag ""; + try_files $uri /index.html; + } + + # Get the hostname from environment here? + location /api/v1 { + proxy_pass http://{{ getenv "BACKEND_HOSTNAME" "shuffle-backend" }}:5001; + proxy_buffering off; + proxy_http_version 1.1; + + proxy_connect_timeout 900; + proxy_send_timeout 900; + proxy_read_timeout 900; + send_timeout 900; + } + } +} diff --git a/frontend/entrypoint.sh b/frontend/entrypoint.sh new file mode 100755 index 00000000..09be2558 --- /dev/null +++ b/frontend/entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# generate configs +/usr/local/bin/confd -backend="env" -confdir="/etc/confd" -onetime + +# run main command +exec "$@" diff --git a/frontend/nginx.conf b/frontend/nginx.conf deleted file mode 100644 index 59405dca..00000000 --- a/frontend/nginx.conf +++ /dev/null @@ -1,89 +0,0 @@ -user nobody nogroup; -worker_processes auto; # auto-detect number of logical CPU cores - -events { - worker_connections 512; # set the max number of simultaneous connections (per worker process) -} - -http { - client_max_body_size 250M; - - include mime.types; - - # thanks stackoverflow http://stackoverflow.com/a/5132440/2406040 - gzip on; - gzip_http_version 1.1; - gzip_vary on; - gzip_comp_level 6; - gzip_proxied any; - gzip_types text/plain text/css application/json application/javascript application/x-javascript text/javascript text/xml application/xml application/rss+xml application/atom+xml application/rdf+xml; - - # make sure gzip does not lose large gzipped js or css files - # see http://blog.leetsoft.com/2007/07/25/nginx-gzip-ssl.html - gzip_buffers 16 8k; - - # Disable gzip for certain browsers. - gzip_disable “MSIE [1-6].(?!.*SV1)”; - - server { - listen 80; - server_name "localhost"; - location / { - # avoid clickjacking - add_header X-Frame-Options DENY; - # block MIME sniffing - add_header X-Content-Type-Options nosniff; - - # security headers - add_header X-XSS-Protection "1; mode=block"; - # add_header Content-Security-Policy "default-src 'self'"; - add_header Referrer-Policy "no-referrer"; - server_tokens off; - - root /usr/share/nginx/html; - gzip_static on; - expires 1y; - add_header Cache-Control public; - add_header ETag ""; - try_files $uri /index.html; - } - - location /api/v1 { - proxy_pass http://shuffle-backend:5001; - } - } - - server { - listen 443 ssl; - server_name "localhost"; - ssl_certificate fullchain.cert.pem; - ssl_certificate_key privkey.pem; - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; - ssl_ciphers HIGH:!aNULL:!MD5; - - location / { - # avoid clickjacking - add_header X-Frame-Options DENY; - # block MIME sniffing - add_header X-Content-Type-Options nosniff; - - # security headers - add_header X-XSS-Protection "1; mode=block"; - # add_header Content-Security-Policy "default-src 'self'"; - add_header Referrer-Policy "no-referrer"; - server_tokens off; - - root /usr/share/nginx/html; - gzip_static on; - expires 1y; - add_header Cache-Control public; - add_header ETag ""; - try_files $uri /index.html; - } - - # Get the hostname from environment here? - location /api/v1 { - proxy_pass http://shuffle-backend:5001; - } - } -} From 9c14d4e93f71703be7fd8dacdd182d6f36fe3526 Mon Sep 17 00:00:00 2001 From: frikky Date: Wed, 8 Jul 2020 01:59:52 +0200 Subject: [PATCH 04/16] Reduced execution amount to 20 --- backend/go-app/walkoff.go | 4 ++-- frontend/src/Workflows.js | 13 +++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 660db8fd..1a50d9f5 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -1925,7 +1925,7 @@ func cleanupExecutions(resp http.ResponseWriter, request *http.Request) { var workflowExecutions []WorkflowExecution _, err = dbclient.GetAll(ctx, q, &workflowExecutions) if err != nil { - log.Printf("Error getting workflowexec: %s", err) + log.Printf("Error getting workflowexec (cleanup): %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting all workflowexecutions"}`))) return @@ -4404,7 +4404,7 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) { } // Query for the specifci workflowId - q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(50) + q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(20) var workflowExecutions []WorkflowExecution _, err = dbclient.GetAll(ctx, q, &workflowExecutions) if err != nil { diff --git a/frontend/src/Workflows.js b/frontend/src/Workflows.js index d0c4e226..e3dccbec 100644 --- a/frontend/src/Workflows.js +++ b/frontend/src/Workflows.js @@ -180,15 +180,20 @@ const Workflows = (props) => { .then((response) => { if (response.status !== 200) { console.log("Status not 200 for WORKFLOW EXECUTION :O!") - alert.error("Failed loading executions for current workflow") } return response.json() }) .then((responseJson) => { - setWorkflowExecutions(responseJson) - if (responseJson.length > 0) { - setSelectedExecution(responseJson[0]) + if (responseJson.success === true) { + setWorkflowExecutions(responseJson) + if (responseJson.length > 0) { + setSelectedExecution(responseJson[0]) + } + } else if (!responseJson.success && responseJson.reason !== undefined) { + alert.error("Failed loading executions: "+responseJson.reason) + } else { + alert.error("Failed loading executions for workflow") } }) .catch(error => { From 987b393ae105eafae3de2cbc45bc5cdb0da90780 Mon Sep 17 00:00:00 2001 From: frikky Date: Wed, 8 Jul 2020 02:21:07 +0200 Subject: [PATCH 05/16] #91: Fixed app_sdk execution result --- backend/app_sdk/app_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 89f1f0a7..e54ea97d 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -810,11 +810,11 @@ class AppBase: result += newres else: try: - result += str(result) + result += str(newres) except ValueError: result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres) print("Can't handle type %s value from function" % (type(newres))) - print("POST NEWRES: ", newres) + print("POST NEWRES RESULT: ", result) else: print("APP_SDK DONE: Starting MULTI execution with", multi_parameters) # 1. Use number of executions based on longest array From ab804a3fdc718261d268bd9757025fb52c108cc8 Mon Sep 17 00:00:00 2001 From: frikky Date: Wed, 8 Jul 2020 03:35:14 +0200 Subject: [PATCH 06/16] #90: Built auto-detection of shuffle network with orborus --- functions/onprem/orborus/orborus.go | 52 +++++++++++++++++++++++++---- functions/onprem/worker/worker.go | 10 +++--- 2 files changed, 50 insertions(+), 12 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 25396a63..d2913af4 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -27,6 +27,7 @@ import ( var baseUrl = os.Getenv("BASE_URL") var baseimagename = "frikky/shuffle" +var shuffleNetwork = "" // Filled in init if found var dockerApiVersion = os.Getenv("DOCKER_API_VERSION") var environment = os.Getenv("ENVIRONMENT_NAME") @@ -61,6 +62,44 @@ func init() { if err != nil { panic(fmt.Sprintf("Unable to create docker client: %s", err)) } + + // BElow: + // Checking if orborus is running on docker within a specific network + ctx := context.Background() + networkName := "" + dockerNetworks, err := dockercli.NetworkList(ctx, types.NetworkListOptions{}) + for _, item := range dockerNetworks { + if strings.Contains(strings.ToLower(item.Name), "shuffle") { + networkName = item.Name + break + } + } + + if len(networkName) > 0 { + containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{ + All: true, + }) + if err != nil { + log.Printf("Failed getting containers during init - running without network check: %s", err) + } + _ = networkName + + // Skip random containers. Only handle things related to Shuffle. + for _, container := range containers { + for key, value := range container.NetworkSettings.Networks { + _ = value + if key == networkName { + for _, name := range container.Names { + if strings.Contains(strings.ToLower(name), "orborus") { + // BEING HERE MEANS THAT ORBORUS HAS BEEN FOUND IN THE SPECIFIED NETWORK + shuffleNetwork = networkName + break + } + } + } + } + } + } } // Deploys the internal worker whenever something happens @@ -82,18 +121,19 @@ func deployWorker(image string, identifier string, env []string) { Env: env, } - // Set the network + // Look for Shuffle network and set it + + // FIXME: Move this out of here and have it be a global setting. During init? networkConfig := &network.NetworkingConfig{} - if baseUrl == "http://shuffle-backend:5001" { + if len(shuffleNetwork) > 0 { networkConfig = &network.NetworkingConfig{ EndpointsConfig: map[string]*network.EndpointSettings{ - "shuffle_shuffle": { - NetworkID: "shuffle_shuffle", + shuffleNetwork: { + NetworkID: shuffleNetwork, }, }, } - } else { - // USE PROXY + env = append(env, fmt.Sprintf("DOCKER_NETWORK", shuffleNetwork)) } //test := &network.EndpointSettings{ diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 983eed17..ded99776 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -347,17 +347,15 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] } networkConfig := &network.NetworkingConfig{} - if baseUrl == "http://shuffle-backend:5001" { + shuffleNetwork := os.Getenv("DOCKER_NETWORK") + if len(shuffleNetwork) > 0 { networkConfig = &network.NetworkingConfig{ EndpointsConfig: map[string]*network.EndpointSettings{ - "shuffle_shuffle": { - NetworkID: "shuffle_shuffle", + shuffleNetwork: { + NetworkID: shuffleNetwork, }, }, } - } else { - // FIXME: Default config - //log.Printf("Bad config: %s. Using default network", baseUrl) } cont, err := cli.ContainerCreate( From 82f08e3c6b3d2df46fd5b787816d7cbbcb48f266 Mon Sep 17 00:00:00 2001 From: frikky Date: Thu, 9 Jul 2020 06:04:35 +0200 Subject: [PATCH 07/16] Changed Orborus network detection --- functions/onprem/orborus/orborus.go | 62 ++++++++++++++++------------- 1 file changed, 34 insertions(+), 28 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index d2913af4..d86ba371 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -63,43 +63,48 @@ func init() { panic(fmt.Sprintf("Unable to create docker client: %s", err)) } - // BElow: - // Checking if orborus is running on docker within a specific network + // FIXME: Move this to global variables? + containerIdentifier := "orborus" + networkIdentifier := "shuffle" + ctx := context.Background() - networkName := "" - dockerNetworks, err := dockercli.NetworkList(ctx, types.NetworkListOptions{}) - for _, item := range dockerNetworks { - if strings.Contains(strings.ToLower(item.Name), "shuffle") { - networkName = item.Name - break - } + containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{ + All: true, + }) + if err != nil { + log.Printf("Failed getting containers during init - running without network check: %s", err) } - if len(networkName) > 0 { - containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{ - All: true, - }) - if err != nil { - log.Printf("Failed getting containers during init - running without network check: %s", err) + // Skip random containers. Only handle things related to Shuffle. + for _, container := range containers { + found := false + //log.Printf("Running? %#v", container) + if container.State != "running" { + continue } - _ = networkName - // Skip random containers. Only handle things related to Shuffle. - for _, container := range containers { - for key, value := range container.NetworkSettings.Networks { - _ = value - if key == networkName { - for _, name := range container.Names { - if strings.Contains(strings.ToLower(name), "orborus") { - // BEING HERE MEANS THAT ORBORUS HAS BEEN FOUND IN THE SPECIFIED NETWORK - shuffleNetwork = networkName - break - } - } + for _, name := range container.Names { + if !strings.Contains(strings.ToLower(name), containerIdentifier) { + found = true + continue + } + } + + if found { + for key, _ := range container.NetworkSettings.Networks { + if strings.Contains(strings.ToLower(key), networkIdentifier) { + shuffleNetwork = key + break } } } } + + if len(shuffleNetwork) > 0 { + log.Printf("Found shuffle network \"%s\" for container %s", shuffleNetwork, containerIdentifier) + } else { + log.Printf("Running Shuffle without a docker network") + } } // Deploys the internal worker whenever something happens @@ -133,6 +138,7 @@ func deployWorker(image string, identifier string, env []string) { }, }, } + env = append(env, fmt.Sprintf("DOCKER_NETWORK", shuffleNetwork)) } From 65d4035cfdceaa521690a81ba8be647989bb391d Mon Sep 17 00:00:00 2001 From: frikky Date: Thu, 9 Jul 2020 11:42:36 +0200 Subject: [PATCH 08/16] Added PoC autocomplete feature based on app return example --- backend/go-app/walkoff.go | 11 ++++ frontend/src/AngularWorkflow.js | 65 +++++++++++++++++++- frontend/src/Apps.js | 102 ++++++++++++++++++++++++++++++++ frontend/src/Workflows.js | 12 ++-- 4 files changed, 182 insertions(+), 8 deletions(-) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 7f40ec0f..b7b2baae 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -142,6 +142,7 @@ type WorkflowAppAction struct { } `json:"execution_variable" datastore:"execution_variables"` Returns struct { Description string `json:"description" datastore:"returns" yaml:"description,omitempty"` + Example string `json:"example" datastore:"example" yaml:"example"` ID string `json:"id" datastore:"id" yaml:"id,omitempty"` Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` } `json:"returns" datastore:"returns"` @@ -4205,6 +4206,16 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin } } + /* + if workflowapp.Name == "thehive" { + for _, action := range workflowapp.Actions { + if len(action.Returns.Example) > 0 { + log.Printf("ACTION: %#v", action) + } + } + } + */ + if skip { continue } diff --git a/frontend/src/AngularWorkflow.js b/frontend/src/AngularWorkflow.js index 5eaad198..d3d567ee 100644 --- a/frontend/src/AngularWorkflow.js +++ b/frontend/src/AngularWorkflow.js @@ -67,6 +67,7 @@ import cxtmenu from 'cytoscape-cxtmenu'; import { w3cwebsocket as W3CWebSocket } from "websocket"; import { useAlert } from "react-alert"; +import { GetParsedPaths } from "./Apps"; const surfaceColor = "#27292D" const inputColor = "#383B40" @@ -2256,6 +2257,7 @@ const AngularWorkflow = (props) => { const [showDropdown, setShowDropdown] = React.useState(false) const [showDropdownNumber, setShowDropdownNumber] = React.useState(0) const [actionlist, setActionlist] = React.useState([]) + const [jsonList, setJsonList] = React.useState([]) useEffect(() => { if (selectedActionParameters !== null && selectedActionParameters.length === 0) { @@ -2317,6 +2319,20 @@ const AngularWorkflow = (props) => { } } + if (event.target.value[event.target.value.length-1] === ".") { + console.log("GET THE LAST ARGUMENT FOR !") + //const [jsonList, getJsonList] = React.useState([]) + const inputdata = {"data": "1.2.3.4", "dataType": "4.5.6.6"} + const returnJson = GetParsedPaths(inputdata, "") + console.log(jsonList) + setJsonList(returnJson) + + if (!showDropdown) { + setShowDropdown(true) + setShowDropdownNumber(count) + } + } + selectedActionParameters[count].value = event.target.value selectedAction.parameters[count].value = event.target.value setSelectedAction(selectedAction) @@ -2605,7 +2621,49 @@ const AngularWorkflow = (props) => {
{datafield} - {showDropdown && showDropdownNumber === count && data.variant === "STATIC_VALUE" ? + {showDropdown && showDropdownNumber === count && data.variant === "STATIC_VALUE" && jsonList.length > 0 ? + + Autocomplete + + + : null} + {showDropdown && showDropdownNumber === count && data.variant === "STATIC_VALUE" && jsonList.length === 0 ? Autocomplete { + console.log("CHOSE AN AUTHENTICATION OPTION: ", e.target.value) + selectedAction.selectedAuthentication = e.target.value + selectedAction.authentication_id = e.target.value.id + setSelectedAction(selectedAction) + setUpdate("update auth") + }} + style={{backgroundColor: inputColor, color: "white", height: "50px"}} + > + {selectedAction.authentication.map(data => ( + + {data.label} - ({data.app.app_version}) + + ))} + + + {/* + + + curaction.authentication = authenticationOptions + if (curaction.selectedAuthentication === null || curaction.selectedAuthentication === undefined || curaction.selectedAuthentication.length === "") + */} + + + +
+
+ : null} + {environments !== undefined && environments !== null && environments.length > 1 ?
Environment
: null} - {/*requiresAuthentication ? -
- -
- : null*/}
@@ -3033,7 +3183,7 @@ const AngularWorkflow = (props) => { newActionname = newActionname.replace("_", " ") newActionname = newActionname.charAt(0).toUpperCase()+newActionname.substring(1) return ( - + {newActionname} @@ -5243,52 +5393,136 @@ const AngularWorkflow = (props) => { : null - const AuthenticationData = () => { - console.log("AUTH: ", selectedApp.authentication) - const [tmpVar, setTmpVar] = React.useState("") + const AuthenticationData = (props) => { + const selectedApp = props.app + + const [authenticationOption, setAuthenticationOptions] = React.useState({ + app: JSON.parse(JSON.stringify(selectedApp)), + fields: {}, + label: "", + usage: [{ + workflow_id: workflow.id, + }], + id: uuid.v4(), + active: true, + }) + if (selectedApp.authentication === undefined) { return null } - if (selectedApp.authentication.parameters.length === undefined || - selectedApp.authentication.parameters.length === 0) { + if (selectedApp.authentication.parameters.length === undefined || selectedApp.authentication.parameters.length === 0) { return null } - // Yes, it should be possible to have more than one, but.. :) - // This data should be written to a KMS, then have the ID point back - const currentAuth = selectedApp.authentication.parameters[0] - if (currentAuth.scheme.toLowerCase() === "bearer") { - return
- Insert your API token for {selectedApp.name} - { - setTmpVar(event.target.value) - }} - onBlur={() => { - selectedApp.authentication.parameters[0].value = tmpVar - setSelectedApp(selectedApp) - }} - /> -
+ authenticationOption.app.actions = [] + + for (var key in selectedApp.authentication.parameters) { + if (authenticationOption.fields[selectedApp.authentication.parameters[key].name] === undefined) { + authenticationOption.fields[selectedApp.authentication.parameters[key].name] = "" + } + } + + const handleSubmitCheck = () => { + console.log(authenticationOption) + if (authenticationOption.label.length === 0) { + alert.info("Label can't be empty") + } + + for (var key in selectedApp.authentication.parameters) { + if (authenticationOption.fields[selectedApp.authentication.parameters[key].name].length === 0) { + alert.info("Field "+selectedApp.authentication.parameters[key].name+" can't be empty") + return + } + } + + selectedAction.authentication_id = authenticationOption.id + selectedAction.selectedAuthentication = authenticationOption + selectedAction.authentication.push(authenticationOption) + setSelectedAction(selectedAction) + + var newFields = [] + for (const key in authenticationOption.fields) { + const value = authenticationOption.fields[key] + newFields.push({ + key: key, + value: value, + }) + } + + authenticationOption.fields = newFields + setNewAppAuth(authenticationOption) } return (
- NOT IMPLEMENTED
- Unknown auth: {currentAuth.scheme} + + What is this? +  These are required fields for authenticating with TheHive +
+ {selectedApp.link.length > 0 ? : null} + Label (to remember it) + { + authenticationOption.label = event.target.value + }} + /> + +
+ {selectedApp.authentication.parameters.map((data, index) => { + return ( +
+ {data.name} + { + authenticationOption.fields[data.name] = event.target.value + }} + /> +
+ ) + })} + + + + +
) } @@ -5328,40 +5562,22 @@ const AngularWorkflow = (props) => { // This whole part is redundant. Made it part of Arguments instead. const authenticationModal = authenticationModalOpen ? - { - setAuthenticationModalOpen(false) - setAppAuthentication({}) + //setAuthenticationModalOpen(false) }} PaperProps={{ style: { backgroundColor: surfaceColor, color: "white", - minWidth: "800px", + minWidth: 600, + padding: 15, }, }} >
Authentication for {selectedApp.name}
 - - What is this? -
- {selectedApp.link.length > 0 ? : null} -
- - - - - - +
: null const loadedCheck = isLoaded && isLoggedIn && workflowDone ? diff --git a/frontend/src/Apps.js b/frontend/src/Apps.js index 140fdf7d..8d0a23ce 100644 --- a/frontend/src/Apps.js +++ b/frontend/src/Apps.js @@ -289,8 +289,8 @@ const Apps = (props) => { { if (selectedApp.id !== data.id) { setSelectedApp(data) + console.log(data) if (data.actions !== undefined && data.actions !== null && data.actions.length > 0) { - console.log(data.actions[0]) setSelectedAction(data.actions[0]) } else { setSelectedAction({}) From 7578f1abae2eac3e91beae37b54efbe7adb3a625 Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 17 Jul 2020 05:13:37 +0200 Subject: [PATCH 11/16] Fixed appauth deletion --- backend/go-app/walkoff.go | 120 +++++++++++++++++++++++++++++++++++--- frontend/src/Admin.js | 108 ++++++++++++++++++++++++---------- 2 files changed, 187 insertions(+), 41 deletions(-) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 03ef8e81..39ecabaf 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -78,12 +78,14 @@ type Org struct { } type AppAuthenticationStorage struct { - Active bool `json:"active" datastore:"active"` - Label string `json:"label" datastore:"label"` - Id string `json:"id" datastore:"id"` - App WorkflowApp `json:"app" datastore:"app"` - Fields []AuthenticationStore `json:"fields" datastore:"fields"` - Usage []AuthenticationUsage `json:"usage" datastore:"usage"` + Active bool `json:"active" datastore:"active"` + Label string `json:"label" datastore:"label"` + Id string `json:"id" datastore:"id"` + App WorkflowApp `json:"app" datastore:"app"` + Fields []AuthenticationStore `json:"fields" datastore:"fields"` + Usage []AuthenticationUsage `json:"usage" datastore:"usage"` + WorkflowCount int64 `json:"workflow_count" datastore:"workflow_count"` + NodeCount int64 `json:"node_count" datastore:"node_count"` } type AuthenticationUsage struct { @@ -1398,6 +1400,61 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(`{"success": true}`)) } +// Adds app auth tracking +func updateAppAuth(auth AppAuthenticationStorage, workflowId, nodeId string, add bool) error { + workflowFound := false + workflowIndex := 0 + nodeFound := false + for index, workflow := range auth.Usage { + if workflow.WorkflowId == workflowId { + // Check if node exists + workflowFound = true + workflowIndex = index + log.Printf("Found workflow: %#v", workflow) + for _, actionId := range workflow.Nodes { + if actionId == nodeId { + nodeFound = true + break + } + } + + break + } + } + + // FIXME: Add a way to use !add to remove + updateAuth := false + if !workflowFound && add { + log.Printf("Adding workflow things to auth!") + usageItem := AuthenticationUsage{ + WorkflowId: workflowId, + Nodes: []string{nodeId}, + } + + auth.Usage = append(auth.Usage, usageItem) + auth.WorkflowCount += 1 + auth.NodeCount += 1 + updateAuth = true + } else if !nodeFound && add { + log.Printf("Adding node things to auth!") + auth.Usage[workflowIndex].Nodes = append(auth.Usage[workflowIndex].Nodes, nodeId) + auth.NodeCount += 1 + updateAuth = true + } + + if updateAuth { + log.Printf("Updating auth!") + ctx := context.Background() + err := setWorkflowAppAuthDatastore(ctx, auth, auth.Id) + if err != nil { + log.Printf("Failed setting up app auth %s: %s", auth.Id, err) + return err + } + } + + return nil +} + // Saves a workflow to an ID func saveWorkflow(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) @@ -1496,7 +1553,6 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { //log.Printf("Action: %#v", action.Authentication) for _, action := range workflow.Actions { - log.Printf("Auth: %s", action.AuthenticationId) allNodes = append(allNodes, action.ID) if action.Environment == "" { @@ -1640,6 +1696,14 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } } + allAuths, err := getAllWorkflowAppAuth(ctx) + if userErr != nil { + log.Printf("Api authentication failed in get all apps: %s", userErr) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + // Check every app action and param to see whether they exist newActions = []Action{} for _, action := range workflow.Actions { @@ -1657,6 +1721,31 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } } + // FIXME: Check auth + if len(action.AuthenticationId) > 0 { + authFound := false + + for _, auth := range allAuths { + if auth.Id == action.AuthenticationId { + authFound = true + + // Fix stuff here + err := updateAppAuth(auth, workflow.ID, action.ID, true) + if err != nil { + log.Printf("Failed updating the app auth reference: %s (not critical)", err) + } + break + } + } + + if !authFound { + log.Printf("App auth %s doesn't exist", action.AuthenticationId) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "App auth %s doesn't exist"}`, action.AuthenticationId))) + return + } + } + if builtin { newActions = append(newActions, action) } else { @@ -3020,16 +3109,29 @@ func deleteAppAuthentication(resp http.ResponseWriter, request *http.Request) { log.Printf("%#v", location) var fileId string if location[1] == "api" { - if len(location) <= 4 { + if len(location) <= 5 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } - fileId = location[4] + fileId = location[5] } + // FIXME: Set affected workflows to have errors + // 1. Get the auth + // 2. Loop the workflows (.Usage) and set them to have errors + // 3. Loop the nodes in workflows and do the same + log.Printf("ID: %s", fileId) + ctx := context.Background() + err := DeleteKey(ctx, "workflowappauth", fileId) + if err != nil { + log.Printf("Failed deleting workflowapp") + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed deleting workflow app"}`))) + return + } resp.WriteHeader(200) resp.Write([]byte(`{"success": true}`)) diff --git a/frontend/src/Admin.js b/frontend/src/Admin.js index 7d66bcac..ca71c372 100644 --- a/frontend/src/Admin.js +++ b/frontend/src/Admin.js @@ -38,11 +38,40 @@ const Admin = (props) => { const [selectedUser, setSelectedUser] = React.useState({}) const [newPassword, setNewPassword] = React.useState(""); const [selectedUserModalOpen, setSelectedUserModalOpen] = React.useState(false) - const [selectedAuthentication, setSelectedAuthentcation] = React.useState({}) + const [selectedAuthentication, setSelectedAuthentication] = React.useState({}) const [selectedAuthenticationModalOpen, setSelectedAuthenticationModalOpen] = React.useState(false) const alert = useAlert() + const deleteAuthentication = (data) => { + alert.info("Deleting auth "+data.label) + + // Just use this one? + const url = globalUrl+'/api/v1/apps/authentication/'+data.id + console.log("URL: ", url) + fetch(url, { + method: 'DELETE', + credentials: "include", + headers: { + 'Content-Type': 'application/json', + }, + }) + .then(response => + response.json().then(responseJson => { + console.log("RESP: ", responseJson) + if (responseJson["success"] === false) { + alert.error("Failed stopping schedule") + } else { + getAppAuthentication() + alert.success("Successfully stopped schedule!") + } + }), + ) + .catch(error => { + console.log("Error in userdata: ", error) + }); + } + const deleteSchedule = (data) => { // FIXME - add some check here ROFL console.log("INPUT: ", data) @@ -165,6 +194,7 @@ const Admin = (props) => { const deleteEnvironment = (name) => { // FIXME - add some check here ROFL + alert.info("Deleting environment "+name) var newEnv = [] for (var key in environments) { if (environments[key].Name == name) { @@ -566,7 +596,9 @@ const Admin = (props) => { }, }} > - Add user + + {curTab === 0 ? "Add user" : "Add environment"} + {curTab === 0 ?
@@ -613,7 +645,7 @@ const Admin = (props) => { onChange={(event) => changeModalData("Password", event.target.value)} />
- : curTab === 1 ? + : curTab === 2 ?
Environment Name { - + + + + {environments === undefined ? null : environments.map(environment => { return ( - - - {environment.Name} + + + + ) })} + +
: null From 7c0dda2c52a57f2d62e4d90d958786c764c62d34 Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 17 Jul 2020 07:00:13 +0200 Subject: [PATCH 12/16] #26: Fixed app authentication issues --- backend/go-app/walkoff.go | 114 ++++++++++++++++++++++++-------- frontend/src/Admin.js | 61 +++++++++++------ frontend/src/AngularWorkflow.js | 38 +++++++---- 3 files changed, 153 insertions(+), 60 deletions(-) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 39ecabaf..e8d8bdf7 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -122,16 +122,17 @@ type WorkflowApp struct { } type WorkflowAppActionParameter struct { - Description string `json:"description" datastore:"description" yaml:"description"` - ID string `json:"id" datastore:"id" yaml:"id,omitempty"` - Name string `json:"name" datastore:"name" yaml:"name"` - Example string `json:"example" datastore:"example" yaml:"example"` - Value string `json:"value" datastore:"value" yaml:"value,omitempty"` - Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` - ActionField string `json:"action_field" datastore:"action_field" yaml:"actionfield,omitempty"` - Variant string `json:"variant" datastore:"variant" yaml:"variant,omitempty"` - Required bool `json:"required" datastore:"required" yaml:"required"` - Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` + Description string `json:"description" datastore:"description" yaml:"description"` + ID string `json:"id" datastore:"id" yaml:"id,omitempty"` + Name string `json:"name" datastore:"name" yaml:"name"` + Example string `json:"example" datastore:"example" yaml:"example"` + Value string `json:"value" datastore:"value" yaml:"value,omitempty"` + Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` + ActionField string `json:"action_field" datastore:"action_field" yaml:"actionfield,omitempty"` + Variant string `json:"variant" datastore:"variant" yaml:"variant,omitempty"` + Required bool `json:"required" datastore:"required" yaml:"required"` + Configuration bool `json:"configuration" datastore:"configuration" yaml:"configuration"` + Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` } type SchemaDefinition struct { @@ -621,6 +622,7 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque resp.Write([]byte("OK")) } +// FIXME: Authenticate this one (especially since we have a default: shuffle) func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -1410,7 +1412,6 @@ func updateAppAuth(auth AppAuthenticationStorage, workflowId, nodeId string, add // Check if node exists workflowFound = true workflowIndex = index - log.Printf("Found workflow: %#v", workflow) for _, actionId := range workflow.Nodes { if actionId == nodeId { nodeFound = true @@ -1721,19 +1722,18 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } } - // FIXME: Check auth + // Check auth + // 1. Find the auth in question + // 2. Update the node and workflow info in the auth + // 3. Get the values in the auth and add them to the action values if len(action.AuthenticationId) > 0 { authFound := false - for _, auth := range allAuths { if auth.Id == action.AuthenticationId { authFound = true - // Fix stuff here - err := updateAppAuth(auth, workflow.ID, action.ID, true) - if err != nil { - log.Printf("Failed updating the app auth reference: %s (not critical)", err) - } + // Updates the auth item itself IF necessary + go updateAppAuth(auth, workflow.ID, action.ID, true) break } } @@ -1796,9 +1796,6 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { for _, param := range curappaction.Parameters { found := false - // FIXME: Check if the name exists in authentication.parameters and doesn't use the auth required field - // If it does, the auth should be saved somehow. - // Handles check for parameter exists + value not empty in used fields for _, actionParam := range action.Parameters { if actionParam.Name == param.Name { @@ -1817,6 +1814,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } newParams = append(newParams, actionParam) + break } } @@ -2308,6 +2306,8 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf // FIXME - remove this? newActions := []Action{} defaultResults := []ActionResult{} + + allAuths := []AppAuthenticationStorage{} for _, action := range workflowExecution.Workflow.Actions { action.LargeImage = "" if action.ID == workflowExecution.Start { @@ -2319,6 +2319,47 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf return WorkflowExecution{}, fmt.Sprintf("Environment is not defined for %s", action.Name), errors.New("Environment not defined!") } + // FIXME: Authentication parameters + if len(action.AuthenticationId) > 0 { + if len(allAuths) == 0 { + allAuths, err = getAllWorkflowAppAuth(ctx) + if err != nil { + log.Printf("Api authentication failed in get all app auth: %s", err) + return WorkflowExecution{}, fmt.Sprintf("Api authentication failed in get all app auth: %s", err), err + } + } + + curAuth := AppAuthenticationStorage{Id: ""} + for _, auth := range allAuths { + if auth.Id == action.AuthenticationId { + curAuth = auth + break + } + } + + if len(curAuth.Id) == 0 { + return WorkflowExecution{}, fmt.Sprintf("Auth ID %s doesn't exist", action.AuthenticationId), errors.New(fmt.Sprintf("Auth ID %s doesn't exist", action.AuthenticationId)) + } + + // Rebuild params with the right data. This is to prevent issues on the frontend + newParams := []WorkflowAppActionParameter{} + for _, param := range action.Parameters { + + for _, authparam := range curAuth.Fields { + if param.Name == authparam.Key { + log.Printf("Name: %s - value: %s", param.Name, param.Value) + param.Value = authparam.Value + log.Printf("Name: %s - value: %s\n", param.Name, param.Value) + break + } + } + + newParams = append(newParams, param) + } + + action.Parameters = newParams + } + newActions = append(newActions, action) // If the node is NOT found, it's supposed to be set to SKIPPED, @@ -3471,8 +3512,8 @@ func getAppAuthentication(resp http.ResponseWriter, request *http.Request) { //} ctx := context.Background() allAuths, err := getAllWorkflowAppAuth(ctx) - if userErr != nil { - log.Printf("Api authentication failed in get all app auth: %s", userErr) + if err != nil { + log.Printf("Api authentication failed in get all app auth: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -3484,6 +3525,17 @@ func getAppAuthentication(resp http.ResponseWriter, request *http.Request) { return } + // Cleanup for frontend + newAuth := []AppAuthenticationStorage{} + for _, auth := range allAuths { + newAuthField := auth + for index, _ := range auth.Fields { + newAuthField.Fields[index].Value = "" + } + + newAuth = append(newAuth, newAuthField) + } + newbody, err := json.Marshal(allAuths) if err != nil { log.Printf("Failed unmarshalling all app auths: %s", err) @@ -4586,20 +4638,24 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin appendParams := []WorkflowAppActionParameter{} for _, fieldname := range workflowapp.Authentication.Parameters { found := false - for _, param := range action.Parameters { + for index, param := range action.Parameters { if param.Name == fieldname.Name { found = true + + action.Parameters[index].Configuration = true + log.Printf("Set config to true for field %s!", param.Name) break } } if !found { appendParams = append(appendParams, WorkflowAppActionParameter{ - Name: fieldname.Name, - Description: fieldname.Description, - Example: fieldname.Example, - Required: fieldname.Required, - Schema: fieldname.Schema, + Name: fieldname.Name, + Description: fieldname.Description, + Example: fieldname.Example, + Required: fieldname.Required, + Configuration: true, + Schema: fieldname.Schema, }) } } diff --git a/frontend/src/Admin.js b/frontend/src/Admin.js index ca71c372..6fce60d1 100644 --- a/frontend/src/Admin.js +++ b/frontend/src/Admin.js @@ -21,6 +21,7 @@ import DialogTitle from '@material-ui/core/DialogTitle'; import DialogActions from '@material-ui/core/DialogActions'; import DialogContent from '@material-ui/core/DialogContent'; +import CachedIcon from '@material-ui/icons/Cached'; const surfaceColor = "#27292D" const inputColor = "#383B40" @@ -305,6 +306,7 @@ const Admin = (props) => { }) .then((responseJson) => { if (responseJson.success) { + console.log(responseJson.data) setAuthentication(responseJson.data) } else { alert.error("Failed getting authentications") @@ -333,6 +335,7 @@ const Admin = (props) => { return response.json() }) .then((responseJson) => { + console.log(responseJson) setEnvironments(responseJson) }) .catch(error => { @@ -536,7 +539,7 @@ const Admin = (props) => {
{ const usersView = curTab === 0 ?
-

- User management -

+
+

User management

+ Add, edit, block or change passwords +
+ @@ -941,6 +956,10 @@ const Admin = (props) => { primary="Name" style={{minWidth: 150, maxWidth: 150}} /> + { primary={environment.Name} style={{minWidth: 150, maxWidth: 150, overflow: "hidden"}} /> + @@ -960,11 +983,11 @@ const Admin = (props) => { ) })} - -
: null + // primary={environment.Registered ? "true" : "false"} + const setConfig = (event, newValue) => { if (newValue === 1) { getAppAuthentication() diff --git a/frontend/src/AngularWorkflow.js b/frontend/src/AngularWorkflow.js index 03e02b23..2e316983 100644 --- a/frontend/src/AngularWorkflow.js +++ b/frontend/src/AngularWorkflow.js @@ -54,6 +54,7 @@ import KeyboardArrowLeftIcon from '@material-ui/icons/KeyboardArrowLeft'; import KeyboardArrowRightIcon from '@material-ui/icons/KeyboardArrowRight'; import ArrowBackIcon from '@material-ui/icons/ArrowBack'; import SettingsIcon from '@material-ui/icons/Settings'; +import LockOpenIcon from '@material-ui/icons/LockOpen'; import * as cytoscape from 'cytoscape'; import * as edgehandles from 'cytoscape-edgehandles'; @@ -926,7 +927,9 @@ const AngularWorkflow = (props) => { const newfields = {} for (var filterkey in item.fields) { - newfields[item.fields[filterkey].key] = item.fields[filterkey].value + if (item.fields[filterkey] !== undefined) { + newfields[item.fields[filterkey].key] = item.fields[filterkey].value + } } item.fields = newfields @@ -2585,9 +2588,10 @@ const AngularWorkflow = (props) => { if (!selectedAction.auth_not_required && selectedAction.selectedAuthentication !== undefined && selectedAction.selectedAuthentication.fields !== undefined) { if (selectedAction.selectedAuthentication.fields[data.name] !== undefined) { // FIXME - this should be skipped in the frontend - selectedActionParameters[count].value = selectedAction.selectedAuthentication.fields[data.name] - selectedAction.parameters[count].value = selectedAction.selectedAuthentication.fields[data.name] - setSelectedAction(selectedAction) + //selectedActionParameters[count].value = selectedAction.selectedAuthentication.fields[data.name] + //selectedAction.parameters[count].value = selectedAction.selectedAuthentication.fields[data.name] + //setSelectedAction(selectedAction) + return null } } @@ -2757,8 +2761,16 @@ const AngularWorkflow = (props) => { } return (
-
-
+
+ {data.configuration === true ? + + { + setAuthenticationModalOpen(true) + }}/> + + : +
+ }
{data.name}
@@ -3036,7 +3048,7 @@ const AngularWorkflow = (props) => { /> {selectedAction.authentication.length === 0 && requiresAuthentication ?
- Authentication (reusable): + Authenticate {selectedApp.name}: