From 1648a8921c5eb13aa8f1e5844873acba8eaefdc0 Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 22 Feb 2021 18:51:23 +0100 Subject: [PATCH 01/57] Added simple curl testfiles --- backend/tests/dockerpull.sh | 2 + backend/tests/files.sh | 16 ++++++ backend/tests/hotload.sh | 1 + backend/tests/list_files.sh | 1 + backend/tests/schedules.sh | 2 +- backend/tests/users.sh | 13 +++++ frontend/src/App.jsx | 3 +- frontend/src/components/Header.js | 84 +++++++++++++------------------ frontend/src/views/Admin.jsx | 23 ++++++++- frontend/src/views/Docs.jsx | 11 ++-- 10 files changed, 99 insertions(+), 57 deletions(-) create mode 100644 backend/tests/dockerpull.sh create mode 100755 backend/tests/files.sh create mode 100644 backend/tests/hotload.sh create mode 100644 backend/tests/list_files.sh create mode 100644 backend/tests/users.sh diff --git a/backend/tests/dockerpull.sh b/backend/tests/dockerpull.sh new file mode 100644 index 00000000..3df0d1e5 --- /dev/null +++ b/backend/tests/dockerpull.sh @@ -0,0 +1,2 @@ +#!/bin/sh +curl -XPOST http://localhost:5001/api/v1/get_docker_image -d '{"name":"frikky/shuffle:Testing_1.0.0"}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" diff --git a/backend/tests/files.sh b/backend/tests/files.sh new file mode 100755 index 00000000..d27a4755 --- /dev/null +++ b/backend/tests/files.sh @@ -0,0 +1,16 @@ +curl http://192.168.3.6:5001/api/v1/files/create -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" -d '{"filename": "file.txt", "org_id": "b199646b-16d2-456d-9fd6-b9972e929466", "workflow_id": "global"}' + +curl http://localhost:5001/api/v1/files/create -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" -d '{"filename": "file.txt", "org_id": "b199646b-16d2-456d-9fd6-b9972e929466", "workflow_id": "global"}' + +echo +curl http://localhost:5001/api/v1/files/e19cffe4-e2da-47e9-809e-904f5cb03687/upload -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" -F 'shuffle_file=@files.sh' + +curl http://localhost:5001/api/v1/files/1915981b-b897-4db1-8a2e-44bc34cead3b/content -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" +curl http://localhost:5001/api/v1/files/e19cffe4-e2da-47e9-809e-904f5cb03687 -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" +curl -XDELETE http://localhost:5001/api/v1/files/e19cffe4-e2da-47e9-809e-904f5cb03687 -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" + + #r.HandleFunc("/api/v1/files/{fileId}/content", handleGetFileContent).Methods("GET", "OPTIONS") + #r.HandleFunc("/api/v1/files/create", handleCreateFile).Methods("POST", "OPTIONS") + #r.HandleFunc("/api/v1/files/{fileId}/upload", handleUploadFile).Methods("POST", "OPTIONS") + #r.HandleFunc("/api/v1/files/{fileId}", handleGetFileMeta).Methods("GET", "OPTIONS") + #r.HandleFunc("/api/v1/files/{fileId}", handleDeleteFile).Methods("DELETE", "OPTIONS") diff --git a/backend/tests/hotload.sh b/backend/tests/hotload.sh new file mode 100644 index 00000000..d55544f6 --- /dev/null +++ b/backend/tests/hotload.sh @@ -0,0 +1 @@ +curl http://localhost:5001/api/v1/apps/run_hotload -H "Authorization: Bearer e08c6f22-9a55-4557-b008-04388cc51fb0" diff --git a/backend/tests/list_files.sh b/backend/tests/list_files.sh new file mode 100644 index 00000000..4c63e246 --- /dev/null +++ b/backend/tests/list_files.sh @@ -0,0 +1 @@ +curl -XGET http://192.168.3.6:5001/api/v1/files -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" diff --git a/backend/tests/schedules.sh b/backend/tests/schedules.sh index 339f982d..dc0c15d8 100644 --- a/backend/tests/schedules.sh +++ b/backend/tests/schedules.sh @@ -1,2 +1,2 @@ # Fails cus of unmarshal -curl -XPOST http://localhost:5001/api/v1/workflows/1d9d8ce2-566e-4c3f-8a37-5d6c7d2000b5/schedule -d '{"name": "hey", "frequency": "*/1 * * * *", "execution_argument": "{\"test\": \"hey\"}"}' -H "Authorization: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IjYwZjQwNjBlNThkNzVmZDNmNzBiZWZmODhjNzk0YTc3NTMyN2FhMzEiLCJ0eXAiOiJKV1QifQ.eyJhdWQiOiJodHRwczovL3NodWZmbGVyLmlvL2FwaS92MS93b3JrZmxvd3MvMWQ5ZDhjZTItNTY2ZS00YzNmLThhMzctNWQ2YzdkMjAwMGI1L2V4ZWN1dGUiLCJhenAiOiIxMDMwNzY3ODIwNjE0MjQ2MTg0MjIiLCJlbWFpbCI6InNjaGVkdWxlckBzaHVmZmxlLTI0MTUxNy5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJleHAiOjE1NjU1Mjc1NTEsImlhdCI6MTU2NTUyMzk1MSwiaXNzIjoiaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tIiwic3ViIjoiMTAzMDc2NzgyMDYxNDI0NjE4NDIyIn0.r0EDq9fjhf_5CPTiltyfk_L3uYJp577Uy0yYPcCAl2nv50_z_oUtbWGBpQLL8gcj-NGd3g4E52Qur8k6hCMIQweLS6WAb1279vGffEoCNDfkWb3Oy-yJGP1kzwLvqFJqnHLkSWYXNWvSyWnEimW8Rryx_m1BXS5wcA8l4NIr83kS7fPZrTwjnwFSeGSThwk91DVARzapQb8r0GEgOUyHZ1aBXnV98mikzSUt-5xFKe9eMdD22YJAj0Ru-DxAxs5nOqghX4PMRysWjshjOMrlR1piPWxqAmewp8YKZDCQ5gXskpeAFBDoULT971Wsx_NCohnJsFqx1JfPS9ZYMTW2oQ" +curl -XPOST http://localhost:5001/api/v1/workflows/1d9d8ce2-566e-4c3f-8a37-5d6c7d2000b5/schedule -d '{"name": "hey", "frequency": "*/1 * * * *", "execution_argument": "{\"test\": \"hey\"}"}' -H "Authorization: Bearer WUT" diff --git a/backend/tests/users.sh b/backend/tests/users.sh new file mode 100644 index 00000000..fa78cd6b --- /dev/null +++ b/backend/tests/users.sh @@ -0,0 +1,13 @@ +curl http://localhost:5001/api/v1/users/register -H "Authorization: Bearer 36a3eb38-0070-41c6-b20a-2a3e0941d10e" -d '{"username": "username1", "password": ""}' + +echo +curl http://localhost:5001/api/v1/users -H "Authorization: Bearer 36a3eb38-0070-41c6-b20a-2a3e0941d10e" + +echo UPDATE +curl -XPUT http://localhost:5001/api/v1/users/updateuser -H "Authorization: Bearer 36a3eb38-0070-41c6-b20a-2a3e0941d10e" -d '{"user_id": "id", "role": "admin"}' + +echo +curl -XDELETE http://localhost:5001/api/v1/users/userid -H "Authorization: Bearer 36a3eb38-0070-41c6-b20a-2a3e0941d10e" + +echo +curl -XPOST http://localhost:5001/api/v1/users/generateapikey -H "Authorization: Bearer 36a3eb38-0070-41c6-b20a-2a3e0941d10e" -d '{"user_id": "390efa79-73a3-454b-8b1d-38f56eec14ad"}' diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 18456036..ceee6dbf 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -33,6 +33,7 @@ import { createMuiTheme, MuiThemeProvider } from '@material-ui/core/styles'; import ScrollToTop from "./components/ScrollToTop"; import AlertTemplate from "./components/AlertTemplate"; import { positions, Provider } from "react-alert"; +import {isMobile} from "react-device-detect"; // Production - backend proxy forwarding in nginx var globalUrl = window.location.origin @@ -147,7 +148,7 @@ const App = (message, props) => { } /> } /> } /> - } /> + } /> { window.location.pathname = "/docs/about" }} /> } /> } /> diff --git a/frontend/src/components/Header.js b/frontend/src/components/Header.js index 1143a696..c7cb5e6c 100644 --- a/frontend/src/components/Header.js +++ b/frontend/src/components/Header.js @@ -118,6 +118,39 @@ const Header = props => { // Should be based on some path + const avatarMenu = + + { + setAnchorEl(event.currentTarget); + }}> + + + { + handleClose() + }} + > + { + event.preventDefault() + handleClose() + }}> + + Settings + + + { + event.preventDefault() + handleClose() + handleClickLogout() + }}> + Logout + + + const logoCheck = !homePage ? null : null // Handle top bar or something @@ -197,36 +230,7 @@ const Header = props => {
- { - setAnchorEl(event.currentTarget); - }}> - - - { - handleClose() - }} - > - { - event.preventDefault() - handleClose() - }}> - - Settings - - - { - event.preventDefault() - handleClose() - handleClickLogout() - }}> - Logout - - + {avatarMenu} {userdata === undefined || userdata.admin === undefined || userdata.admin === null || !userdata.admin ? null :
- - -
- Logout -
- - {logoCheck} - - - - - - - + {avatarMenu}
- + // const loadedCheck = diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index 62f926a4..83bff401 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -1441,7 +1441,7 @@ const Admin = (props) => { } const organizationView = curTab === 0 && selectedOrganization.id !== undefined ? -
+

Organization overview

@@ -1457,6 +1457,25 @@ const Admin = (props) => {
:
+ + { + const elementName = "copy_element_shuffle" + const org_id = selectedOrganization.id + var copyText = document.getElementById(elementName); + if (copyText !== null && copyText !== undefined) { + navigator.clipboard.writeText(org_id) + copyText.select(); + copyText.setSelectionRange(0, 99999); /* For mobile devices */ + + /* Copy the text inside the text field */ + document.execCommand("copy"); + + alert.info(org_id + " copied to clipboard") + } + }}> + + + {selectedOrganization.name.length > 0 ? : @@ -2034,7 +2053,7 @@ const Admin = (props) => { /* Copy the text inside the text field */ document.execCommand("copy"); - alert.info(file.id + "copied to clipboard") + alert.info(file.id + " copied to clipboard") } }}> diff --git a/frontend/src/views/Docs.jsx b/frontend/src/views/Docs.jsx index 76258eaf..3f0a045b 100644 --- a/frontend/src/views/Docs.jsx +++ b/frontend/src/views/Docs.jsx @@ -31,7 +31,7 @@ const hrefStyle = { } const Docs = (props) => { - const { isLoaded, globalUrl, inputColor, selectedDoc, serverside, isMobile, update} = props; + const { isLoaded, globalUrl, selectedDoc, serverside, isMobile, update} = props; const theme = useTheme(); const [data, setData] = useState(""); @@ -213,7 +213,7 @@ const Docs = (props) => { function CodeHandler(props) { return ( - 
+			
 				
 					{props.value}
 				
@@ -286,8 +286,8 @@ const Docs = (props) => {
 
 	const mobileStyle = {
 		color: "white",
-		marginLeft: 15,
-		marginRight: 15,
+		marginLeft: 25,
+		marginRight: 25,
 		paddingBottom: 50,
 		backgroundColor: "inherit",
 		display: "flex",
@@ -305,6 +305,7 @@ const Docs = (props) => {
 				 {
 					const path = "/docs/"+item
 					const newname = item.charAt(0).toUpperCase()+item.substring(1).split("_").join(" ").split("-").join(" ")
 					return (
-						 {window.location.pathname = path}}>{newname}
+						 {window.location.pathname = path}}>{newname}
 					)
 				})}
 				

From 0c7dbdf1bf57f3790a5a0c8154aeeca56b8a6b49 Mon Sep 17 00:00:00 2001
From: Harduino 
Date: Thu, 25 Feb 2021 13:54:42 +0300
Subject: [PATCH 02/57] pass SHUFFLE_DOWNLOAD_AUTH_BRANCH to backend service

---
 docker-compose.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker-compose.yml b/docker-compose.yml
index 67fe27a6..bed35f39 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -35,6 +35,7 @@ services:
       - SHUFFLE_FILE_LOCATION=/shuffle-files
       - ORG_ID=${ORG_ID}
       - SHUFFLE_APP_DOWNLOAD_LOCATION=${SHUFFLE_APP_DOWNLOAD_LOCATION}
+      - SHUFFLE_DOWNLOAD_AUTH_BRANCH=${SHUFFLE_DOWNLOAD_AUTH_BRANCH}
       - SHUFFLE_DEFAULT_USERNAME=${SHUFFLE_DEFAULT_USERNAME}
       - SHUFFLE_DEFAULT_PASSWORD=${SHUFFLE_DEFAULT_PASSWORD}
       - SHUFFLE_DEFAULT_APIKEY=${SHUFFLE_DEFAULT_APIKEY}

From ee8b757c3b460aea2b3444729b22a02e87dd0305 Mon Sep 17 00:00:00 2001
From: Harduino 
Date: Thu, 25 Feb 2021 13:58:38 +0300
Subject: [PATCH 03/57] fix build with using golang:latest

---
 backend/Dockerfile                  |  2 +-
 functions/onprem/orborus/Dockerfile |  8 +++++++-
 functions/onprem/worker/Dockerfile  | 10 ++++++++--
 3 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/backend/Dockerfile b/backend/Dockerfile
index 1856d84f..40bb50ea 100644
--- a/backend/Dockerfile
+++ b/backend/Dockerfile
@@ -1,4 +1,4 @@
-from golang as builder
+FROM golang:1.16.0-buster as builder
 
 # Add files
 RUN mkdir /app
diff --git a/functions/onprem/orborus/Dockerfile b/functions/onprem/orborus/Dockerfile
index 3db4c027..3fe760c7 100644
--- a/functions/onprem/orborus/Dockerfile
+++ b/functions/onprem/orborus/Dockerfile
@@ -1,4 +1,4 @@
-from golang as builder
+FROM golang:1.16.0-buster as builder
 
 RUN mkdir /app
 WORKDIR /app
@@ -6,6 +6,12 @@ RUN go get github.com/docker/docker/api/types github.com/docker/docker/api/types
 
 COPY orborus.go /app/orborus.go
 RUN go mod init orborus 
+RUN go get github.com/docker/docker/api/types && \
+    go get github.com/docker/docker/api/types/container && \
+    go get github.com/docker/docker/client && \
+    go get github.com/mackerelio/go-osstat/cpu && \
+    go get github.com/mackerelio/go-osstat/memory && \
+    go get github.com/satori/go.uuid
 RUN go build
 RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o orborus .
 
diff --git a/functions/onprem/worker/Dockerfile b/functions/onprem/worker/Dockerfile
index 483ab136..a6d82d61 100644
--- a/functions/onprem/worker/Dockerfile
+++ b/functions/onprem/worker/Dockerfile
@@ -1,4 +1,4 @@
-from golang as builder
+FROM golang:1.16.0-buster as builder
 
 WORKDIR /app
 
@@ -9,7 +9,13 @@ RUN go get -u github.com/gorilla/mux
 RUN go get -u github.com/patrickmn/go-cache
 
 COPY worker.go /app/worker.go
-RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker .
+RUN go env -w GO111MODULE=auto && \
+    go get github.com/docker/docker/api/types && \
+    go get github.com/docker/docker/api/types/container && \
+    go get github.com/docker/docker/client && \
+    go get github.com/gorilla/mux && \
+    go get github.com/patrickmn/go-cache && \
+    CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker .
 
 FROM alpine:3.12
 

From 9f7f1b9830fc419bb4689e3cc8d0b1136c70a05c Mon Sep 17 00:00:00 2001
From: Harduino 
Date: Thu, 25 Feb 2021 14:11:41 +0300
Subject: [PATCH 04/57] implement environment variable SHUFFLE_APP_FORCE_UPDATE

---
 .env                   |  1 +
 backend/go-app/main.go | 10 +++++++++-
 docker-compose.yml     |  1 +
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/.env b/.env
index 3bb32c35..4ffc4215 100644
--- a/.env
+++ b/.env
@@ -12,6 +12,7 @@ SHUFFLE_APP_DOWNLOAD_LOCATION=https://github.com/frikky/shuffle-apps
 SHUFFLE_DOWNLOAD_AUTH_USERNAME=
 SHUFFLE_DOWNLOAD_AUTH_PASSWORD=
 SHUFFLE_DOWNLOAD_AUTH_BRANCH=
+SHUFFLE_APP_FORCE_UPDATE=false
 
 # User config for first load. Username & PW: min length 3
 SHUFFLE_DEFAULT_USERNAME=
diff --git a/backend/go-app/main.go b/backend/go-app/main.go
index 3031b0b5..7cd4e069 100644
--- a/backend/go-app/main.go
+++ b/backend/go-app/main.go
@@ -7448,6 +7448,14 @@ func runInit(ctx context.Context) {
 		}
 	}
 
+	// form force-flag to download workflow apps
+	forceUpdateEnv := os.Getenv("SHUFFLE_APP_FORCE_UPDATE")
+	forceUpdate := false
+	if len(forceUpdateEnv) > 0 && forceUpdateEnv == "true" {
+		log.Printf("Forcing to rebuild apps")
+		forceUpdate = true
+	}
+
 	// Getting apps to see if we should initialize a test
 	log.Printf("Getting remote workflow apps")
 	workflowapps, err := getAllWorkflowApps(ctx, 500)
@@ -7513,7 +7521,7 @@ func runInit(ctx context.Context) {
 		//iterateAppGithubFolders(fs, dir, "", "testing")
 
 		// FIXME: Get all the apps?
-		iterateAppGithubFolders(fs, dir, "", "", false)
+		iterateAppGithubFolders(fs, dir, "", "", forceUpdate)
 
 		// Hotloads locally
 		location := os.Getenv("SHUFFLE_APP_HOTLOAD_FOLDER")
diff --git a/docker-compose.yml b/docker-compose.yml
index 67fe27a6..3afc58e8 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -38,6 +38,7 @@ services:
       - SHUFFLE_DEFAULT_USERNAME=${SHUFFLE_DEFAULT_USERNAME}
       - SHUFFLE_DEFAULT_PASSWORD=${SHUFFLE_DEFAULT_PASSWORD}
       - SHUFFLE_DEFAULT_APIKEY=${SHUFFLE_DEFAULT_APIKEY}
+      - SHUFFLE_APP_FORCE_UPDATE=${SHUFFLE_APP_FORCE_UPDATE}
       - HTTP_PROXY=${SHUFFLE_HTTP_PROXY}
       - HTTPS_PROXY=${SHUFFLE_HTTPS_PROXY}
     restart: unless-stopped

From e57ec4e2d4e9d2496c615355730a94d36abaf850 Mon Sep 17 00:00:00 2001
From: Harduino 
Date: Thu, 25 Feb 2021 14:23:19 +0300
Subject: [PATCH 05/57] frontend: fix Webpack issue on building Docker image

---
 frontend/Dockerfile   | 6 +-----
 frontend/package.json | 2 +-
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/frontend/Dockerfile b/frontend/Dockerfile
index f48b048d..cd11e347 100644
--- a/frontend/Dockerfile
+++ b/frontend/Dockerfile
@@ -1,5 +1,5 @@
 # Build environment
-FROM node as builder
+FROM node:14 as builder
 
 RUN mkdir /usr/src/app
 
@@ -18,10 +18,6 @@ COPY ./src /usr/src/app/src/
 COPY ./*.sh /usr/src/app/
 COPY ./*.json /usr/src/app/
 
-# There were issues with the webpack installer from package.json
-RUN rm -rf /usr/src/app/node_modules/webpack
-#RUN yarn add webpack@4.42.0
-
 RUN yarn build
 
 # Production environment
diff --git a/frontend/package.json b/frontend/package.json
index 0b99f34c..6805e412 100644
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -57,7 +57,7 @@
     "shellwords": "^0.1.1",
     "simplebar": "^4.2.3",
     "styled-components": "^4.4.0",
-    "webpack": "^4.42.0",
+    "webpack": "4.44.2",
     "websocket": "^1.0.30",
     "yaml": "^1.7.2",
     "yamljs": "^0.3.0",

From 9fe69066223d22fdace49240b8d640ca4331460b Mon Sep 17 00:00:00 2001
From: Harduino 
Date: Thu, 25 Feb 2021 14:27:15 +0300
Subject: [PATCH 06/57] improve work with Kubernetes

---
 functions/onprem/orborus/orborus.go | 2 +-
 functions/onprem/worker/worker.go   | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go
index bf7f2919..a2e3d716 100644
--- a/functions/onprem/orborus/orborus.go
+++ b/functions/onprem/orborus/orborus.go
@@ -106,7 +106,7 @@ func getThisContainerId() {
 	}
 
 	if fCol != "" {
-		cmd := fmt.Sprintf("cat /proc/self/cgroup | grep memory | tail -1 | cut -d/ -f%s", fCol)
+		cmd := fmt.Sprintf("cat /proc/self/cgroup | grep memory | tail -1 | cut -d/ -f%s | grep -o -E '[0-9A-z]{64}'", fCol)
 		out, err := exec.Command("bash", "-c", cmd).Output()
 		if err == nil {
 			containerId = strings.TrimSpace(string(out))
diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go
index dcda8b3e..259b3f06 100644
--- a/functions/onprem/worker/worker.go
+++ b/functions/onprem/worker/worker.go
@@ -53,7 +53,7 @@ var containerId string
 // form container id of current running container
 func getThisContainerId() string {
 	id := ""
-	cmd := fmt.Sprintf("cat /proc/self/cgroup | grep memory | tail -1 | cut -d/ -f3")
+	cmd := fmt.Sprintf("cat /proc/self/cgroup | grep memory | tail -1 | cut -d/ -f3 | grep -o -E '[0-9A-z]{64}'")
 	out, err := exec.Command("bash", "-c", cmd).Output()
 	if err == nil {
 		id = strings.TrimSpace(string(out))
@@ -861,6 +861,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env []
 	}
 
 	// form container id and use it as network source if it's not empty
+	containerId = getThisContainerId()
 	if containerId != "" {
 		hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId))
 	} else {

From 6777c9565906e6ea5de90677e9872950fc1b5e3d Mon Sep 17 00:00:00 2001
From: Harduino 
Date: Thu, 25 Feb 2021 14:31:14 +0300
Subject: [PATCH 07/57] enhance .gitignore file

---
 .gitignore | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.gitignore b/.gitignore
index 5bebe49f..ca9d2090 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,3 +18,7 @@ functions/generated_apps
 
 backend/onprem/app_sdk/apps
 *test.py
+
+shuffle-database
+*.exe
+*debug*

From 61e779d5c69471673ff58c0fb8517d97693f10d2 Mon Sep 17 00:00:00 2001
From: Harduino 
Date: Thu, 25 Feb 2021 14:37:47 +0300
Subject: [PATCH 08/57] fix workflows order on import

---
 backend/go-app/walkoff.go | 31 ++++++++++++++++++++++++++-----
 1 file changed, 26 insertions(+), 5 deletions(-)

diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go
index 5fbdb3e6..fb6f86bc 100644
--- a/backend/go-app/walkoff.go
+++ b/backend/go-app/walkoff.go
@@ -11,6 +11,7 @@ import (
 	"log"
 	"net/http"
 	"os"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -4577,8 +4578,12 @@ func setExampleresult(ctx context.Context, result AppExecutionExample) error {
 
 // Hmm, so I guess this should use uuid :(
 // Consistency PLX
-func setWorkflow(ctx context.Context, workflow Workflow, id string) error {
+func setWorkflow(ctx context.Context, workflow Workflow, id string, optionalEditedSecondsOffset ...int) error {
 	workflow.Edited = int64(time.Now().Unix())
+	if len(optionalEditedSecondsOffset) > 0 {
+		workflow.Edited += int64(optionalEditedSecondsOffset[0])
+	}
+
 	key := datastore.NameKey("workflow", id, nil)
 
 	// New struct, to not add body, author etc
@@ -6326,9 +6331,25 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string,
 // Onlyname is used to
 func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname, userId, orgId string) error {
 	var err error
+	secondsOffset := 0
 
+	// sort file names
+	filenames := []string{}
 	for _, file := range dir {
-		if len(onlyname) > 0 && file.Name() != onlyname {
+		filename := file.Name()
+		filenames = append(filenames, filename)
+	}
+	sort.Strings(filenames)
+
+	// iterate through sorted filenames
+	for _, filename := range filenames {
+		secondsOffset -= 10
+		if len(onlyname) > 0 && filename != onlyname {
+			continue
+		}
+
+		file, err := fs.Stat(filename)
+		if err != nil {
 			continue
 		}
 
@@ -6349,7 +6370,6 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra
 			}
 		case mode.IsRegular():
 			// Check the file
-			filename := file.Name()
 			if strings.HasSuffix(filename, ".json") {
 				path := fmt.Sprintf("%s%s", extra, file.Name())
 				fileReader, err := fs.Open(path)
@@ -6401,9 +6421,10 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra
 						}
 					}
 				*/
-
+				
+				log.Printf("Import workflow from file: %s", filename)
 				ctx := context.Background()
-				err = setWorkflow(ctx, workflow, workflow.ID)
+				err = setWorkflow(ctx, workflow, workflow.ID, secondsOffset)
 				if err != nil {
 					log.Printf("Failed setting (download) workflow: %s", err)
 					continue

From e4eda5367760d2c524449aba8e65329696dd9ae8 Mon Sep 17 00:00:00 2001
From: frikky 
Date: Fri, 26 Feb 2021 07:34:21 +0100
Subject: [PATCH 09/57] Remapped shared frontend imports for package sizing

---
 frontend/src/views/Admin.jsx           | 56 +++-------------------
 frontend/src/views/AngularWorkflow.jsx | 65 ++------------------------
 frontend/src/views/AppCreator.jsx      | 50 ++++++--------------
 frontend/src/views/Apps.jsx            | 45 ++++--------------
 frontend/src/views/Docs.jsx            | 32 ++++---------
 frontend/src/views/SettingsPage.jsx    |  6 +--
 frontend/src/views/Workflows.jsx       | 32 +------------
 7 files changed, 45 insertions(+), 241 deletions(-)

diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx
index 83bff401..9df961ac 100644
--- a/frontend/src/views/Admin.jsx
+++ b/frontend/src/views/Admin.jsx
@@ -1,60 +1,18 @@
 import React, { useState, useEffect } from 'react';
 
 import { makeStyles } from '@material-ui/styles';
+import { useTheme } from '@material-ui/core/styles';
 import {Link} from 'react-router-dom';
-import Paper from '@material-ui/core/Paper';
-import Card from '@material-ui/core/Card';
-import Tooltip from '@material-ui/core/Tooltip';
-import FormControlLabel from '@material-ui/core/FormControlLabel';
-import Typography from '@material-ui/core/Typography';
-import Switch from '@material-ui/core/Switch';
-import Select from '@material-ui/core/Select';
-import MenuItem from '@material-ui/core/MenuItem';
-import Divider from '@material-ui/core/Divider';
-import TextField from '@material-ui/core/TextField';
-import Button from '@material-ui/core/Button';
-import Tabs from '@material-ui/core/Tabs';
-import Tab from '@material-ui/core/Tab';
-import Grid from '@material-ui/core/Grid';
-import List from '@material-ui/core/List';
-import ListItem from '@material-ui/core/ListItem';
-import ListItemText from '@material-ui/core/ListItemText';
-import ListItemAvatar from '@material-ui/core/ListItemAvatar';
-import ListItemSecondaryAction from '@material-ui/core/ListItemSecondaryAction';
-import IconButton from '@material-ui/core/IconButton';
-import Avatar from '@material-ui/core/Avatar';
-import Zoom from '@material-ui/core/Zoom';
+
+import {Paper, Card, Tooltip, FormControlLabel, Typography, Switch, Select, MenuItem, Divider, TextField, Button, Tabs, Tab, Grid, List, ListItem, ListItemText, ListItemAvatar, ListItemSecondaryAction, IconButton, Avatar, Zoom,  Dialog, DialogTitle, DialogActions, DialogContent, CircularProgress } from '@material-ui/core';
+
+import {Edit as EditIcon, FileCopy as FileCopyIcon, Publish as PublishIcon, SelectAll as SelectAllIcon, OpenInNew as OpenInNewIcon, CloudDownload as CloudDownloadIcon, Description as DescriptionIcon, Polymer as PolymerIcon, CheckCircle as CheckCircleIcon, Close as CloseIcon, Apps as AppsIcon, Image as ImageIcon, Delete as DeleteIcon, Cached as CachedIcon, AccessibilityNew as AccessibilityNewIcon, Lock as LockIcon, Eco as EcoIcon, Schedule as ScheduleIcon, Cloud as CloudIcon, Business as BusinessIcon} from '@material-ui/icons';
+
 import { useAlert } from "react-alert";
 import Dropzone from '../components/Dropzone';
-
-import { Dialog, DialogTitle, DialogActions, DialogContent } from '@material-ui/core';
-import { useTheme } from '@material-ui/core/styles';
 import HandlePayment from './HandlePayment'
 import OrgHeader from '../components/OrgHeader'
 
-import CircularProgress from '@material-ui/core/CircularProgress';
-import EditIcon from '@material-ui/icons/Edit';
-import FileCopyIcon from '@material-ui/icons/FileCopy';
-import PublishIcon from '@material-ui/icons/Publish';
-import SelectAllIcon from '@material-ui/icons/SelectAll';
-import OpenInNewIcon from '@material-ui/icons/OpenInNew';
-import CloudDownloadIcon from '@material-ui/icons/CloudDownload';
-import DescriptionIcon from '@material-ui/icons/Description';
-import PolymerIcon from '@material-ui/icons/Polymer';
-import CheckCircleIcon from '@material-ui/icons/CheckCircle';
-import CloseIcon from '@material-ui/icons/Close';
-import AppsIcon from '@material-ui/icons/Apps';
-import ImageIcon from '@material-ui/icons/Image';
-import DeleteIcon from '@material-ui/icons/Delete';
-import CachedIcon from '@material-ui/icons/Cached';
-import AccessibilityNewIcon from '@material-ui/icons/AccessibilityNew';
-import LockIcon from '@material-ui/icons/Lock';
-import EcoIcon from '@material-ui/icons/Eco';
-import ScheduleIcon from '@material-ui/icons/Schedule';
-import CloudIcon from '@material-ui/icons/Cloud';
-import BusinessIcon from '@material-ui/icons/Business';
-
-
 const useStyles = makeStyles({
 	notchedOutline: {
 		borderColor: "#f85a3e !important"
@@ -1645,7 +1603,7 @@ const Admin = (props) => {
 				}
 
 					

-						
+
: null diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 79979409..8b10e7ff 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -2,74 +2,15 @@ import React, {useState, useEffect, useLayoutEffect} from 'react'; import { useInterval } from 'react-powerhooks'; import uuid from "uuid"; - import {Link} from 'react-router-dom'; import { Prompt } from 'react-router' -import TextField from '@material-ui/core/TextField'; -import Drawer from '@material-ui/core/Drawer'; -import Button from '@material-ui/core/Button'; -import Paper from '@material-ui/core/Paper'; -import Grid from '@material-ui/core/Grid'; -import Tabs from '@material-ui/core/Tabs'; -import InputAdornment from '@material-ui/core/InputAdornment'; -import Tab from '@material-ui/core/Tab'; -import ButtonBase from '@material-ui/core/ButtonBase'; -import Tooltip from '@material-ui/core/Tooltip'; -import Select from '@material-ui/core/Select'; -import MenuItem from '@material-ui/core/MenuItem'; -import Divider from '@material-ui/core/Divider'; -import Dialog from '@material-ui/core/Dialog'; -import Modal from '@material-ui/core/Modal'; -import DialogActions from '@material-ui/core/DialogActions'; -import DialogTitle from '@material-ui/core/DialogTitle'; -import InputLabel from '@material-ui/core/InputLabel'; -import DialogContent from '@material-ui/core/DialogContent'; -import FormControl from '@material-ui/core/FormControl'; -import IconButton from '@material-ui/core/IconButton'; -import Menu from '@material-ui/core/Menu'; -import Input from '@material-ui/core/Input'; -import FormGroup from '@material-ui/core/FormGroup'; -import FormControlLabel from '@material-ui/core/FormControlLabel'; -import Typography from '@material-ui/core/Typography'; -import Checkbox from '@material-ui/core/Checkbox'; -import Breadcrumbs from '@material-ui/core/Breadcrumbs'; -import CircularProgress from '@material-ui/core/CircularProgress'; -import Switch from '@material-ui/core/Switch'; import ReactJson from 'react-json-view' import { useBeforeunload } from 'react-beforeunload'; import NestedMenuItem from "material-ui-nested-menu-item"; -import Fade from '@material-ui/core/Fade'; -import ArrowUpwardIcon from '@material-ui/icons/ArrowUpward'; -import VisibilityIcon from '@material-ui/icons/Visibility'; -import DoneIcon from '@material-ui/icons/Done'; -import CloseIcon from '@material-ui/icons/Close'; -import ErrorIcon from '@material-ui/icons/Error'; -import FindReplaceIcon from '@material-ui/icons/FindReplace'; -import ArrowLeftIcon from '@material-ui/icons/ArrowLeft'; -import CachedIcon from '@material-ui/icons/Cached'; -import AddIcon from '@material-ui/icons/Add'; -import DirectionsRunIcon from '@material-ui/icons/DirectionsRun'; -import PolymerIcon from '@material-ui/icons/Polymer'; -import FormatListNumberedIcon from '@material-ui/icons/FormatListNumbered'; -import CreateIcon from '@material-ui/icons/Create'; -import PlayArrowIcon from '@material-ui/icons/PlayArrow'; -import AspectRatioIcon from '@material-ui/icons/AspectRatio'; -import MoreVertIcon from '@material-ui/icons/MoreVert'; -import AppsIcon from '@material-ui/icons/Apps'; -import ScheduleIcon from '@material-ui/icons/Schedule'; -import FavoriteBorderIcon from '@material-ui/icons/FavoriteBorder'; -import PauseIcon from '@material-ui/icons/Pause'; -import DeleteIcon from '@material-ui/icons/Delete'; -import AddCircleOutlineIcon from '@material-ui/icons/AddCircleOutline'; -import SaveIcon from '@material-ui/icons/Save'; -import KeyboardArrowLeftIcon from '@material-ui/icons/KeyboardArrowLeft'; -import KeyboardArrowRightIcon from '@material-ui/icons/KeyboardArrowRight'; -import ArrowBackIcon from '@material-ui/icons/ArrowBack'; -import SettingsIcon from '@material-ui/icons/Settings'; -import LockOpenIcon from '@material-ui/icons/LockOpen'; -import ExpandMoreIcon from '@material-ui/icons/ExpandMore'; -import VpnKeyIcon from '@material-ui/icons/VpnKey'; +import {TextField, Drawer, Button, Paper, Grid, Tabs, InputAdornment, Tab, ButtonBase, Tooltip, Select, MenuItem, Divider, Dialog, Modal, DialogActions, DialogTitle, InputLabel, DialogContent, FormControl, IconButton, Menu, Input, FormGroup, FormControlLabel, Typography, Checkbox, Breadcrumbs, CircularProgress, Switch, Fade} from '@material-ui/core'; + +import {ArrowUpward as ArrowUpwardIcon, Visibility as VisibilityIcon, Done as DoneIcon, Close as CloseIcon, Error as ErrorIcon, FindReplace as FindreplaceIcon, ArrowLeft as ArrowLeftIcon, Cached as CachedIcon, DirectionsRun as DirectionsRunIcon, Add as AddIcon, Polymer as PolymerIcon, FormatListNumbered as FormatListNumberedIcon, Create as CreateIcon, PlayArrow as PlayArrowIcon, AspectRatio as AspectRatioIcon, MoreVert as MoreVertIcon, Apps as AppsIcon, Schedule as ScheduleIcon, FavoriteBorder as FavoriteBorderIcon, Pause as PauseIcon, Delete as DeleteIcon, AddCircleOutline as AddCircleOutlineIcon, Save as SaveIcon, KeyboardArrowLeft as KeyboardArrowLeftIcon, KeyboardArrowRight as KeyboardArrowRightIcon, ArrowBack as ArrowBackIcon, Settings as SettingsIcon, LockOpen as LockOpenIcon, ExpandMore as ExpandMoreIcon, VpnKey as VpnKeyIcon} from '@material-ui/icons'; import * as cytoscape from 'cytoscape'; import * as edgehandles from 'cytoscape-edgehandles'; diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 645c77d9..6d62055e 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -2,33 +2,13 @@ import React, {useState, useEffect} from 'react'; import { makeStyles } from '@material-ui/styles'; import {BrowserView, MobileView} from "react-device-detect"; +import {Paper, Typography, FormControlLabel, Button, Divider, Select, MenuItem, FormControl, Switch, Dialog, DialogTitle, DialogContent, DialogActions, TextField, Tooltip, Breadcrumbs, CircularProgress, Chip} from '@material-ui/core'; +import {CheckCircle as CheckCircleIcon, AttachFile as AttachFileIcon, Apps as AppsIcon, ErrorOutline as ErrorOutlineIcon} from '@material-ui/icons'; + + import {Link} from 'react-router-dom'; -import Paper from '@material-ui/core/Paper'; -import Typography from '@material-ui/core/Typography'; -import FormControlLabel from '@material-ui/core/FormControlLabel'; -import Button from '@material-ui/core/Button'; -import Divider from '@material-ui/core/Divider'; -import Select from '@material-ui/core/Select'; -import MenuItem from '@material-ui/core/MenuItem'; -import FormControl from '@material-ui/core/FormControl'; -import Switch from '@material-ui/core/Switch'; -import Dialog from '@material-ui/core/Dialog'; -import DialogTitle from '@material-ui/core/DialogTitle'; -import DialogContent from '@material-ui/core/DialogContent'; -import DialogActions from '@material-ui/core/DialogActions'; -import TextField from '@material-ui/core/TextField'; -import Tooltip from '@material-ui/core/Tooltip'; -import CheckCircleIcon from '@material-ui/icons/CheckCircle'; -import AttachFileIcon from '@material-ui/icons/AttachFile'; -import Breadcrumbs from '@material-ui/core/Breadcrumbs'; -import AppsIcon from '@material-ui/icons/Apps'; -import CircularProgress from '@material-ui/core/CircularProgress'; - -import Chip from '@material-ui/core/Chip'; -import ChipInput from 'material-ui-chip-input' - import YAML from 'yaml' -import ErrorOutline from '@material-ui/icons/ErrorOutline'; +import ChipInput from 'material-ui-chip-input' import { useAlert } from "react-alert"; import words from "shellwords" @@ -340,12 +320,10 @@ const AppCreator = (props) => { return response.json() }) .then((responseJson) => { - console.log("THE BODY IS HERE") setIsAppLoaded(true) if (!responseJson.success) { alert.error("Failed to verify") } else{ - console.log("HMM 2") var jsonvalid = false var tmpvalue = "" try { @@ -1303,13 +1281,13 @@ const AppCreator = (props) => { id: 'outlined-age-simple', }} > - {apikeySelection.map(data => { + {apikeySelection.map((data, index) => { if (data === undefined) { return null } return ( - + {data} )} @@ -1327,7 +1305,7 @@ const AppCreator = (props) => { const requiredColor = data.required === true ? "green" : "red" //const required = data.required === true ?
{data.required.toString()}
:
{flipRequired(index)}} style={{display: "inline", color: "red", cursor: "pointer"}}>{data.required.toString()}
return ( - +
{flipRequired(index)}}> Required:
{data.required.toString()}
@@ -1367,7 +1345,7 @@ const AppCreator = (props) => { {actions.slice(0,actionAmount).map((data, index) => { var error = data.errors.length > 0 ? - + : @@ -1391,7 +1369,7 @@ const AppCreator = (props) => { const url = data.url const hasFile = data["file_field"] !== undefined && data["file_field"] !== null && data["file_field"].length > 0 return ( - + {error}
{ @@ -2057,8 +2035,8 @@ const AppCreator = (props) => { value={newWorkflowCategories.length === 0 ? "Select a category" : newWorkflowCategories[0]} style={{backgroundColor: inputColor, color: "white", height: "50px"}} > - {categories.map(data => ( - + {categories.map((data, index) => ( + {data} ))} @@ -2332,8 +2310,8 @@ const AppCreator = (props) => { value={authenticationOption} style={{backgroundColor: inputColor, color: "white", height: "50px"}} > - {authenticationOptions.map(data => ( - + {authenticationOptions.map((data, index) => ( + {data} ))} diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 2ee8e716..050b1d27 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -2,42 +2,15 @@ import React, { useEffect } from 'react'; import { useInterval } from 'react-powerhooks'; -import AppsIcon from '@material-ui/icons/Apps'; -import Grid from '@material-ui/core/Grid'; -import Select from '@material-ui/core/Select'; -import Paper from '@material-ui/core/Paper'; -import Divider from '@material-ui/core/Divider'; -import ButtonBase from '@material-ui/core/ButtonBase'; -import Button from '@material-ui/core/Button'; -import TextField from '@material-ui/core/TextField'; -import FormControl from '@material-ui/core/FormControl'; -import MenuItem from '@material-ui/core/MenuItem'; -import Tooltip from '@material-ui/core/Tooltip'; -import FormControlLabel from '@material-ui/core/FormControlLabel'; -import Switch from '@material-ui/core/Switch'; -import Input from '@material-ui/core/Input'; -import YAML from 'yaml' -import {Link} from 'react-router-dom'; -import Breadcrumbs from '@material-ui/core/Breadcrumbs'; -import ReactJson from 'react-json-view' -import Chip from '@material-ui/core/Chip'; +import {Grid, Select, Paper, Divider, ButtonBase, Button, TextField, FormControl, MenuItem, Tooltip, FormControlLabel, Switch, Input, Breadcrumbs, Chip, Dialog, DialogTitle, DialogActions, DialogContent, CircularProgress} from '@material-ui/core'; +import {Apps as AppsIcon, Cached as CachedIcon, Publish as PublishIcon, CloudDownload as CloudDownloadIcon, Edit as EditIcon, Delete as DeleteIcon} from '@material-ui/icons'; + import { useTheme } from '@material-ui/core/styles'; -import CachedIcon from '@material-ui/icons/Cached'; -import CloudDownloadIcon from '@material-ui/icons/CloudDownload'; -import PublishIcon from '@material-ui/icons/Publish'; -import CloudDownload from '@material-ui/icons/CloudDownload'; -import EditIcon from '@material-ui/icons/Edit'; -import DeleteIcon from '@material-ui/icons/Delete'; - +import YAML from 'yaml' +import {Link} from 'react-router-dom'; +import ReactJson from 'react-json-view' import { useAlert } from "react-alert"; - -import Dialog from '@material-ui/core/Dialog'; -import DialogTitle from '@material-ui/core/DialogTitle'; -import DialogActions from '@material-ui/core/DialogActions'; -import DialogContent from '@material-ui/core/DialogContent'; -import CircularProgress from '@material-ui/core/CircularProgress'; - import Dropzone from '../components/Dropzone'; const surfaceColor = "#27292D" @@ -420,7 +393,7 @@ const Apps = (props) => { {data.activated && data.private_id !== undefined && data.private_id.length > 0 && data.generated ? {downloadApp(data)}}> - + : null} @@ -467,7 +440,7 @@ const Apps = (props) => { color="primary" style={{marginTop: 10, marginRight: 8}} > - + : null @@ -910,6 +883,7 @@ const Apps = (props) => { }
+ {isCloud ? null : { setCursearch(event.target.value) }} /> + }
{apps.length > 0 ? filteredApps.length > 0 ? diff --git a/frontend/src/views/Docs.jsx b/frontend/src/views/Docs.jsx index 3f0a045b..58c2983f 100644 --- a/frontend/src/views/Docs.jsx +++ b/frontend/src/views/Docs.jsx @@ -1,19 +1,12 @@ -import React, {useState, useEffect} from 'react'; +import React, {useState,} from 'react'; import { useTheme } from '@material-ui/core/styles'; -import Divider from '@material-ui/core/Divider'; import ReactMarkdown from 'react-markdown'; import {BrowserView, MobileView} from "react-device-detect"; -import Button from '@material-ui/core/Button'; -import Menu from '@material-ui/core/Menu'; -import MenuItem from '@material-ui/core/MenuItem'; -import Typography from '@material-ui/core/Typography'; -import Paper from '@material-ui/core/Paper'; -import List from '@material-ui/core/List'; -import ListItem from '@material-ui/core/ListItem'; - import {Link} from 'react-router-dom'; +import {Divider, Button, Menu, MenuItem, Typography, Paper, List} from '@material-ui/core'; + const Body = { maxWidth: '1000px', minWidth: '768px', @@ -31,13 +24,13 @@ const hrefStyle = { } const Docs = (props) => { - const { isLoaded, globalUrl, selectedDoc, serverside, isMobile, update} = props; + const { globalUrl, selectedDoc, serverside, isMobile, } = props; const theme = useTheme(); const [data, setData] = useState(""); const [firstrequest, setFirstrequest] = useState(true); const [list, setList] = useState([]); - const [listLoaded, setListLoaded] = useState(false); + const [, setListLoaded] = useState(false); const [anchorEl, setAnchorEl] = React.useState(null); const [baseUrl, setBaseUrl] = React.useState(serverside === true ? "" : window.location.href) @@ -153,10 +146,10 @@ const Docs = (props) => { // H# if (!found) { - var elements = parent.getElementsByTagName('h3') + elements = parent.getElementsByTagName('h3') console.log(name) - var found = false - for (var key in elements) { + found = false + for (key in elements) { const element = elements[key] if (element.innerHTML === undefined) { continue @@ -221,15 +214,6 @@ const Docs = (props) => { ) } - function TextWrapper(props) { - console.log(props) - return ( - - {props.value} - - ) - } - function Heading(props) { const element = React.createElement(`h${props.level}`, {style: {marginTop: 40}}, props.children) return ( diff --git a/frontend/src/views/SettingsPage.jsx b/frontend/src/views/SettingsPage.jsx index 31dfde8d..b1f53c1c 100644 --- a/frontend/src/views/SettingsPage.jsx +++ b/frontend/src/views/SettingsPage.jsx @@ -1,11 +1,7 @@ import React, {useState, useEffect} from 'react'; -import Paper from '@material-ui/core/Paper'; -import Button from '@material-ui/core/Button'; -import Divider from '@material-ui/core/Divider'; +import {Paper, Button, Divider, TextField} from '@material-ui/core'; import {Link} from 'react-router-dom'; - -import TextField from '@material-ui/core/TextField'; import { useAlert } from "react-alert"; import { useTheme } from '@material-ui/core/styles'; diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 05bc5196..1344514d 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -1,31 +1,9 @@ import React, { useEffect} from 'react'; import { useInterval } from 'react-powerhooks'; -import Grid from '@material-ui/core/Grid'; -import Paper from '@material-ui/core/Paper'; -import Tooltip from '@material-ui/core/Tooltip'; -import Divider from '@material-ui/core/Divider'; -import Button from '@material-ui/core/Button'; -import TextField from '@material-ui/core/TextField'; -import FormControl from '@material-ui/core/FormControl'; -import IconButton from '@material-ui/core/IconButton'; -import Menu from '@material-ui/core/Menu'; -import MenuItem from '@material-ui/core/MenuItem'; -import FormControlLabel from '@material-ui/core/FormControlLabel'; -import Chip from '@material-ui/core/Chip'; -import Switch from '@material-ui/core/Switch'; -import Typography from '@material-ui/core/Typography'; -import Zoom from '@material-ui/core/Zoom'; +import {Grid, Paper, Tooltip, Divider, Button, TextField, FormControl, IconButton, Menu, MenuItem, FormControlLabel, Chip, Switch, Typography, Zoom, CircularProgress, Dialog, DialogTitle, DialogActions, DialogContent} from '@material-ui/core'; +import {Cached as CachedIcon, GetApp as GetAppIcon, Apps as AppsIcon, Edit as EditIcon, MoreVert as MoreVertIcon, PlayArrow as PlayArrowIcon, Add as AddIcon, Publish as PublishIcon, CloudDownload as CloudDownloadIcon} from '@material-ui/icons'; -import CircularProgress from '@material-ui/core/CircularProgress'; -import CachedIcon from '@material-ui/icons/Cached'; -import GetAppIcon from '@material-ui/icons/GetApp'; -import AppsIcon from '@material-ui/icons/Apps'; -import EditIcon from '@material-ui/icons/Edit'; -import MoreVertIcon from '@material-ui/icons/MoreVert'; -import PlayArrowIcon from '@material-ui/icons/PlayArrow'; -import AddIcon from '@material-ui/icons/Add'; -import PublishIcon from '@material-ui/icons/Publish'; //import JSONPretty from 'react-json-pretty'; //import JSONPrettyMon from 'react-json-pretty/dist/monikai' import ReactJson from 'react-json-view' @@ -35,12 +13,6 @@ import {Link} from 'react-router-dom'; import { useAlert } from "react-alert"; import ChipInput from 'material-ui-chip-input' -import Dialog from '@material-ui/core/Dialog'; -import DialogTitle from '@material-ui/core/DialogTitle'; -import DialogActions from '@material-ui/core/DialogActions'; -import DialogContent from '@material-ui/core/DialogContent'; -import CloudDownloadIcon from '@material-ui/icons/CloudDownload'; - const inputColor = "#383B40" const surfaceColor = "#27292D" From 403648a5769d86e5007fd46743902703ce6346e0 Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 26 Feb 2021 08:06:45 +0100 Subject: [PATCH 10/57] #275: Added check for whether the ID exists or not instead of opening new FD's --- functions/onprem/worker/worker.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 259b3f06..7a11bb0c 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -52,6 +52,10 @@ var containerId string // form container id of current running container func getThisContainerId() string { + if len(containerId) > 0 { + return containerId + } + id := "" cmd := fmt.Sprintf("cat /proc/self/cgroup | grep memory | tail -1 | cut -d/ -f3 | grep -o -E '[0-9A-z]{64}'") out, err := exec.Command("bash", "-c", cmd).Output() From 5658849df8febb84f2bf08948a68f9e8db9a7fb3 Mon Sep 17 00:00:00 2001 From: amitk Date: Sun, 28 Feb 2021 09:07:48 +0530 Subject: [PATCH 11/57] App icon changes and new package for app icon crop --- frontend/package.json | 1 + frontend/src/views/AppCreator.jsx | 130 +++++++++++++++++++++++++++++- 2 files changed, 129 insertions(+), 2 deletions(-) diff --git a/frontend/package.json b/frontend/package.json index bb24de65..e3b88353 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -35,6 +35,7 @@ "react": "^16.14.0", "react-alert": "^5.5.0", "react-alert-template-basic": "^1.0.0", + "react-avatar-editor": "^11.1.0", "react-beforeunload": "^2.2.1", "react-chartjs-2": "^2.8.0", "react-cookie": "^4.0.1", diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 5b4cb70f..27d57a24 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -32,6 +32,13 @@ import ErrorOutline from '@material-ui/icons/ErrorOutline'; import { useAlert } from "react-alert"; import words from "shellwords" +import AvatarEditor from 'react-avatar-editor'; +import AddAPhotoIcon from '@material-ui/icons/AddAPhoto'; +import AddAPhotoOutlinedIcon from '@material-ui/icons/AddAPhotoOutlined'; +import ZoomInOutlinedIcon from '@material-ui/icons/ZoomInOutlined'; +import ZoomOutOutlinedIcon from '@material-ui/icons/ZoomOutOutlined'; +import LoopIcon from '@material-ui/icons/Loop'; + const surfaceColor = "#27292D" const inputColor = "#383B40" @@ -65,6 +72,18 @@ const boxStyle = { backgroundColor: surfaceColor, } +const dividerStyle = { + marginBottom: "10px", + marginTop: "10px", + height: "1px", + width: "100%", + backgroundColor: "grey", +} + +const appIconStyle = { + marginLeft: "5px", +} + const useStyles = makeStyles({ notchedOutline: { borderColor: "#f85a3e !important" @@ -2122,8 +2141,114 @@ const AppCreator = (props) => { //
: // - const imageData = file.length > 0 ? file : fileBase64 + const [imageUploadError, setImageUploadError] = useState(""); + const [openImageModal, setOpenImageModal] = useState(""); + const [scale, setScale] = useState(1); + const [rotate, setRotatation] = useState(0); + const [disableImageUpload, setDisableImageUpload] = useState(true); + + let imageData = fileBase64; + let croppedData = file.length > 0 ? file : fileBase64 + const imageInfo = + + const zoomIn = () => { + setScale(scale+0.1); + } + const zoomOut = () => { + setScale(scale-0.1); + } + const rotatation = () => { + setRotatation(rotate+10); + } + + const onPositionChange = () => { + setDisableImageUpload(false); + } + + const onCancelSaveAppIcon = () => { + setFile(""); + setOpenImageModal(false) + setImageUploadError("") + } + + let editor; + const setEditorRef = (imgEditor) => { editor = imgEditor; } + + const onSaveAppIcon = () => { + if(editor){ + setFile(""); + const canvas = editor.getImageScaledToCanvas(); + setFileBase64(canvas.toDataURL()); + setOpenImageModal(false) + setDisableImageUpload(true); + } + } + + const errorText = imageUploadError.length > 0 ?
Error: {imageUploadError}
: null + const imageUploadModalView = openImageModal ? + + +
Upload App Icon
 + {errorText} + + setRotatation(0)} + /> + + + + + + + + + + + + + + + + + + + + + + : null; // Random names for type & autoComplete. Didn't research :^) const landingpageDataBrowser = @@ -2139,12 +2264,13 @@ const AppCreator = (props) => { {name} + {imageUploadModalView}

General information

Click here to learn more about app creation
-
{upload.click()}}> +
{setOpenImageModal(true)}}> upload = ref} onChange={editHeaderImage} /> {imageInfo}
From fc7a414ee67912665f26e2647bf13af6ec03ccf3 Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 1 Mar 2021 01:51:12 +0100 Subject: [PATCH 12/57] Fixed some annoyances --- frontend/src/views/AppCreator.jsx | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 0b251832..401b0262 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -2305,13 +2305,19 @@ const AppCreator = (props) => { {imageUploadModalView} + upload = ref} onChange={editHeaderImage} />

General information

Click here to learn more about app creation
-
{setOpenImageModal(true)}}> - upload = ref} onChange={editHeaderImage} /> +
{ + if (fileBase64.length === 0) { + upload.click() + } + + setOpenImageModal(true) + }}> {imageInfo}
From f91307e0a2c4978c0520242e068ff95b8ab4149e Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 1 Mar 2021 11:57:48 +0100 Subject: [PATCH 13/57] Minor fixes to frontend --- README.md | 2 +- backend/go-app/walkoff.go | 7 +++++- frontend/src/views/AngularWorkflow.jsx | 8 +++---- frontend/src/views/Apps.jsx | 4 +++- functions/onprem/worker/Dockerfile | 12 ++++------ functions/onprem/worker/worker.go | 32 ++++++++++++++++++++++++++ 6 files changed, 50 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 895bd472..c09bd090 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ These are the main areas to contribute in: * Workflow creation (GUI & Conceptualizing) * Content Creation (Blogs, videos etc) -Contributing guidelines for Github are outlined [here](https://github.com/frikky/Shuffle/blob/master/.github/CONTRIBUTING.md). +Contributing guidelines are outlined [here](https://github.com/frikky/Shuffle/blob/master/.github/CONTRIBUTING.md). ## Contributors ![ICPL logo](https://github.com/frikky/Shuffle/blob/launch/frontend/src/assets/img/icpl_logo.png) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index fb6f86bc..9c0556db 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -195,6 +195,11 @@ type WorkflowApp struct { DocumentationUrl string `json:"documentation_url" datastore:"documentation_url"` GithubUrl string `json:"github_url" datastore:"github_url"` } + FolderMount struct { + FolderMount bool `json:"folder_mount" datastore:"folder_mount"` + SourceFolder string `json:"source_folder" datastore:"source_folder"` + DestinationFolder string `json:"destination_folder" datastore:"destination_folder"` + } Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions,noindex"` Authentication Authentication `json:"authentication" yaml:"authentication" required:false datastore:"authentication"` Tags []string `json:"tags" yaml:"tags" required:false datastore:"activated"` @@ -6421,7 +6426,7 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra } } */ - + log.Printf("Import workflow from file: %s", filename) ctx := context.Background() err = setWorkflow(ctx, workflow, workflow.ID, secondsOffset) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 8b10e7ff..4cd2a35b 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -987,7 +987,7 @@ const AngularWorkflow = (props) => { setApps(responseJson) //getAppAuthentication() - setFilteredApps(responseJson.filter(app => !internalIds.includes(app.name))) + setFilteredApps(responseJson.filter(app => !internalIds.includes(app.name) && !(!app.activated && app.generated))) setPrioritizedApps(responseJson.filter(app => internalIds.includes(app.name))) }) .catch(error => { @@ -6637,7 +6637,7 @@ const AngularWorkflow = (props) => { Actions
- {executionData.status !== undefined && executionData.status !== "ABORTED" && executionData.status !== "FINISHED" && executionData.status !== "FAILURE" && executionData.status !== "WAITING" ? : null} + {executionData.status !== undefined && executionData.status !== "ABORTED" && executionData.status !== "FINISHED" && executionData.status !== "FAILURE" && executionData.status !== "WAITING" && !(executionData.results === undefined || executionData.results === null || executionData.results.length === 0 && executionData.status === "EXECUTING")? : null}
{executionData.results === undefined || executionData.results === null || executionData.results.length === 0 && executionData.status === "EXECUTING" ? @@ -6712,7 +6712,7 @@ const AngularWorkflow = (props) => { }} name={"Results for "+data.action.label} /> - {data.action.app_name === "shuffle-subflow" ? + {data.action.app_name === "shuffle-subflow" && validate.result.success !== undefined && validate.result.success === true ? {validate.valid && data.action.parameters !== undefined && data.action.parameters !== null ? See subflow execution @@ -6724,7 +6724,7 @@ const AngularWorkflow = (props) => { } : -
+
Result  {data.result}
diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 050b1d27..99a9b0bb 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -201,7 +201,8 @@ const Apps = (props) => { }) .then((responseJson) => { //console.log("Apps: ", responseJson) - responseJson = sortByKey(responseJson, "large_image") + //responseJson = sortByKey(responseJson, "large_image") + responseJson = sortByKey(responseJson, "generated") setApps(responseJson) setFilteredApps(responseJson) @@ -781,6 +782,7 @@ const Apps = (props) => { var tmpapps = searchableApps.filter(data => data.name.toLowerCase().includes(searchfield) || data.description.toLowerCase().includes(searchfield)) newapps.push(...tmpapps) + console.log(newapps) setFilteredApps(newapps) //if ((newapps.length === 0 || searchBackend) && !appSearchLoading) { diff --git a/functions/onprem/worker/Dockerfile b/functions/onprem/worker/Dockerfile index a6d82d61..a95b099e 100644 --- a/functions/onprem/worker/Dockerfile +++ b/functions/onprem/worker/Dockerfile @@ -2,20 +2,16 @@ FROM golang:1.16.0-buster as builder WORKDIR /app +COPY worker.go /app/worker.go +RUN go env -w GO111MODULE=auto + RUN go get -u github.com/docker/docker/api/types RUN go get -u github.com/docker/docker/api/types/container RUN go get -u github.com/docker/docker/client RUN go get -u github.com/gorilla/mux RUN go get -u github.com/patrickmn/go-cache -COPY worker.go /app/worker.go -RUN go env -w GO111MODULE=auto && \ - go get github.com/docker/docker/api/types && \ - go get github.com/docker/docker/api/types/container && \ - go get github.com/docker/docker/client && \ - go get github.com/gorilla/mux && \ - go get github.com/patrickmn/go-cache && \ - CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker . +RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker . FROM alpine:3.12 diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 7a11bb0c..1672fb22 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -18,6 +18,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/mount" dockerclient "github.com/docker/docker/client" "github.com/gorilla/mux" @@ -877,6 +878,37 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] hostConfig.AutoRemove = true } + // FIXME: Add proper foldermounts here + //log.Printf("\n\nPRE FOLDERMOUNT\n\n") + //volumeBinds := []string{"/tmp/shuffle-mount:/rules"} + //volumeBinds := []string{"/tmp/shuffle-mount:/rules"} + volumeBinds := []string{} + if len(volumeBinds) > 0 { + log.Printf("[INFO] Setting up binds for container!") + hostConfig.Binds = volumeBinds + hostConfig.Mounts = []mount.Mount{} + for _, bind := range volumeBinds { + if !strings.Contains(bind, ":") || strings.Contains(bind, "..") || strings.HasPrefix(bind, "~") { + log.Printf("[WARNING] Bind %s is invalid.", bind) + continue + } + + log.Printf("[INFO] Appending bind %s", bind) + bindSplit := strings.Split(bind, ":") + sourceFolder := bindSplit[0] + destinationFolder := bindSplit[0] + hostConfig.Mounts = append(hostConfig.Mounts, mount.Mount{ + Type: mount.TypeBind, + Source: sourceFolder, + Target: destinationFolder, + }) + } + } else { + log.Printf("[WARNING] No mounted folders") + } + // hostConfig.Binds = volumeBinds + //} + config := &container.Config{ Image: image, Env: env, From 030f2e2f333f8a313152bb28f7c626a877c625f0 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 2 Mar 2021 08:58:08 +0100 Subject: [PATCH 14/57] #284: Quickfix button to see all actions --- frontend/src/views/AppCreator.jsx | 33 ++++++++++++++++++------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 6d62055e..5ed4bcb0 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -2067,8 +2067,24 @@ const AppCreator = (props) => {
const actionView = -
-

Actions ({actions.length})

+
+
+ {actionAmount > 0 && actionAmount < actions.length ? + + : null} +
+

Actions ({actionAmount} / {actions.length})

Actions are the tasks performed by an app. Read more about actions and apps here.
@@ -2091,18 +2107,7 @@ const AppCreator = (props) => { setActionsModalOpen(true) }}>New action {/* - {actionAmount} {actions.length} - {actionAmount > 0 && actionAmount < actions.length ? null : - - } + {actionAmount} {actions.length} */}
From 3c6e973f3d80551aac4cbce7fc9d55e4a8bc33c3 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 2 Mar 2021 08:58:32 +0100 Subject: [PATCH 15/57] BUGFIX: Subflows and workflow abort issues --- backend/go-app/main.go | 12 +- backend/go-app/walkoff.go | 82 +++++++++++++- frontend/src/defaultCytoscapeStyle.js | 2 +- frontend/src/views/AngularWorkflow.jsx | 28 +++-- functions/onprem/orborus/orborus.go | 8 +- functions/onprem/worker/build.sh | 2 +- functions/onprem/worker/worker.go | 148 ++++++++++++++++--------- 7 files changed, 205 insertions(+), 77 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 7cd4e069..d1d5c5aa 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3559,9 +3559,8 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { // bodyWrapper = string(parsedBody) //} - url := &url.URL{} newRequest := &http.Request{ - URL: url, + URL: &url.URL{}, Method: "POST", Body: ioutil.NopCloser(bytes.NewReader(b)), } @@ -7234,9 +7233,10 @@ func runInit(ctx context.Context) { if count == 0 && err == nil && len(activeOrgs) == 1 { log.Printf("Setting up environment with org %s", activeOrgs[0].Id) item := Environment{ - Name: "Shuffle", - Type: "onprem", - OrgId: activeOrgs[0].Id, + Name: "Shuffle", + Type: "onprem", + OrgId: activeOrgs[0].Id, + Default: true, } err = setEnvironment(ctx, &item) @@ -7419,6 +7419,7 @@ func runInit(ctx context.Context) { log.Printf("Failed getting schedules during service init: %s", err) } else { log.Printf("Setting up %d schedule(s)", len(schedules)) + url := &url.URL{} for _, schedule := range schedules { if schedule.Environment == "cloud" { log.Printf("Skipping cloud schedule") @@ -7428,6 +7429,7 @@ func runInit(ctx context.Context) { //log.Printf("Schedule: %#v", schedule) job := func() { request := &http.Request{ + URL: url, Method: "POST", Body: ioutil.NopCloser(strings.NewReader(schedule.WrappedArgument)), } diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 9c0556db..0588c6bd 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -10,6 +10,7 @@ import ( "io/ioutil" "log" "net/http" + "net/url" "os" "sort" "strconv" @@ -650,6 +651,7 @@ func createSchedule(ctx context.Context, scheduleId, workflowId, name, startNode log.Printf("WRAPPER BODY: \n%s", bodyWrapper) job := func() { request := &http.Request{ + URL: &url.URL{}, Method: "POST", Body: ioutil.NopCloser(strings.NewReader(bodyWrapper)), } @@ -1014,7 +1016,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Success"}`))) return } else { - //log.Printf("[WARNING] Failed to handle new execution variant: %s", err) + //log.Printf("[WARNING] Handling other execution variant: %s", err) } var actionResult ActionResult @@ -2899,6 +2901,8 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) { return } + //log.Printf("\n\nINSIDE ABORT\n\n") + location := strings.Split(request.URL.String(), "/") var fileId string if location[1] == "api" { @@ -2974,6 +2978,7 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) { workflowExecution.CompletedAt = int64(time.Now().Unix()) workflowExecution.Status = "ABORTED" + log.Printf("[INFO] Running shutdown of %s", workflowExecution.ExecutionId) lastResult := "" newResults := []ActionResult{} @@ -2996,6 +3001,79 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) { workflowExecution.Result = lastResult } + addResult := true + for _, result := range workflowExecution.Results { + if result.Status != "SKIPPED" { + addResult = false + } + } + + extra := 0 + for _, trigger := range workflowExecution.Workflow.Triggers { + //log.Printf("Appname trigger (0): %s", trigger.AppName) + if trigger.AppName == "User Input" || trigger.AppName == "Shuffle Workflow" { + extra += 1 + } + } + + parsedReason := "An error occurred during execution of this node" + reason, reasonok := request.URL.Query()["reason"] + if reasonok { + parsedReason = reason[0] + } + + if len(workflowExecution.Results) == 0 || addResult { + newaction := Action{ + ID: workflowExecution.Start, + } + + for _, action := range workflowExecution.Workflow.Actions { + if action.ID == workflowExecution.Start { + newaction = action + break + } + } + + workflowExecution.Results = append(workflowExecution.Results, ActionResult{ + Action: newaction, + ExecutionId: workflowExecution.ExecutionId, + Authorization: workflowExecution.Authorization, + Result: parsedReason, + StartedAt: workflowExecution.StartedAt, + CompletedAt: workflowExecution.StartedAt, + Status: "FAILURE", + }) + } else if len(workflowExecution.Results) >= len(workflowExecution.Workflow.Actions)+extra { + log.Printf("[INFO] DONE - Nothing to add during abort!") + } else { + //log.Printf("VALIDATING INPUT!") + node, nodeok := request.URL.Query()["node"] + if nodeok { + nodeId := node[0] + log.Printf("[INFO] Found abort node %s", nodeId) + newaction := Action{ + ID: nodeId, + } + + for _, action := range workflowExecution.Workflow.Actions { + if action.ID == nodeId { + newaction = action + break + } + } + + workflowExecution.Results = append(workflowExecution.Results, ActionResult{ + Action: newaction, + ExecutionId: workflowExecution.ExecutionId, + Authorization: workflowExecution.Authorization, + Result: parsedReason, + StartedAt: workflowExecution.StartedAt, + CompletedAt: workflowExecution.StartedAt, + Status: "FAILURE", + }) + } + } + err = setWorkflowExecution(ctx, *workflowExecution, true) if err != nil { log.Printf("Error saving workflow execution for updates when aborting %s: %s", topic, err) @@ -3517,7 +3595,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf //log.Printf("SHOULD SET TRIGGER %s TO BE SKIPPED", trigger.ID) curaction := Action{ - AppName: trigger.AppName, + AppName: "shuffle-subflow", AppVersion: trigger.AppVersion, Label: trigger.Label, Name: trigger.Name, diff --git a/frontend/src/defaultCytoscapeStyle.js b/frontend/src/defaultCytoscapeStyle.js index e0732bac..ab8fae1e 100644 --- a/frontend/src/defaultCytoscapeStyle.js +++ b/frontend/src/defaultCytoscapeStyle.js @@ -183,7 +183,7 @@ const data = [{ css: { 'background-color': "#f85a3e", 'border-color': '#f85a3e', - 'border-width': '5px', + 'border-width': '8px', 'transition-property': 'border-width', 'transition-duration': '0.25s', }, diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 4cd2a35b..0fc7178b 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -786,7 +786,7 @@ const AngularWorkflow = (props) => { curelements[i].addClass("not-executing-highlight") } - if (executionArgument.length > 0) { + if (executionArgument !== undefined && executionArgument !== null && executionArgument.length > 0) { //alert.success("Starting execution WITH an execution argument") } else { //alert.success("Starting execution") @@ -2794,8 +2794,10 @@ const AngularWorkflow = (props) => { } var exampledata = item.example === undefined ? "" : item.example + console.log("EXAMPLE: ", exampledata) // Find previous execution and their variables - if (exampledata === "" && workflowExecutions.length > 0) { + //exampledata === "" && + if (workflowExecutions.length > 0) { // Look for the ID const found = false for (var key in workflowExecutions) { @@ -3172,8 +3174,8 @@ const AngularWorkflow = (props) => { { setMenuPosition({ - top: event.pageY, - left: event.pageX, + top: event.pageY+10, + left: event.pageX+10, }) setShowDropdownNumber(count) setShowDropdown(true) @@ -3235,8 +3237,8 @@ const AngularWorkflow = (props) => { { setMenuPosition({ - top: event.pageY, - left: event.pageX, + top: event.pageY+10, + left: event.pageX+10, }) setShowDropdownNumber(count) setShowDropdown(true) @@ -3586,7 +3588,7 @@ const AngularWorkflow = (props) => { // FIXME: Should be recursive in here const icon = pathdata.type === "value" ? : pathdata.type === "list" ? : return ( - {}} + {}} onClick={() => { handleItemClick([innerdata, pathdata]) }} @@ -6663,19 +6665,25 @@ const AngularWorkflow = (props) => { if (action !== undefined && action !== null) { imgSrc = action.large_image } + + /* + if (imgSrc.length === 0) { + console.log("CHECK IF ITS A + } + */ } var actionimg = curapp === null ? null : - {data.action.app_name} + {data.action.app_name} if (triggers.length > 2) { if (data.action.app_name === "shuffle-subflow") { - actionimg = {"Shuffle + actionimg = {"Shuffle } if (data.action.app_name === "User Input") { - actionimg = {"Shuffle + actionimg = {"Shuffle } } diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index a2e3d716..30d6394a 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -101,7 +101,7 @@ func getThisContainerId() { log.Printf("[INFO] Running containerized in Docker!") default: - fCol = "3" // for backward-compatibility with production + fCol = "0" // for backward-compatibility with production log.Printf("[WARNING] RUNNING_MODE not set - defaulting to Docker (NOT Kubernetes).") } @@ -119,8 +119,10 @@ func getThisContainerId() { //docker-76c537e9a4b7c7233011f5d70e6b7f2d600b6413ac58a96519b8dca7a3f7117a.scope } } else { - containerId = "shuffle-orborus" - log.Printf("[WARNING] Failed getting container ID: %s", err) + if fCol != "0" { + containerId = "shuffle-orborus" + log.Printf("[WARNING] Failed getting container ID: %s", err) + } } } diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh index ba1b2e38..907f053e 100644 --- a/functions/onprem/worker/build.sh +++ b/functions/onprem/worker/build.sh @@ -10,5 +10,5 @@ docker build . -t frikky/shuffle:$NAME -t frikky/shuffle:$NAME_$VERSION -t docke #docker push frikky/shuffle:$NAME_$VERSION #docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION #docker tag frikky/shuffle:0.8.51 ghcr.io/frikky/shuffle-worker:0.8.5 -docker tag frikky/shuffle:$NAME ghcr.io/frikky/shuffle-worker:0.8.52 +#docker tag frikky/shuffle:$NAME ghcr.io/frikky/shuffle-worker:0.8.52 docker push ghcr.io/frikky/$NAME:$VERSION diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 1672fb22..f3d9250a 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -11,6 +11,7 @@ import ( "log" "net" "net/http" + "net/url" "os" "os/exec" "strings" @@ -774,8 +775,22 @@ type AppExecutionExample struct { } // removes every container except itself (worker) -func shutdown(executionId, workflowId string) { +func shutdown(workflowExecution WorkflowExecution, nodeId string, reason string, handleResultSend bool) { log.Printf("[INFO] Shutdown started") + //reason := "Error in execution" + + sleepDuration := 1 + if handleResultSend { + data, err := json.Marshal(workflowExecution) + if err == nil { + sendResult(workflowExecution, data) + log.Printf("[WARNING] Sent shutdown update") + } else { + log.Printf("[WARNING] DIDNT send update") + } + + time.Sleep(time.Duration(sleepDuration) * time.Second) + } // Might not be necessary because of cleanupEnv hostconfig autoremoval if cleanupEnv == "true" && len(containerIds) > 0 { @@ -801,7 +816,20 @@ func shutdown(executionId, workflowId string) { log.Printf("[INFO] NOT cleaning up containers. IDS: %d, CLEANUP env: %s", len(containerIds), cleanupEnv) } - fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s/executions/%s/abort", baseUrl, workflowId, executionId) + fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s/executions/%s/abort", baseUrl, workflowExecution.Workflow.ID, workflowExecution.ExecutionId) + + path := fmt.Sprintf("?reason=%s", url.QueryEscape(reason)) + if len(nodeId) > 0 { + path += fmt.Sprintf("&node=%s", url.QueryEscape(nodeId)) + } + if len(environment) > 0 { + path += fmt.Sprintf("&env=%s", url.QueryEscape(environment)) + } + + //fmt.Println(url.QueryEscape(query)) + fullUrl += path + log.Printf("Abort URL: %s", fullUrl) + req, err := http.NewRequest( "GET", fullUrl, @@ -844,7 +872,6 @@ func shutdown(executionId, workflowId string) { log.Printf("[INFO] Failed abort request: %s", err) } - sleepDuration := 1 log.Printf("[INFO] Finished shutdown (after %d seconds).", sleepDuration) // Allows everything to finish in subprocesses time.Sleep(time.Duration(sleepDuration) * time.Second) @@ -931,7 +958,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] err = cli.ContainerStart(context.Background(), cont.ID, types.ContainerStartOptions{}) if err != nil { log.Printf("[ERROR] Failed to start container in environment %s: %s", environment, err) - //shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + //shutdown(workflowExecution, workflowExecution.Workflow.ID, true) return err } @@ -1202,7 +1229,7 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { if exit && len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions) { log.Printf("Shutting down.") - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, "", "", true) } // Look for the NEXT missing action @@ -1547,18 +1574,18 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { reader, err := dockercli.ImagePull(context.Background(), image, pullOptions) if err != nil { log.Printf("[ERROR] Failed getting %s. The couldn't be find locally, AND is missing.", image) - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, action.ID, err.Error(), true) } buildBuf := new(strings.Builder) _, err = io.Copy(buildBuf, reader) if err != nil { log.Printf("[ERROR] Error in IO copy: %s", err) - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, action.ID, err.Error(), true) } else { if strings.Contains(buildBuf.String(), "errorDetail") { log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), image) - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, action.ID, err.Error(), true) } log.Printf("[INFO] Successfully downloaded %s", image) @@ -1571,7 +1598,7 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { if strings.Contains(err.Error(), "No such image") { //log.Printf("[WARNING] Failed deploying %s from image %s: %s", identifier, image, err) log.Printf("[ERROR] Image doesn't exist. Shutting down") - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, action.ID, err.Error(), true) } } } @@ -1599,18 +1626,18 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { reader, err := dockercli.ImagePull(context.Background(), image, pullOptions) if err != nil { log.Printf("[ERROR] Failed getting %s. The couldn't be find locally, AND is missing.", image) - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, action.ID, err.Error(), true) } buildBuf := new(strings.Builder) _, err = io.Copy(buildBuf, reader) if err != nil { log.Printf("[ERROR] Error in IO copy: %s", err) - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, action.ID, err.Error(), true) } else { if strings.Contains(buildBuf.String(), "errorDetail") { log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), image) - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, action.ID, err.Error(), true) } log.Printf("[INFO] Successfully downloaded %s", image) @@ -1623,7 +1650,7 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { if strings.Contains(err.Error(), "No such image") { //log.Printf("[WARNING] Failed deploying %s from image %s: %s", identifier, image, err) log.Printf("[ERROR] Image doesn't exist. Shutting down") - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, action.ID, err.Error(), true) } } } @@ -1663,7 +1690,7 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { if shutdownCheck { log.Println("[INFO] BREAKING BECAUSE RESULTS IS SAME LENGTH AS ACTIONS. SHOULD CHECK ALL RESULTS FOR WHETHER THEY'RE DONE") validateFinished(workflowExecution) - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, "", "", true) } } @@ -1781,7 +1808,7 @@ func executionInit(workflowExecution WorkflowExecution) error { //reader, err := dockercli.ImagePull(context.Background(), image, pullOptions) //if err != nil { // log.Printf("Failed getting %s. The app is missing or some other issue", image) - // shutdown(workflowExecution.ExecutionId) + // shutdown(workflowExecution) //} ////io.Copy(os.Stdout, reader) @@ -1799,7 +1826,7 @@ func handleExecution(client *http.Client, req *http.Request, workflowExecution W err := executionInit(workflowExecution) if err != nil { log.Printf("[INFO] Workflow setup failed: %s", workflowExecution.ExecutionId, err) - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, "", "", true) } log.Printf("Startaction: %s", startAction) @@ -1835,7 +1862,11 @@ func handleExecution(client *http.Client, req *http.Request, workflowExecution W if newresp.StatusCode != 200 { log.Printf("[ERROR] Bad statuscode: %d, %s", newresp.StatusCode, string(body)) - //shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + + if strings.Contains(string(body), "Workflowexecution is already finished") { + shutdown(workflowExecution, "", "", false) + } + time.Sleep(time.Duration(sleepTime) * time.Second) continue } @@ -1849,13 +1880,13 @@ func handleExecution(client *http.Client, req *http.Request, workflowExecution W if workflowExecution.Status == "FINISHED" || workflowExecution.Status == "SUCCESS" { log.Printf("[INFO] Workflow %s is finished. Exiting worker.", workflowExecution.ExecutionId) - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, "", "", true) } log.Printf("[INFO] Status: %s, Results: %d, actions: %d", workflowExecution.Status, len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)+extra) if workflowExecution.Status != "EXECUTING" { log.Printf("[WARNING] Exiting as worker execution has status %s!", workflowExecution.Status) - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, "", "", true) } } @@ -2545,6 +2576,34 @@ func getWorkflowExecution(ctx context.Context, id string) (*WorkflowExecution, e return &WorkflowExecution{}, errors.New("No workflowexecution defined yet") } +func sendResult(workflowExecution WorkflowExecution, data []byte) { + fullUrl := fmt.Sprintf("%s/api/v1/streams", baseUrl) + req, err := http.NewRequest( + "POST", + fullUrl, + bytes.NewBuffer([]byte(data)), + ) + + if err != nil { + log.Printf("[ERROR] Failed creating finishing request: %s", err) + shutdown(workflowExecution, "", "", false) + } + + newresp, err := topClient.Do(req) + if err != nil { + log.Printf("[ERROR] Error running finishing request: %s", err) + shutdown(workflowExecution, "", "", false) + } + + body, err := ioutil.ReadAll(newresp.Body) + log.Printf("[INFO] BACKEND STATUS: %d", newresp.StatusCode) + if err != nil { + log.Printf("[ERROR] Failed reading body: %s", err) + } else { + log.Printf("[INFO] NEWRESP (from backend): %s", string(body)) + } +} + func validateFinished(workflowExecution WorkflowExecution) { log.Printf("[INFO] Status: %s, Actions: %d, Extra: %d, Results: %d\n", workflowExecution.Status, len(workflowExecution.Workflow.Actions), extra, len(workflowExecution.Results)) @@ -2557,34 +2616,10 @@ func validateFinished(workflowExecution WorkflowExecution) { data, err := json.Marshal(workflowExecution) if err != nil { log.Printf("[ERROR] Failed to unmarshal data for backend") - shutdown(workflowExecution.ExecutionId, "") + shutdown(workflowExecution, "", "", true) } - fullUrl := fmt.Sprintf("%s/api/v1/streams", baseUrl) - req, err := http.NewRequest( - "POST", - fullUrl, - bytes.NewBuffer([]byte(data)), - ) - - if err != nil { - log.Printf("[ERROR] Failed creating finishing request: %s", err) - shutdown(workflowExecution.ExecutionId, "") - } - - newresp, err := topClient.Do(req) - if err != nil { - log.Printf("[ERROR] Error running finishing request: %s", err) - shutdown(workflowExecution.ExecutionId, "") - } - - body, err := ioutil.ReadAll(newresp.Body) - log.Printf("[INFO] BACKEND STATUS: %d", newresp.StatusCode) - if err != nil { - log.Printf("[ERROR] Failed reading body: %s", err) - } else { - log.Printf("[INFO] NEWRESP (from backend): %s", string(body)) - } + sendResult(workflowExecution, data) } } @@ -2649,8 +2684,9 @@ func setWorkflowExecution(ctx context.Context, workflowExecution WorkflowExecuti handleExecutionResult(workflowExecution) validateFinished(workflowExecution) if dbSave { - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, "", "", false) } + return nil } @@ -2691,7 +2727,7 @@ func webserverSetup(workflowExecution WorkflowExecution) net.Listener { listener, err := getAvailablePort() if err != nil { log.Printf("Failed to created listener: %s", err) - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, "", "", true) } port := listener.Addr().(*net.TCPAddr).Port @@ -2754,14 +2790,17 @@ func main() { log.Printf("[INFO] Running normal execution with auth %s and ID %s", authorization, executionId) } + workflowExecution := WorkflowExecution{ + ExecutionId: executionId, + } if len(authorization) == 0 { log.Println("[INFO] No AUTHORIZATION key set in env") - shutdown(executionId, "") + shutdown(workflowExecution, "", "", false) } if len(executionId) == 0 { log.Println("[INFO] No EXECUTIONID key set in env") - shutdown(executionId, "") + shutdown(workflowExecution, "", "", false) } data = fmt.Sprintf(`{"execution_id": "%s", "authorization": "%s"}`, executionId, authorization) @@ -2774,7 +2813,7 @@ func main() { if err != nil { log.Println("[ERROR] Failed making request builder for backend") - shutdown(executionId, "") + shutdown(workflowExecution, "", "", true) } topClient = client @@ -2802,7 +2841,6 @@ func main() { continue } - var workflowExecution WorkflowExecution err = json.Unmarshal(body, &workflowExecution) if err != nil { log.Printf("[ERROR] Failed workflowExecution unmarshal: %s", err) @@ -2837,7 +2875,7 @@ func main() { err := executionInit(workflowExecution) if err != nil { log.Printf("[INFO] Workflow setup failed: %s", workflowExecution.ExecutionId, err) - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, "", "", true) } go func() { @@ -2856,7 +2894,7 @@ func main() { if workflowExecution.Status == "FINISHED" || workflowExecution.Status == "SUCCESS" { log.Printf("[INFO] Workflow %s is finished. Exiting worker.", workflowExecution.ExecutionId) - shutdown(executionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, "", "", true) } if workflowExecution.Status == "EXECUTING" || workflowExecution.Status == "RUNNING" { @@ -2864,11 +2902,11 @@ func main() { err = handleExecution(client, req, workflowExecution) if err != nil { log.Printf("[INFO] Workflow %s is finished: %s", workflowExecution.ExecutionId, err) - shutdown(executionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, "", "", true) } } else { log.Printf("[INFO] Workflow %s has status %s. Exiting worker.", workflowExecution.ExecutionId, workflowExecution.Status) - shutdown(executionId, workflowExecution.Workflow.ID) + shutdown(workflowExecution, workflowExecution.Workflow.ID, "", true) } time.Sleep(time.Duration(sleepTime) * time.Second) From 14a1d53bbb7a380b144ba468b0a5809596824428 Mon Sep 17 00:00:00 2001 From: amitk Date: Wed, 3 Mar 2021 09:59:06 +0530 Subject: [PATCH 16/57] Added placeholder image --- frontend/src/views/AppCreator.jsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 27d57a24..9d5fb272 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -38,6 +38,7 @@ import AddAPhotoOutlinedIcon from '@material-ui/icons/AddAPhotoOutlined'; import ZoomInOutlinedIcon from '@material-ui/icons/ZoomInOutlined'; import ZoomOutOutlinedIcon from '@material-ui/icons/ZoomOutOutlined'; import LoopIcon from '@material-ui/icons/Loop'; +import AddPhotoAlternateIcon from '@material-ui/icons/AddPhotoAlternate'; const surfaceColor = "#27292D" const inputColor = "#383B40" @@ -2150,7 +2151,7 @@ const AppCreator = (props) => { let imageData = fileBase64; let croppedData = file.length > 0 ? file : fileBase64 - const imageInfo = + const imageInfo = const zoomIn = () => { setScale(scale+0.1); @@ -2272,6 +2273,7 @@ const AppCreator = (props) => {
{setOpenImageModal(true)}}> upload = ref} onChange={editHeaderImage} /> + {imageInfo}
From de92b0f9780622bbedbfd08205e0e8ad67ef1cb0 Mon Sep 17 00:00:00 2001 From: amitk Date: Wed, 3 Mar 2021 15:06:43 +0530 Subject: [PATCH 17/57] Alternate image to be displayed if no image. --- frontend/src/views/AppCreator.jsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 9d5fb272..68898952 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -2152,6 +2152,8 @@ const AppCreator = (props) => { let croppedData = file.length > 0 ? file : fileBase64 const imageInfo = + + const alternateImg = const zoomIn = () => { setScale(scale+0.1); @@ -2273,7 +2275,7 @@ const AppCreator = (props) => {
{setOpenImageModal(true)}}> upload = ref} onChange={editHeaderImage} /> - + {!imageData && (alternateImg)} {imageInfo}
From 516a6df35619412f64e6190a3aecef29b750b0ce Mon Sep 17 00:00:00 2001 From: frikky Date: Wed, 3 Mar 2021 16:16:43 +0100 Subject: [PATCH 18/57] Minor fixes to app creation process --- backend/go-app/main.go | 7 ++++++- backend/go-app/walkoff.go | 13 ++++++++----- frontend/src/views/AppCreator.jsx | 3 ++- frontend/src/views/Apps.jsx | 2 +- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index d1d5c5aa..6e7bd0b4 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -6673,6 +6673,7 @@ func createFs(basepath, pathname string) (billy.Filesystem, error) { // Hotloads new apps from a folder func handleAppHotload(location string, forceUpdate bool) error { + basepath := "base" fs, err := createFs(basepath, location) if err != nil { @@ -6695,7 +6696,11 @@ func handleAppHotload(location string, forceUpdate bool) error { return err } - cacheKey := fmt.Sprintf("workflowapps-sorted") + cacheKey := fmt.Sprintf("workflowapps-sorted-100") + requestCache.Delete(cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") + requestCache.Delete(cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted") requestCache.Delete(cacheKey) return nil diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 0588c6bd..f2882d9a 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -214,7 +214,7 @@ type WorkflowAppActionParameter struct { Description string `json:"description" datastore:"description,noindex" yaml:"description"` ID string `json:"id" datastore:"id" yaml:"id,omitempty"` Name string `json:"name" datastore:"name" yaml:"name"` - Example string `json:"example" datastore:"example" yaml:"example"` + Example string `json:"example" datastore:"example,noindex" yaml:"example"` Value string `json:"value" datastore:"value,noindex" yaml:"value,omitempty"` Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` Options []string `json:"options" datastore:"options" yaml:"options"` @@ -259,12 +259,12 @@ type WorkflowAppAction struct { } `json:"execution_variable" datastore:"execution_variables"` Returns struct { Description string `json:"description" datastore:"returns" yaml:"description,omitempty"` - Example string `json:"example" datastore:"example" yaml:"example"` + Example string `json:"example" datastore:"example,noindex" yaml:"example"` ID string `json:"id" datastore:"id" yaml:"id,omitempty"` Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` } `json:"returns" datastore:"returns"` AuthenticationId string `json:"authentication_id" datastore:"authentication_id"` - Example string `json:"example" datastore:"example" yaml:"example"` + Example string `json:"example,noindex" datastore:"example" yaml:"example"` AuthNotRequired bool `json:"auth_not_required" datastore:"auth_not_required" yaml:"auth_not_required"` } @@ -327,7 +327,7 @@ type Action struct { } `json:"position,omitempty"` Priority int `json:"priority,omitempty" datastore:"priority"` AuthenticationId string `json:"authentication_id" datastore:"authentication_id"` - Example string `json:"example,omitempty" datastore:"example"` + Example string `json:"example,omitempty" datastore:"example,noindex"` AuthNotRequired bool `json:"auth_not_required,omitempty" datastore:"auth_not_required" yaml:"auth_not_required"` Category string `json:"category" datastore:"category"` } @@ -461,7 +461,7 @@ type AuthenticationParams struct { Description string `json:"description" datastore:"description,noindex" yaml:"description"` ID string `json:"id" datastore:"id" yaml:"id"` Name string `json:"name" datastore:"name" yaml:"name"` - Example string `json:"example" datastore:"example" yaml:"example"` + Example string `json:"example" datastore:"example,noindex" yaml:"example"` Value string `json:"value,omitempty" datastore:"value,noindex" yaml:"value"` Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` Required bool `json:"required" datastore:"required" yaml:"required"` @@ -6532,9 +6532,12 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin var err error allapps := []WorkflowApp{} + + // These are slow apps to build with some funky mechanisms reservedNames := []string{ "OWA", "NLP", + "YARA", } buildLaterFirst := []buildLaterStruct{} diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 5ed4bcb0..8252f2e4 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -1238,6 +1238,7 @@ const AppCreator = (props) => { }) setActions(actions) + setActionAmount(actionAmount-1) setUpdate(Math.random()) } @@ -1914,7 +1915,7 @@ const AppCreator = (props) => { {fileUploadEnabled ? { } } - runAppSearch("") + //runAppSearch("") }) .catch(error => { alert.error(error.toString()) From 5a53b34ba5d3d3dbe27abbe2ea1c8cc584496ad2 Mon Sep 17 00:00:00 2001 From: frikky Date: Thu, 4 Mar 2021 04:16:09 +0100 Subject: [PATCH 19/57] #269: Fixed subflow issues --- frontend/src/views/AngularWorkflow.jsx | 87 ++++++++++++++++++++++---- 1 file changed, 75 insertions(+), 12 deletions(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 0fc7178b..76ad8d89 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -50,6 +50,34 @@ function useWindowSize() { return size; } +function removeParam(key, sourceURL) { + if (sourceURL === undefined) { + return + } + + var rtn = sourceURL.split("?")[0], + param, + params_arr = [], + queryString = (sourceURL.indexOf("?") !== -1) ? sourceURL.split("?")[1] : ""; + + if (queryString !== "") { + params_arr = queryString.split("&"); + for (var i = params_arr.length - 1; i >= 0; i -= 1) { + param = params_arr[i].split("=")[0]; + if (param === key) { + params_arr.splice(i, 1); + } + } + rtn = rtn + "?" + params_arr.join("&"); + } + + if (rtn === "?") { + return "" + } + + return rtn; +} + const splitter = "|~|" //const referenceUrl = "https://shuffler.io/functions/webhooks/" //const referenceUrl = window.location.origin+"/api/v1/hooks/" @@ -98,6 +126,7 @@ const AngularWorkflow = (props) => { const [rightSideBarOpen, setRightSideBarOpen] = React.useState(false) const [showSkippedActions, setShowSkippedActions] = React.useState(false) const [lastExecution, setLastExecution] = React.useState("") + const [curpath, setCurpath] = useState(typeof window === 'undefined' || window.location === undefined ? "" : window.location.pathname) // 0 = normal, 1 = just done, 2 = normal const [savingState, setSavingState] = React.useState(0) @@ -321,7 +350,7 @@ const AngularWorkflow = (props) => { }) } - const getWorkflowExecution = (id) => { + const getWorkflowExecution = (id, execution_id) => { fetch(globalUrl+"/api/v1/workflows/"+id+"/executions", { method: 'GET', headers: { @@ -343,13 +372,19 @@ const AngularWorkflow = (props) => { setWorkflowExecutions(responseJson) const cursearch = typeof window === 'undefined' || window.location === undefined ? "" : window.location.search - const tmpView = new URLSearchParams(cursearch).get("execution_id") + var tmpView = new URLSearchParams(cursearch).get("execution_id") + if (execution_id !== undefined && execution_id !== null && execution_id.length > 0 && (tmpView === undefined || tmpView === null || tmpView.length === 0)) { + tmpView = execution_id + } + if (tmpView !== undefined && tmpView !== null && tmpView.length > 0) { - //console.log("SHOW EXECUTION ", tmpView) const execution = responseJson.find(data => data.execution_id === tmpView) if (execution !== null && execution !== undefined) { setExecutionData(execution) setExecutionModalView(1) + + const newitem = removeParam("execution_id", cursearch) + props.history.push(curpath+newitem) } } } @@ -596,12 +631,12 @@ const AngularWorkflow = (props) => { } } - getWorkflowExecution(props.match.params.key) + getWorkflowExecution(props.match.params.key, "") } else if (responseJson.status === "FINISHED") { console.log("STOPPING BECAUSE ITS OVAH!") setExecutionRunning(false) stop() - getWorkflowExecution(props.match.params.key) + getWorkflowExecution(props.match.params.key, "") setUpdate(Math.random()) } } @@ -1528,7 +1563,7 @@ const AngularWorkflow = (props) => { getApps() getAppAuthentication() getEnvironments() - getWorkflowExecution(props.match.params.key) + getWorkflowExecution(props.match.params.key, "") getAvailableWorkflows(-1) getSettings() @@ -1536,6 +1571,9 @@ const AngularWorkflow = (props) => { const tmpView = new URLSearchParams(cursearch).get("view") if (tmpView !== undefined && tmpView !== null && tmpView === "executions") { setExecutionModalOpen(true) + + const newitem = removeParam("view", cursearch) + props.history.push(curpath+newitem) } return } @@ -5135,7 +5173,17 @@ const AngularWorkflow = (props) => { if (e.target.value.id !== workflow.id) { const startnode = e.target.value.actions.find(action => action.id === e.target.value.start) if (startnode !== undefined && startnode !== null) { + //ddsetSubworkflowStartnode(innernode) setSubworkflowStartnode(startnode) + + try { + workflow.triggers[selectedTriggerIndex].parameters[3].value = e.target.value.id + } catch { + workflow.triggers[selectedTriggerIndex].parameters[3] = { + "name": "startnode", + "value": e.target.value.id, + } + } } console.log("STARTNODE: ", startnode) } @@ -6236,7 +6284,7 @@ const AngularWorkflow = (props) => { @@ -6482,7 +6530,7 @@ const AngularWorkflow = (props) => { style={{borderRadius: "0px"}} variant="outlined" onClick={() => { - getWorkflowExecution(props.match.params.key) + getWorkflowExecution(props.match.params.key, "") }} color="primary"> Refresh executions @@ -6562,7 +6610,7 @@ const AngularWorkflow = (props) => { { setExecutionRunning(false) stop() - getWorkflowExecution(props.match.params.key) + getWorkflowExecution(props.match.params.key, "") setExecutionModalView(0) setLastExecution(executionData.execution_id) }}> @@ -6600,7 +6648,14 @@ const AngularWorkflow = (props) => { {executionData.execution_source !== undefined && executionData.execution_source !== null && executionData.execution_source.length > 0 && executionData.execution_source !== "default" ?
Source:   {executionData.execution_parent !== null && executionData.execution_parent !== undefined && executionData.execution_parent.length > 0 ? - Parent Workflow + executionData.execution_source === props.match.params.key ? + { + getWorkflowExecution(props.match.params.key, executionData.execution_parent) + }}> + Parent Execution + + : + Parent Workflow : executionData.execution_source } @@ -6659,7 +6714,7 @@ const AngularWorkflow = (props) => { const statusColor = data.status === "FINISHED" || data.status === "SUCCESS" ? "green" : data.status === "ABORTED" || data.status === "FAILURE" ? "red" : "orange" var imgSrc = curapp === undefined ? "" : curapp.large_image - if (imgSrc.length === 0) { + if (imgSrc.length === 0 && workflow.actions !== undefined && workflow.actions !== null) { // Look for the node in the workflow const action = workflow.actions.find(action => action.id === data.action.id) if (action !== undefined && action !== null) { @@ -6723,7 +6778,15 @@ const AngularWorkflow = (props) => { {data.action.app_name === "shuffle-subflow" && validate.result.success !== undefined && validate.result.success === true ? {validate.valid && data.action.parameters !== undefined && data.action.parameters !== null ? - See subflow execution + data.action.parameters[0].value === props.match.params.key ? + { + getWorkflowExecution(props.match.params.key, validate.result.execution_id) + }}> + See sub-execution + + : + { + }}>See subflow execution : "TBD: Load subexecution result for" } From 62e7fa09d84ca73b1b22751ef903ec0b70f4141c Mon Sep 17 00:00:00 2001 From: frikky Date: Thu, 4 Mar 2021 08:17:43 +0100 Subject: [PATCH 20/57] BUG: Fixed bug in app sdk for OpenAPI lists --- backend/app_sdk/app_base.py | 194 ++++++++++++++++++++----- backend/go-app/codegen.go | 147 ++++++++++--------- backend/go-app/docker.go | 1 - backend/go-app/main.go | 12 +- backend/go-app/walkoff.go | 4 - frontend/src/views/AngularWorkflow.jsx | 29 ++-- 6 files changed, 256 insertions(+), 131 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index e428da9f..748ba455 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -147,6 +147,7 @@ class AppBase: #self.action = action loopnames = [] + print(f"Baseparams to check!!: {baseparams}") for key, value in baseparams.items(): check_value = "" for param in self.action["parameters"]: @@ -160,6 +161,7 @@ class AppBase: self.result_wrapper_count = octothorpe_count print("[INFO] NEW OCTOTHORPE WRAPPER: %d" % octothorpe_count) + # This whole thing is hard. # item = [{"data": "1.2.3.4", "dataType": "ip"}] # $item = DONT loop items. @@ -178,12 +180,40 @@ class AppBase: # FIXME: Check the above, and fix so that nested looped items can be # Skipped if wanted - print("\nCHECK: %s" % check_value) + #print("\nCHECK: %s" % check_value) + #try: + # values = parameter["value_replace"] + # if values != None: + # print(values) + # for val in values: + # print(val) + #except: + # pass + should_merge = False if "#" in check_value: should_merge = True + # Specific for OpenAPI body replacement + print("\n\n\nDOING STUFF BELOW HERE") + if not should_merge: + for parameter in self.action["parameters"]: + if parameter["name"] == key: + print("CHECKING BODY FOR VALUE REPLACE DATA!") + try: + values = parameter["value_replace"] + if values != None: + print(values) + for val in values: + if "#" in val["value"]: + should_merge = True + break + except: + pass + + print(f"MERGE: {should_merge}") if isinstance(value, list): + print("Item {value} is a list.") if len(value) <= 1: if len(value) == 1: baseparams[key] = value[0] @@ -206,7 +236,7 @@ class AppBase: all_list_keys.append(key) all_lists.append(baseparams[key]) else: - print("%s is not a list: " % value) + print(f"{value} is not a list") print("Listlengths: %s" % listlengths) if len(listlengths) == 0: @@ -271,20 +301,25 @@ class AppBase: # Runs recursed versions with inner loops and such async def run_recursed_items(self, func, baseparams, loop_wrapper): + print(f"RECURSED ITEMS: {baseparams}") has_loop = False newparams = {} for key, value in baseparams.items(): if isinstance(value, list) and len(value) > 0: - print("In list check") + print(f"In list check for {key}") + try: - value[0] = json.loads(value[0]) + # Added skip for body (OpenAPI) which uses data= in requests + # Can be screwed up if they name theirs body too + if key != "body": + value[0] = json.loads(value[0]) except json.decoder.JSONDecodeError as e: print("JSON casting error: %s" % e) except TypeError as e: print("TypeError: %s" % e) - print("POST list check") + print("POST initial list check") if isinstance(value, list) and len(value) == 1 and isinstance(value[0], list): try: @@ -294,12 +329,11 @@ class AppBase: except KeyError: loop_wrapper[key] = 1 - print("Key %s is a list: %s" % (key, value)) + print(f"Key {key} is a list: {value}") newparams[key] = value[0] has_loop = True else: - print("Key %s is NOT a list within a list" % (key)) - + print(f"Key {key} is NOT a list within a list. Value: {value}") newparams[key] = value results = [] @@ -315,6 +349,7 @@ class AppBase: print("[INFO] Multiplier length: %d" % len(param_multiplier)) for subparams in param_multiplier: + print(f"SUBPARAMS IN MULTI: {subparams}") try: tmp = await func(**subparams) except: @@ -1361,6 +1396,7 @@ class AppBase: return True, "" + # THE START IS ACTUALLY RIGHT HERE :O # Checks whether conditions are met, otherwise set branchcheck, tmpresult = check_branch_conditions(action, fullexecution) if not branchcheck: @@ -1437,6 +1473,10 @@ class AppBase: except (IndexError, KeyError, TypeError) as e: print("Options err: {e}") + # This part is purely for OpenAPI accessibility. + # It replaces the data back into the main item + # Earlier, we handled each of the items and did later string replacement, + # but this has changed to do lists within items and such if parameter["name"] == "body": bodyindex = counter #print("PARAM: %s" % parameter) @@ -1445,16 +1485,27 @@ class AppBase: if values != None: added = 0 for val in values: - newparams.append({ - "name": val["key"], - "value": val["value"], - "variant": "STATIC_VALUE", - "id": "body_replacement", - }) + print(f"VAL: {val}") + #parameter["value"].replace(val["key"], val["value"], -1) + print(f'PARAM1: {action["parameters"][counter]["value"]}') + action["parameters"][counter]["value"] = action["parameters"][counter]["value"].replace(val["key"], val["value"], 1) + #action["parameters"][counter]["value"].replace(r"${url}", r"$Find_URLs.valid.#.data", 1) + print(f'PARAM2: {action["parameters"][counter]["value"]}') + #newparams.append({ + # "name": val["key"], + # "value": val["value"], + # "variant": "STATIC_VALUE", + # "id": "body_replacement", + # "schema": { + # "type": "string", + # }, + #}) - print("Added param %s for body" % val["key"]) + print(f'[INFO] Added param {val["key"]} for body with value {val["value"]} (using OpenAPI)') added += 1 + #action["parameters"]["body"] + print("ADDED %d parameters for body" % added) except KeyError as e: print("KeyError body OpenAPI: %s" % e) @@ -1462,6 +1513,7 @@ class AppBase: break + print(action["parameters"]) for parameter in newparams: action["parameters"].append(parameter) @@ -1478,6 +1530,7 @@ class AppBase: multi_parameters = json.loads(json.dumps(params)) multiexecution = False multi_execution_lists = [] + remove_params = [] for parameter in action["parameters"]: check, value, is_loop = parse_params(action, fullexecution, parameter) if check: @@ -1506,6 +1559,7 @@ class AppBase: print("Before first part in multiexec!") handled = False if len(actualitem[0]) > 2 and actualitem[0][1] == "SHUFFLE_NO_SPLITTER": + print("(1) Pre replacement: %s" % actualitem[0][2]) tmpitem = value @@ -1555,10 +1609,10 @@ class AppBase: tmpitem = tmpitem.replace(actualitem[0][0], replacement, 1) # This code handles files. - print("(1) ------------ PARAM: %s" % parameter["schema"]["type"]) resultarray = [] isfile = False try: + print("(1) ------------ PARAM: %s" % parameter["schema"]["type"]) if parameter["schema"]["type"] == "file" and len(value) > 0: print("(1) SHOULD HANDLE FILE IN MULTI. Get based on value %s" % tmpitem) # This is silly :) @@ -1572,8 +1626,10 @@ class AppBase: print("(1) FILE VALUE FOR VAL %s: %s" % (tmp_file_split, file_value)) isfile = True + except NameError as e: + print("(1) SCHEMA NAMEERROR IN FILE HANDLING: %s" % e) except KeyError as e: - print("(1) SCHEMA ERROR IN FILE HANDLING: %s" % e) + print("(1) SCHEMA KEYERROR IN FILE HANDLING: %s" % e) except json.decoder.JSONDecodeError as e: print("(1) JSON ERROR IN FILE HANDLING: %s" % e) @@ -1589,7 +1645,7 @@ class AppBase: multi_execution_lists.append(new_replacement) #print("MULTI finished: %s" % json_replacement) else: - print("(2) Pre replacement. ") #% actualitem) + print("(2) Pre replacement (loop with variables). ") #% actualitem) # This is here to handle for loops within variables.. kindof # 1. Find the length of the longest array # 2. Build an array with the base values based on parameter["value"] @@ -1664,22 +1720,56 @@ class AppBase: multi_execution_lists.append(resultarray) multi_parameters[parameter["name"]] = resultarray + + #if parameter["id"] == "body_replacement": + # print("Should run body MULTI replacement in index %d with %s" % (bodyindex, parameter)) + # try: + # print("PREBODY: %s" % params["body"]) + + # parsedarray = str(resultarray) + # try: + # parsedarray = json.dumps(resultarray) + # except: + # pass + + # if f'\"{parameter["name"]}\"' in params["body"]: + # params["body"] = params["body"].replace(f'\"{parameter["name"]}\"' , parsedarray, -1) + # multi_parameters["body"] = multi_parameters["body"].replace(f'\"{parameter["name"]}\"' , parsedarray, -1) + # else: + # params["body"] = params["body"].replace(parameter["name"], parsedarray, -1) + # multi_parameters["body"] = multi_parameters["body"].replace(parameter["name"], parsedarray, -1) + + # #print("POSTBODY: %s" % params["body"]) + # #if isinstance(multi_parameters, list): + # # print("MULTIPARAM AS LIST (NOT REPLACING!!)!") + # # for multiparam in multi_parameters: + # # print(f"MULTIPARAM: {multiparam}") + # # #multi_parameters["body"] = multi_parameters["body"].replace(parameter["name"], str(parameter["value"]), -1) + # #else: + + # except KeyError as e: + # print("KEYERROR: %s" % e) + + # remove_params.append(parameter["name"]) + # #bodyindex = counter + # continue + else: # Parses things like int(value) print("Normal parsing (not looping)")#with data %s" % value) value = parse_wrapper_start(value) - if parameter["id"] == "body_replacement": - print("Should run body replacement in index %d with %s" % (bodyindex, parameter)) - try: - print("PREBODY: %s" % params["body"]) - params["body"] = params["body"].replace(parameter["name"], parameter["value"], -1) - print("POSTBODY: %s" % params["body"]) - except KeyError as e: - print("KEYERROR: %s" % e) + #if parameter["id"] == "body_replacement": + # print("Should run body replacement in index %d with %s" % (bodyindex, parameter)) + # try: + # print("PREBODY: %s" % params["body"]) + # params["body"] = params["body"].replace(parameter["name"], parameter["value"], -1) + # print("POSTBODY: %s" % params["body"]) + # except KeyError as e: + # print("KEYERROR: %s" % e) - #bodyindex = counter - continue + # #bodyindex = counter + # continue #for parameter in action["parameters"]: #if parameter["name"] == "body": @@ -1702,6 +1792,8 @@ class AppBase: except KeyError as e: print("SCHEMA ERROR IN FILE HANDLING: %s" % e) + + #remove_params.append(parameter["name"]) # Fix lists here # FIXME: This doesn't really do anything anymore print("CHECKING multi execution list!") @@ -1723,7 +1815,7 @@ class AppBase: #print("New list length: %d" % len(filteredlist)) if len(filteredlist) > 1: - print("Calculating new multi-loop length with %d lists" % len(filteredlist)) + print(f"Calculating new multi-loop length with {len(filteredlist)} lists") tmplength = 1 for innerlist in filteredlist: tmplength = len(innerlist)*tmplength @@ -1732,6 +1824,25 @@ class AppBase: minlength = tmplength print("New multi execution length: %d\n" % tmplength) + + # Cleaning up extra list params + for subparam in remove_params: + #print(f"DELETING {subparam}") + try: + del params[subparam] + except: + pass + #print(f"Error with subparam deletion of {subparam} in {params}") + try: + del multi_parameters[subparam] + except: + #print(f"Error with subparam deletion of {subparam} in {multi_parameters} (2)") + pass + + print() + print(f"Param: {params}") + print(f"Multiparams: {multi_parameters}") + print() if not multiexecution: #newparams.append({ @@ -1744,7 +1855,7 @@ class AppBase: #print("[INFO] APP_SDK DONE: Starting NORMAL execution of function") print("[INFO] Running normal execution\n") newres = await func(**params) - print("\n[INFO] Returned from execution with datalength!")#, newres) + print("\n[INFO] Returned from execution!")#, newres) if isinstance(newres, tuple): print("[INFO] Handling return as tuple") # Handles files. @@ -1772,6 +1883,17 @@ class AppBase: elif isinstance(newres, str): print("[INFO] Handling return as string of length %d" % len(newres)) result += newres + elif isinstance(newres, dict) or isinstance(newres, list): + try: + result += json.dumps(newres, indent=4) + except json.JSONDecodeError as e: + print("Failed decoding result: %s" % e) + + try: + result += str(newres) + except ValueError: + result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres) + print("Can't handle type %s value from function" % (type(newres))) else: try: result += str(newres) @@ -1898,10 +2020,16 @@ class AppBase: # Dump the result as a string of a list #print("RESULTS: %s" % results) - if isinstance(results, list): + if isinstance(results, list) or isinstance(results, dict): print("JSON OBJECT? ", json_object) + + # This part is weird lol if json_object: - result = json.dumps(results) + try: + result = json.dumps(results) + except json.JSONDecodeError as e: + print(f"Failed to decode: {e}") + result = results else: result = "[" for item in results: @@ -1926,7 +2054,7 @@ class AppBase: else: print("Normal result - no list?") result = results - + print("RESULT: %s" % result) action_result["status"] = "SUCCESS" action_result["result"] = str(result) diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index 74776e88..eecc0ee7 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -1010,16 +1010,6 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [ optionalQueries := []string{} parameters := []string{} optionalParameters := []WorkflowAppActionParameter{} - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Schema: SchemaDefinition{ - Type: "string", - }, - }) headersFound := []string{} if len(path.Connect.Parameters) > 0 { @@ -1106,6 +1096,17 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [ } } + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + // ensuring that they end up last in the specification // (order is ish important for optional params) - they need to be last. for _, optionalParam := range optionalParameters { @@ -1145,16 +1146,6 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor // FIXME - remove this when authentication is properly introduced parameters := []string{} optionalParameters := []WorkflowAppActionParameter{} - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify the SSL certificate request", - Multiline: false, - Required: false, - Example: "False - default=True", - Schema: SchemaDefinition{ - Type: "string", - }, - }) headersFound := []string{} if len(path.Get.Parameters) > 0 { @@ -1241,6 +1232,17 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor } } + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify the SSL certificate request", + Multiline: false, + Required: false, + Example: "False - default=True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + // ensuring that they end up last in the specification // (order is ish important for optional params) - they need to be last. for _, optionalParam := range optionalParameters { @@ -1280,16 +1282,6 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo optionalQueries := []string{} parameters := []string{} optionalParameters := []WorkflowAppActionParameter{} - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Schema: SchemaDefinition{ - Type: "string", - }, - }) headersFound := []string{} if len(path.Head.Parameters) > 0 { @@ -1374,6 +1366,17 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo } } + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + // ensuring that they end up last in the specification // (order is ish important for optional params) - they need to be last. for _, optionalParam := range optionalParameters { @@ -1413,16 +1416,6 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [] optionalQueries := []string{} parameters := []string{} optionalParameters := []WorkflowAppActionParameter{} - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Schema: SchemaDefinition{ - Type: "string", - }, - }) headersFound := []string{} if len(path.Delete.Parameters) > 0 { @@ -1508,6 +1501,17 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [] } } + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + // ensuring that they end up last in the specification // (order is ish important for optional params) - they need to be last. for _, optionalParam := range optionalParameters { @@ -1546,16 +1550,6 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo optionalQueries := []string{} parameters := []string{} optionalParameters := []WorkflowAppActionParameter{} - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Schema: SchemaDefinition{ - Type: "string", - }, - }) fileField := "" if path.Post.RequestBody != nil { @@ -1674,6 +1668,17 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo } } + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + // ensuring that they end up last in the specification // (order is ish important for optional params) - they need to be last. for _, optionalParam := range optionalParameters { @@ -1718,16 +1723,6 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W optionalQueries := []string{} parameters := []string{} optionalParameters := []WorkflowAppActionParameter{} - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Schema: SchemaDefinition{ - Type: "string", - }, - }) headersFound := []string{} if len(path.Patch.Parameters) > 0 { @@ -1812,6 +1807,17 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W } } + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + // ensuring that they end up last in the specification // (order is ish important for optional params) - they need to be last. for _, optionalParam := range optionalParameters { @@ -1851,16 +1857,6 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor optionalQueries := []string{} parameters := []string{} optionalParameters := []WorkflowAppActionParameter{} - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Schema: SchemaDefinition{ - Type: "string", - }, - }) headersFound := []string{} if len(path.Put.Parameters) > 0 { @@ -1946,6 +1942,17 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor } } + optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ + Name: "ssl_verify", + Description: "Check if you want to verify request", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + // ensuring that they end up last in the specification // (order is ish important for optional params) - they need to be last. for _, optionalParam := range optionalParameters { diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index 776dbea0..27c8e436 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -24,7 +24,6 @@ import ( "net/http" "os" "strings" - //"google.golang.org/appengine" ) // Parses a directory with a Dockerfile into a tar for Docker images.. diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 6e7bd0b4..1595ba82 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -62,16 +62,10 @@ import ( // githttp "gopkg.in/src-d/go-git.v4/plumbing/transport/http" // Web - // "github.com/gorilla/handlers" "github.com/gorilla/mux" + "github.com/patrickmn/go-cache" "google.golang.org/grpc" http2 "gopkg.in/src-d/go-git.v4/plumbing/transport/http" - // Old items (cloud) - // "google.golang.org/appengine" - // "google.golang.org/appengine/memcache" - // applog "google.golang.org/appengine/log" - //cloudrun "google.golang.org/api/run/v1" - "github.com/patrickmn/go-cache" ) // This is used to handle onprem vs offprem databases etc @@ -5915,7 +5909,7 @@ func getOpenapi(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("[INFO] API LENGTH GET: %d, ID: %s", len(parsedApi.Body), id) + log.Printf("[INFO] API LENGTH GET FOR OPENAPI %s: %d, ID: %s", id, len(parsedApi.Body), id) parsedApi.Success = true data, err := json.Marshal(parsedApi) @@ -6579,7 +6573,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { Body: string(body), } - log.Printf("[INFO] API LENGTH: %d, ID: %s", len(parsed.Body), newmd5) + log.Printf("[INFO] API LENGTH FOR %s: %d, ID: %s", api.Name, len(parsed.Body), newmd5) // FIXME: Might cause versioning issues if we re-use the same!! // FIXME: Need a way to track different versions of the same app properly. // Hint: Save API.id somewhere, and use newmd5 to save latest version diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index f2882d9a..78ef796e 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -2188,8 +2188,6 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // Here to check access rights ctx := context.Background() - log.Println("GetWorkflow start") - tmpworkflow, err := getWorkflow(ctx, fileId) if err != nil { log.Printf("Failed getting the workflow locally (save workflow): %s", err) @@ -2198,8 +2196,6 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { return } - log.Println("GetWorkflow end") - // FIXME - have a check for org etc too.. if user.Id != tmpworkflow.Owner && user.Role != "admin" { log.Printf("Wrong user (%s) for workflow %s (save)", user.Username, tmpworkflow.ID) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 76ad8d89..f606150e 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -2832,7 +2832,6 @@ const AngularWorkflow = (props) => { } var exampledata = item.example === undefined ? "" : item.example - console.log("EXAMPLE: ", exampledata) // Find previous execution and their variables //exampledata === "" && if (workflowExecutions.length > 0) { @@ -3485,10 +3484,11 @@ const AngularWorkflow = (props) => { // Handles the fields under OpenAPI body to be parsed. if (data.name.startsWith("${") && data.name.endsWith("}")) { + console.log("INSIDE VALUE REPLACE: ", data.name, toComplete) // PARAM FIX - Gonna use the ID field, even though it's a hack const paramcheck = selectedAction.parameters.find(param => param.name === "body") if (paramcheck !== undefined) { - if (paramcheck["value_replace"] === undefined) { + if (paramcheck["value_replace"] === undefined || paramcheck["value_replace"] === null) { paramcheck["value_replace"] = [{ "key": data.name, "value": toComplete, @@ -4559,14 +4559,14 @@ const AngularWorkflow = (props) => {
- - + +
@@ -5205,7 +5205,10 @@ const AngularWorkflow = (props) => { })} } - {workflow.triggers[selectedTriggerIndex].parameters[0].value.length === 0 ? null : Explore selected workflow} + {workflow.triggers[selectedTriggerIndex].parameters[0].value.length === 0 ? null : + workflow.triggers[selectedTriggerIndex].parameters[0].value === props.match.params.key ? null : + Explore selected workflow + }
@@ -6935,9 +6938,7 @@ const AngularWorkflow = (props) => { -
{ - //event.preventDefault() - }}> +
{curapp === null ? null : {selectedResult.app_name}} From f88d88ba9e990f45251a53799688bb175a19d23d Mon Sep 17 00:00:00 2001 From: frikky Date: Thu, 4 Mar 2021 17:06:25 +0100 Subject: [PATCH 21/57] BUG: Minor app sdk and build fixes --- backend/app_sdk/app_base.py | 20 +++++++++++--------- backend/go-app/codegen.go | 9 +++++++-- frontend/src/views/AngularWorkflow.jsx | 4 +++- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 748ba455..a4d95d19 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -391,7 +391,8 @@ class AppBase: print("Ret length: %d" % len(ret)) if len(ret) == 1: - ret = ret[0] + #ret = ret[0] + print("DONT make list of 1 into 0!!") print("Return from execution: %s" % ret) if ret == None: @@ -418,7 +419,8 @@ class AppBase: results.append(ret) if len(results) == 1: - results = results[0] + #results = results[0] + print("DONT MAKE LIST FROM 1 TO 0!!") print("\nLOOP: %s\nRESULTS: %s" % (loop_wrapper, results)) return results @@ -521,11 +523,11 @@ class AppBase: data["filename"] = curfile["filename"] filename = curfile["filename"] except KeyError as e: - print("KeyError in file setup: %s" % e) + print(f"KeyError in file setup: {e}") pass ret = requests.post("%s%s" % (self.url, create_path), headers=headers, json=data) - print("Ret CREATE: %s" % ret.text) + print(f"Ret CREATE: {ret.text}") cur_id = "" if ret.status_code == 200: print("RET: %s" % ret.text) @@ -546,7 +548,7 @@ class AppBase: continue new_headers = { - "Authorization": "Bearer %s" % self.authorization, + "Authorization": f"Bearer {self.authorization}", } upload_path = "/api/v1/files/%s/upload?execution_id=%s" % (cur_id, full_execution["execution_id"]) @@ -1255,16 +1257,16 @@ class AppBase: self.logger.info("Checking %s %s %s" % (sourcevalue, check, destinationvalue)) if check == "=" or check.lower() == "equals": - if sourcevalue.lower() == destinationvalue.lower(): + if str(sourcevalue).lower() == str(destinationvalue).lower(): return True elif check == "!=" or check.lower() == "does not equal": - if sourcevalue.lower() != destinationvalue.lower(): + if str(sourcevalue).lower() != str(destinationvalue).lower(): return True elif check.lower() == "startswith": - if sourcevalue.lower().startswith(destinationvalue.lower()): + if str(sourcevalue).lower().startswith(str(destinationvalue).lower()): return True elif check.lower() == "endswith": - if sourcevalue.lower().endswith(destinationvalue.lower()): + if str(sourcevalue).lower().endswith(str(destinationvalue).lower()): return True elif check.lower() == "contains": if destinationvalue.lower() in sourcevalue.lower(): diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index eecc0ee7..9644ccd3 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -392,7 +392,11 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet %s %s %s - return requests.%s(url, headers=headers%s%s%s%s).text + ret = requests.%s(url, headers=headers%s%s%s%s) + try: + return ret.json() + except json.decoder.JSONDecodeError: + return ret.text `, functionname, authenticationParameter, @@ -419,6 +423,7 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet ) // Use lowercase when checking + log.Println(data) /* if strings.Contains(functionname, "filter") { //log.Printf("FUNCTION: %s", data) @@ -1564,7 +1569,7 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo newName := string(fmt.Sprintf("%s", string(fieldname))) if newName[0] == 0x22 && newName[len(newName)-1] == 0x22 { parsedName := newName[1 : len(newName)-1] - log.Printf("Parse name: %s", parsedName) + //log.Printf("[INFO] Parse name: %s", parsedName) fileField = parsedName curParam := WorkflowAppActionParameter{ diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index f606150e..6dd31e3c 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -2700,6 +2700,8 @@ const AngularWorkflow = (props) => { selectedAction.label = event.target.value setSelectedAction(selectedAction) + console.log("SHOULD CHANGE NAME EVERYWHERE ITS USED TOO BASED ON OLD NAME!") + /* if (nodeaction.label !== curaction.label) { console.log("BEACH!") @@ -7032,7 +7034,7 @@ const AngularWorkflow = (props) => { Execution Variable - Execution Variables are TEMPORARY variables that you can ony be set and used during execution. Learn more here + Execution Variables are TEMPORARY variables that you can ony be set and used during execution. Learn more here setNewVariableName(event.target.value)} color="primary" From 7a4c921695a2676db69aa640024f5ada4554bfe6 Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 6 Mar 2021 13:38:36 +0100 Subject: [PATCH 22/57] Fixed admin issues with cloud --- backend/go-app/codegen.go | 1 - frontend/src/components/Header.js | 3 +- frontend/src/views/Admin.jsx | 60 ++++++++++++++++++------------- 3 files changed, 37 insertions(+), 27 deletions(-) diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index 9644ccd3..a9b4c1b6 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -423,7 +423,6 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet ) // Use lowercase when checking - log.Println(data) /* if strings.Contains(functionname, "filter") { //log.Printf("FUNCTION: %s", data) diff --git a/frontend/src/components/Header.js b/frontend/src/components/Header.js index c7cb5e6c..eb6d6e0c 100644 --- a/frontend/src/components/Header.js +++ b/frontend/src/components/Header.js @@ -151,9 +151,10 @@ const Header = props => { - const logoCheck = !homePage ? null : null + // Handle top bar or something + const logoCheck = !homePage ? null : null const loginTextBrowser = !isLoggedIn ?
diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index 9df961ac..8b1d3ec5 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -430,11 +430,13 @@ const Admin = (props) => { const submitUser = (data) => { console.log("INPUT: ", data) + setLoginInfo("") // Just use this one? var data = { "username": data.Username, "password": data.Password } var baseurl = globalUrl const url = baseurl + '/api/v1/users/register'; + fetch(url, { method: 'POST', credentials: "include", @@ -446,7 +448,7 @@ const Admin = (props) => { .then(response => response.json().then(responseJson => { if (responseJson["success"] === false) { - setLoginInfo("Error in input: " + responseJson.reason) + setLoginInfo("Error: " + responseJson.reason) } else { setLoginInfo("") setModalOpen(false) @@ -912,11 +914,11 @@ const Admin = (props) => { } else if (newValue === 2) { getAppAuthentication() } else if (newValue === 3) { - getEnvironments() + getFiles() } else if (newValue === 4) { getSchedules() } else if (newValue === 5) { - getFiles() + getEnvironments() } else if (newValue === 6) { getOrgs() } @@ -1157,11 +1159,11 @@ const Admin = (props) => { }, }} > - + Edit user
{
+ @@ -1738,7 +1747,7 @@ const Admin = (props) => { { { fullWidth onChange={(e) => { console.log("VALUE: ", e.target.value) - setUser(data.id, "role", e.target.value) + + if (isCloud) { + setUser(data.username, "role", e.target.value) + } else { + setUser(data.id, "role", e.target.value) + } }} style={{ backgroundColor: theme.palette.surfaceColor, color: "white", height: "50px" }} > @@ -1795,33 +1809,29 @@ const Admin = (props) => { User } - style = {{ minWidth: 150, maxWidth: 150}} + style ={{ minWidth: 135, maxWidth: 135, marginRight: 15,}} /> - + + - + ) })} @@ -1867,7 +1877,7 @@ const Admin = (props) => { uploadFiles(files) } - const filesView = curTab === 5 ? + const filesView = curTab === 3 ? 1366 ? 1366 : 1200, margin: "auto", padding: 20 }} onDrop={uploadFile}>
@@ -2311,7 +2321,7 @@ const Admin = (props) => {
: null - const environmentView = curTab === 3 ? + const environmentView = curTab === 5 ?

Environments

@@ -2555,10 +2565,10 @@ const Admin = (props) => { > Organization/> Users /> - {isCloud ? null : App Authentication/>} + App Authentication/> + Files /> + Schedules /> {isCloud ? null : Environments/>} - {isCloud ? null : Schedules />} - {isCloud ? null : Files />} {window.location.protocol == "http:" && window.location.port === "3000" ? Hybrid/> : null} {window.location.protocol == "http:" && window.location.port === "3000" ? Organizations/> : null} {window.location.protocol === "http:" && window.location.port === "3000" ? Categories/> : null} From 32718fbd3bca1822bc9b238f874b1731fc4dce9f Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 6 Mar 2021 15:12:58 +0100 Subject: [PATCH 23/57] Minor fixes to main.go --- backend/go-app/main.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 1595ba82..a2a5784c 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -663,7 +663,7 @@ func handleApiAuthentication(resp http.ResponseWriter, request *http.Request) (U sessionToken := c.Value session, err := getSession(ctx, sessionToken) if err != nil { - log.Printf("Session %s doesn't exist (session auth): %s", sessionToken, err) + log.Printf("[WARNING] Session %s doesn't exist (session auth): %s", sessionToken, err) return User{}, err } @@ -1286,7 +1286,7 @@ func handleLogout(resp http.ResponseWriter, request *http.Request) { //sessionToken = c.Value //session, err := getSession(ctx, sessionToken) //if err != nil { - // log.Printf("Session %s doesn't exist (logout): %s", sessionToken, err) + // log.Printf("[WARNING] Session %s doesn't exist (logout): %s", sessionToken, err) // resp.WriteHeader(401) // resp.Write([]byte(`{"success": false, "reason": "Couldn't find your session"}`)) // return @@ -2471,7 +2471,7 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - log.Printf("Username: %s", data.Username) + log.Printf("[INFO] Login Username: %s", data.Username) q := datastore.NewQuery("Users").Filter("Username =", data.Username) var users []User _, err = dbclient.GetAll(ctx, q, &users) @@ -2509,7 +2509,7 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) { // FIXME - have timeout here loginData := `{"success": true}` if len(Userdata.Session) != 0 { - log.Println("User session exists - resetting") + log.Println("[INFO] User session exists - resetting") expiration := time.Now().Add(3600 * time.Second) http.SetCookie(resp, &http.Cookie{ @@ -7204,7 +7204,7 @@ func runInit(ctx context.Context) { } else { if len(users) < 5 && len(users) > 0 { for _, user := range users { - log.Printf("Username: %s, role: %s", user.Username, user.Role) + log.Printf("[INFO] Username: %s, role: %s", user.Username, user.Role) } } else { log.Printf("Found %d users.", len(users)) From 00d0f444e559ca1c0fbbeb3892b86c47c8babcec Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 6 Mar 2021 15:27:44 +0100 Subject: [PATCH 24/57] Minor fixes to the upload view --- frontend/src/views/AppCreator.jsx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index afb22314..b7d4ebdb 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -319,6 +319,8 @@ const AppCreator = (props) => { const checkQuery = () => { var urlParams = new URLSearchParams(window.location.search) if (!urlParams.has("id")) { + setActionAmount(0) + setIsAppLoaded(true) return } @@ -2105,7 +2107,7 @@ const AppCreator = (props) => { : null}
-

Actions ({actionAmount} / {actions.length})

+

Actions {actionAmount > 0 ? ({actionAmount} / {actions.length}) : null}

Actions are the tasks performed by an app. Read more about actions and apps here.
@@ -2199,7 +2201,9 @@ const AppCreator = (props) => { const imageInfo = - const alternateImg = + const alternateImg = { + upload.click() + }}/> const zoomIn = () => { setScale(scale+0.1); From 67373bfe3e6ffe50bf9aedaf7d876d33097840ef Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 6 Mar 2021 18:23:48 +0100 Subject: [PATCH 25/57] Added eradication to AppCreator --- frontend/src/views/AppCreator.jsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 5432a748..036cc4be 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -2015,8 +2015,10 @@ const AppCreator = (props) => { "SIEM", "Network", "Assets", + "Eradication", "Other", ] + const tagView =
{/* From beeba414bc2606c68486fe67ca3edd15c85a4590 Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 7 Mar 2021 11:56:33 +0100 Subject: [PATCH 26/57] Added automated authentication selection if it exists --- backend/app_sdk/app_base.py | 25 ++++++------ backend/go-app/walkoff.go | 37 +++++++++++++----- frontend/src/views/AngularWorkflow.jsx | 53 +++++++++++++++++++++++++- frontend/src/views/AppCreator.jsx | 6 +-- 4 files changed, 96 insertions(+), 25 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index a4d95d19..219506b7 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -1447,7 +1447,7 @@ class AppBase: params = {} try: for item in action["authentication"]: - print("AUTH: ", key, value) + #print("AUTH: ", key, value) params[item["key"]] = item["value"] except KeyError: print("No authentication specified!") @@ -1469,11 +1469,11 @@ class AppBase: if "||" in parameter["value"]: splitvalue = parameter["value"].split("||") if len(splitvalue) > 1: - print(f'[INFO] Parsed split || options of actions["parameters"]["name"]') + #print(f'[INFO] Parsed split || options of actions["parameters"]["name"]') action["parameters"][counter]["value"] = splitvalue[1] except (IndexError, KeyError, TypeError) as e: - print("Options err: {e}") + print("[WARNING] Options err: {e}") # This part is purely for OpenAPI accessibility. # It replaces the data back into the main item @@ -1487,12 +1487,12 @@ class AppBase: if values != None: added = 0 for val in values: - print(f"VAL: {val}") + #print(f"VAL: {val}") #parameter["value"].replace(val["key"], val["value"], -1) - print(f'PARAM1: {action["parameters"][counter]["value"]}') + #print(f'PARAM1: {action["parameters"][counter]["value"]}') action["parameters"][counter]["value"] = action["parameters"][counter]["value"].replace(val["key"], val["value"], 1) #action["parameters"][counter]["value"].replace(r"${url}", r"$Find_URLs.valid.#.data", 1) - print(f'PARAM2: {action["parameters"][counter]["value"]}') + #print(f'PARAM2: {action["parameters"][counter]["value"]}') #newparams.append({ # "name": val["key"], # "value": val["value"], @@ -1503,7 +1503,8 @@ class AppBase: # }, #}) - print(f'[INFO] Added param {val["key"]} for body with value {val["value"]} (using OpenAPI)') + #print(f'[INFO] Added param {val["key"]} for body with value {val["value"]} (using OpenAPI)') + print(f'[INFO] Added param {val["key"]} for body (using OpenAPI)') added += 1 #action["parameters"]["body"] @@ -1515,7 +1516,7 @@ class AppBase: break - print(action["parameters"]) + #print(action["parameters"]) for parameter in newparams: action["parameters"].append(parameter) @@ -1841,10 +1842,10 @@ class AppBase: #print(f"Error with subparam deletion of {subparam} in {multi_parameters} (2)") pass - print() - print(f"Param: {params}") - print(f"Multiparams: {multi_parameters}") - print() + #print() + #print(f"Param: {params}") + #print(f"Multiparams: {multi_parameters}") + #print() if not multiexecution: #newparams.append({ diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 78ef796e..c48c52c3 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -6103,6 +6103,11 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { log.Printf("[INFO] Starting app hotloading") + cacheKey := fmt.Sprintf("workflowapps-sorted-100") + requestCache.Delete(cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") + requestCache.Delete(cacheKey) + // Just need to be logged in // FIXME - should have some permissions? user, err := handleApiAuthentication(resp, request) @@ -6135,6 +6140,11 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { return } + cacheKey = fmt.Sprintf("workflowapps-sorted-100") + requestCache.Delete(cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") + requestCache.Delete(cacheKey) + resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } @@ -6737,7 +6747,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin if len(removeApps) > 0 { for _, item := range removeApps { - log.Printf("[WARNING] Removing duplicate: %s", item) + log.Printf("[WARNING] Removing duplicate app: %s", item) err = DeleteKey(ctx, "workflowapp", item) if err != nil { log.Printf("[ERROR] Failed deleting duplicate %s: %s", item, err) @@ -6758,15 +6768,17 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin continue } - err = increaseStatisticsField(ctx, "total_apps_created", workflowapp.ID, 1, "") - if err != nil { - log.Printf("Failed to increase total apps created stats: %s", err) - } + /* + err = increaseStatisticsField(ctx, "total_apps_created", workflowapp.ID, 1, "") + if err != nil { + log.Printf("Failed to increase total apps created stats: %s", err) + } - err = increaseStatisticsField(ctx, "total_apps_loaded", workflowapp.ID, 1, "") - if err != nil { - log.Printf("Failed to increase total apps loaded stats: %s", err) - } + err = increaseStatisticsField(ctx, "total_apps_loaded", workflowapp.ID, 1, "") + if err != nil { + log.Printf("Failed to increase total apps loaded stats: %s", err) + } + */ //log.Printf("Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion) @@ -6801,6 +6813,12 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin return buildLaterFirst, buildLaterList, err } + // This is getting silly + cacheKey := fmt.Sprintf("workflowapps-sorted-100") + requestCache.Delete(cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") + requestCache.Delete(cacheKey) + //log.Printf("BUILDLATERFIRST: %d, BUILDLATERLIST: %d", len(buildLaterFirst), len(buildLaterList)) if len(extra) == 0 { log.Printf("[INFO] Starting build of %d containers (FIRST)", len(buildLaterFirst)) @@ -6811,6 +6829,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin } else { if len(item.Tags) > 0 { log.Printf("[INFO] Successfully built image %s", item.Tags[0]) + } else { log.Printf("[INFO] Successfully built Docker image") } diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 6dd31e3c..14e7d5af 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -2450,7 +2450,8 @@ const AngularWorkflow = (props) => { authentication: [], execution_variable: undefined, example: example, - category: app.categories !== null && app.categories !== undefined && app.categories.length > 0 ? app.categories[0] : "" + category: app.categories !== null && app.categories !== undefined && app.categories.length > 0 ? app.categories[0] : "", + authentication_id: "", } // FIXME: overwrite category if the ACTION chosen has a different category @@ -2511,6 +2512,56 @@ const AngularWorkflow = (props) => { console.log("SHOULD STITCH WITH STARTNODE") cy.add(edgeToBeAdded) } + + // AUTHENTICATION + if (app.authentication.required) { + // Setup auth here :) + const authenticationOptions = [] + var findAuthId = "" + if (newAppData.authentication_id !== null && newAppData.authentication_id !== undefined && newAppData.authentication_id.length > 0) { + findAuthId = newAppData.authentication_id + } + + var tmpAuth = JSON.parse(JSON.stringify(appAuthentication)) + for (var key in tmpAuth) { + var item = tmpAuth[key] + + const newfields = {} + for (var filterkey in item.fields) { + newfields[item.fields[filterkey].key] = item.fields[filterkey].value + } + + item.fields = newfields + if (item.app.name === app.name) { + authenticationOptions.push(item) + if (item.id === findAuthId) { + newAppData.selectedAuthentication = item + } + } + } + + if (authenticationOptions !== undefined && authenticationOptions !== null && authenticationOptions.length > 0) { + for (var key in authenticationOptions) { + const option = authenticationOptions[key] + if (option.active) { + newAppData.selectedAuthentication = option + newAppData.authentication_id = option.id + break + } + } + } + + //newAppData.authentication = authenticationOptions + //if (newAppData.selectedAuthentication === null || newAppData.selectedAuthentication === undefined || newAppData.selectedAuthentication.length === "") { + // newAppData.selectedAuthentication = {} + //} else { + // console.log("CAN WE SELECT AUTH?: ", authenticationOptions) + //} + } else { + newAppData.authentication = [] + newAppData.authentication_id = "" + newAppData.selectedAuthentication = {} + } workflow.actions.push(newAppData) setWorkflow(workflow) diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 036cc4be..e4ae6b40 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -2010,11 +2010,11 @@ const AppCreator = (props) => { const categories = [ "Communication", "Cases", - "EDR", - "Intel", "SIEM", - "Network", "Assets", + "Intel", + "IAM", + "Network", "Eradication", "Other", ] From 62ca9e932172d06fb67483734330932941f04d18 Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 7 Mar 2021 16:57:25 +0100 Subject: [PATCH 27/57] Set branch/condition failures to SKIPPED --- backend/app_sdk/app_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 219506b7..1cb3dc80 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -1404,7 +1404,7 @@ class AppBase: if not branchcheck: self.logger.info("Failed one or more branch conditions.") action_result["result"] = tmpresult - action_result["status"] = "FAILURE" + action_result["status"] = "SKIPPED" try: ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result) self.logger.info("Result: %d" % ret.status_code) @@ -1430,7 +1430,7 @@ class AppBase: try: func = getattr(self, actionname, None) if func == None: - self.logger.debug("Failed executing %s because func is None." % actionname) + self.logger.debug(f"Failed executing {actionname} because func is None.") action_result["status"] = "FAILURE" action_result["result"] = "Function %s doesn't exist." % actionname elif callable(func): From c7ba0a5a138f6629089bec041a81a3e4a8bb75d8 Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 7 Mar 2021 16:58:08 +0100 Subject: [PATCH 28/57] #279: Changed codegen to be based on params --- backend/app_sdk/build.sh | 2 +- backend/go-app/codegen.go | 22 +++++++++++++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 25fa59d6..ffed66c5 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,6 +1,6 @@ #!/bin/bash NAME=shuffle-app_sdk -VERSION=0.8.60 +VERSION=0.8.61 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index a9b4c1b6..71802bf4 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -260,9 +260,14 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet queryString += ", " } + /* + queryData += fmt.Sprintf(` + if %s: + url += f"&%s={%s}"`, query, query, query) + */ queryData += fmt.Sprintf(` if %s: - url += f"&%s={%s}"`, query, query, query) + params["%s"] = %s`, query, query, query) } } else { //log.Printf("No optional queries?") @@ -384,6 +389,7 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet // Extra param for authentication scheme(s) // The last weird one is the body.. Tabs & spaces sucks. data := fmt.Sprintf(` async def %s(self%s%s%s%s%s%s%s): + params={} %s url=f"%s%s" %s @@ -392,7 +398,7 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet %s %s %s - ret = requests.%s(url, headers=headers%s%s%s%s) + ret = requests.%s(url, headers=headers, params=params%s%s%s%s) try: return ret.json() except json.decoder.JSONDecodeError: @@ -423,13 +429,11 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet ) // Use lowercase when checking - /* - if strings.Contains(functionname, "filter") { - //log.Printf("FUNCTION: %s", data) - log.Println(data) - log.Printf("Queries: %s", queryString) - } - */ + if strings.Contains(functionname, "login") { + //log.Printf("FUNCTION: %s", data) + log.Println(data) + log.Printf("Queries: %s", queryString) + } //log.Printf(data) return functionname, data From db1abbd0518d65ecbd28b543174cb60375e0b02f Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 8 Mar 2021 11:33:58 +0100 Subject: [PATCH 29/57] Minor changes for 0.8.61 --- docker-compose.yml | 8 ++++---- functions/onprem/orborus/build.sh | 2 +- functions/onprem/orborus/orborus.go | 2 +- functions/onprem/worker/build.sh | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index cc260615..d50df814 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: frontend: #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.60 + image: ghcr.io/frikky/shuffle-frontend:0.8.61 container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -17,7 +17,7 @@ services: - backend backend: #build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.60 + image: ghcr.io/frikky/shuffle-backend:0.8.61 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: @@ -47,7 +47,7 @@ services: - database orborus: #build: ./functions/onprem/orborus - image: ghcr.io/frikky/shuffle-orborus:0.8.60 + image: ghcr.io/frikky/shuffle-orborus:0.8.61 container_name: shuffle-orborus hostname: shuffle-orborus networks: @@ -56,7 +56,7 @@ services: - /var/run/docker.sock:/var/run/docker.sock environment: - SHUFFLE_APP_SDK_VERSION=0.8.60 - - SHUFFLE_WORKER_VERSION=0.8.60 + - SHUFFLE_WORKER_VERSION=0.8.61 - ORG_ID=${ORG_ID} - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT} diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index a0fcec2c..3156eee0 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-orborus -VERSION=0.8.60 +VERSION=0.8.61 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 30d6394a..4323426b 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -259,7 +259,7 @@ func initializeImages() { log.Printf("[WARNING] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %s", appSdkVersion) } if workerVersion == "" { - workerVersion = "0.8.60" + workerVersion = "0.8.61" log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %s", workerVersion) } diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh index 907f053e..10eb7b03 100644 --- a/functions/onprem/worker/build.sh +++ b/functions/onprem/worker/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-worker -VERSION=0.8.60 +VERSION=0.8.61 echo "Running docker build with $NAME:$VERSION" #CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin . From 5e1ddc60fdc7b1fc3c4eecb11a6f067e664d8e8a Mon Sep 17 00:00:00 2001 From: frikky Date: Wed, 10 Mar 2021 10:35:21 +0100 Subject: [PATCH 30/57] Fixed worker issues --- backend/go-app/codegen.go | 12 +-- backend/go-app/main.go | 1 + backend/go-app/walkoff.go | 10 ++- frontend/src/views/AngularWorkflow.jsx | 4 +- functions/onprem/worker/worker.go | 100 ++++++++++++++++++++++++- 5 files changed, 113 insertions(+), 14 deletions(-) diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index 71802bf4..4389219f 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -429,11 +429,13 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet ) // Use lowercase when checking - if strings.Contains(functionname, "login") { - //log.Printf("FUNCTION: %s", data) - log.Println(data) - log.Printf("Queries: %s", queryString) - } + /* + if strings.Contains(functionname, "login") { + //log.Printf("FUNCTION: %s", data) + log.Println(data) + log.Printf("Queries: %s", queryString) + } + */ //log.Printf(data) return functionname, data diff --git a/backend/go-app/main.go b/backend/go-app/main.go index a2a5784c..cc620eed 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -8429,6 +8429,7 @@ func initHandlers() { // Orgs r.HandleFunc("/api/v1/orgs", handleGetOrgs).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/orgs/", handleGetOrgs).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}", handleGetOrg).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}", handleEditOrg).Methods("POST", "OPTIONS") //r.HandleFunc("/api/v1/orgs/{orgId}", handleEditOrg).Methods("POST", "OPTIONS") diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index c48c52c3..c8acad84 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -7056,8 +7056,12 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) { } if err != nil { - log.Printf("Cursorerror: %s", err) - break + if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") { + log.Printf("[WARNING] Cursorerror in app grab WARNING: %s", err) + } else { + log.Printf("[ERROR] Cursorerror in app grab: %s", err) + break + } } else { //log.Printf("NEXTCURSOR: %s", nextCursor) nextStr := fmt.Sprintf("%s", nextCursor) @@ -7115,7 +7119,7 @@ func getAllSchedules(ctx context.Context, orgId string) ([]ScheduleOld, error) { //FIXME: Add cursor func getAllWorkflowApps(ctx context.Context, maxLen int) ([]WorkflowApp, error) { var apps []WorkflowApp - query := datastore.NewQuery("workflowapp").Order("-edited").Limit(20) + query := datastore.NewQuery("workflowapp").Order("-edited").Limit(10) //query := datastore.NewQuery("workflowapp").Order("-edited").Limit(40) cacheKey := fmt.Sprintf("workflowapps-sorted-%d", maxLen) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 14e7d5af..f328fbb7 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -342,7 +342,7 @@ const AngularWorkflow = (props) => { setAuthenticationModalOpen(false) // Needs a refresh with the new authentication.. - alert.success("Successfully saved new app auth") + //alert.success("Successfully saved new app auth") } }) .catch(error => { @@ -1582,7 +1582,7 @@ const AngularWorkflow = (props) => { if (elements.length === 0 && !graphSetup && Object.getOwnPropertyNames(workflow).length > 0) { setGraphSetup(true) setupGraph() - } else if (!established && cy !== undefined && apps.length > 0 && Object.getOwnPropertyNames(workflow).length > 0){ + } else if (!established && cy !== undefined && apps !== null && apps !== undefined && apps.length > 0 && Object.getOwnPropertyNames(workflow).length > 0){ setEstablished(true) cy.edgehandles({ handleNodes: (el) => el.isNode(), diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index f3d9250a..02718009 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -19,6 +19,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" + //"github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/mount" dockerclient "github.com/docker/docker/client" @@ -776,7 +777,7 @@ type AppExecutionExample struct { // removes every container except itself (worker) func shutdown(workflowExecution WorkflowExecution, nodeId string, reason string, handleResultSend bool) { - log.Printf("[INFO] Shutdown started") + log.Printf("[INFO] Shutdown started with reason %s", reason) //reason := "Error in execution" sleepDuration := 1 @@ -881,6 +882,7 @@ func shutdown(workflowExecution WorkflowExecution, nodeId string, reason string, // Deploys the internal worker whenever something happens func deployApp(cli *dockerclient.Client, image string, identifier string, env []string) error { // form basic hostConfig + ctx := context.Background() hostConfig := &container.HostConfig{ LogConfig: container.LogConfig{ Type: "json-file", @@ -942,7 +944,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] } cont, err := cli.ContainerCreate( - context.Background(), + ctx, config, hostConfig, nil, @@ -955,7 +957,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] return err } - err = cli.ContainerStart(context.Background(), cont.ID, types.ContainerStartOptions{}) + err = cli.ContainerStart(ctx, cont.ID, types.ContainerStartOptions{}) if err != nil { log.Printf("[ERROR] Failed to start container in environment %s: %s", environment, err) //shutdown(workflowExecution, workflowExecution.Workflow.ID, true) @@ -963,6 +965,73 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] } log.Printf("[INFO] Container %s was created for %s", cont.ID, identifier) + + // Waiting to see if it exits.. Stupid, but stable(r) + time.Sleep(2 * time.Second) + + stats, err := cli.ContainerInspect(ctx, cont.ID) + if err != nil { + log.Printf("[ERROR] Failed getting container stats") + } else { + //log.Printf("[INFO] Info for container: %#v", stats) + //log.Printf("%#v", stats.Config) + //log.Printf("%#v", stats.ContainerJSONBase.State) + log.Printf("STATUS: %s", stats.ContainerJSONBase.State.Status) + if stats.ContainerJSONBase.State.Status == "exited" { + logOptions := types.ContainerLogsOptions{ + ShowStdout: true, + } + + out, err := cli.ContainerLogs(ctx, cont.ID, logOptions) + if err != nil { + log.Printf("[INFO] Failed getting logs: %s", err) + } else { + log.Printf("IN ELSE FOR DEPLOY") + buf := new(strings.Builder) + io.Copy(buf, out) + logs := buf.String() + log.Printf("Logs: %s", logs) + //log.Printf(logs) + // check errors + /* + if strings.Contains(logs, "Error") { + log.Printf("ERROR IN %s?", cont.ID) + log.Println(logs) + //return errors.New(fmt.Sprintf("ERROR FROM CONTAINER %s", cont.ID)) + } else { + log.Printf("NORMAL EXEC OF %s?", cont.ID) + } + */ + } + + log.Printf("ERROR IN CONTAINER DEPLOYMENT - ITS EXITED!") + + return errors.New(fmt.Sprintf(`{"success": false, "reason": "Container %s exited prematurely.","debug": "docker logs -f %s"}`, cont.ID, cont.ID)) + } + } + + /* + //log.Printf("%#v", stats.Config.Status) + //ContainerJSONtoConfig(cj dockType.ContainerJSON) ContainerConfig { + listOptions := types.ContainerListOptions{ + Filters: filters.Args{ + map[string][]string{"ancestor": {":"}}, + }, + } + containers, err := cli.ContainerList(ctx, listOptions) + */ + + //log.Printf("%#v", cont.Status) + //config := ContainerJSONtoConfig(stats) + //log.Printf("CONFIG: %#v", config) + + /* + logOptions := types.ContainerLogsOptions{ + ShowStdout: true, + } + + */ + containerIds = append(containerIds, cont.ID) return nil } @@ -1570,6 +1639,10 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { if cleanupEnv == "true" { err = deployApp(dockercli, images[0], identifier, env) if err != nil { + if strings.Contains(err.Error(), "exited prematurely") { + shutdown(workflowExecution, action.ID, err.Error(), true) + } + log.Printf("[WARNING] Failed CLEANUP execution. Downloading image remotely.") reader, err := dockercli.ImagePull(context.Background(), image, pullOptions) if err != nil { @@ -1595,6 +1668,10 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { if err != nil { log.Printf("[ERROR] Failed deploying image for the FOURTH time. Aborting if the image doesn't exist") + if strings.Contains(err.Error(), "exited prematurely") { + shutdown(workflowExecution, action.ID, err.Error(), true) + } + if strings.Contains(err.Error(), "No such image") { //log.Printf("[WARNING] Failed deploying %s from image %s: %s", identifier, image, err) log.Printf("[ERROR] Image doesn't exist. Shutting down") @@ -1606,6 +1683,10 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { err = deployApp(dockercli, image, identifier, env) if err != nil { + if strings.Contains(err.Error(), "exited prematurely") { + shutdown(workflowExecution, action.ID, err.Error(), true) + } + // Trying to replace with lowercase to deploy again. This seems to work with Dockerhub well. // FIXME: Should try to remotely download directly if this persists. image = fmt.Sprintf("%s:%s_%s", baseimagename, strings.ToLower(action.AppName), action.AppVersion) @@ -1615,6 +1696,10 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { err = deployApp(dockercli, image, identifier, env) if err != nil { + if strings.Contains(err.Error(), "exited prematurely") { + shutdown(workflowExecution, action.ID, err.Error(), true) + } + image = fmt.Sprintf("%s/%s:%s_%s", registryName, baseimagename, strings.ToLower(action.AppName), action.AppVersion) if strings.Contains(image, " ") { image = strings.ReplaceAll(image, " ", "-") @@ -1622,6 +1707,10 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { err = deployApp(dockercli, image, identifier, env) if err != nil { + if strings.Contains(err.Error(), "exited prematurely") { + shutdown(workflowExecution, action.ID, err.Error(), true) + } + log.Printf("[WARNING] Failed deploying image THRICE. Attempting to download the latter as last resort.") reader, err := dockercli.ImagePull(context.Background(), image, pullOptions) if err != nil { @@ -1645,8 +1734,11 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { err = deployApp(dockercli, image, identifier, env) if err != nil { - log.Printf("[ERROR] Failed deploying image for the FOURTH time. Aborting if the image doesn't exist") + if strings.Contains(err.Error(), "exited prematurely") { + shutdown(workflowExecution, action.ID, err.Error(), true) + } + if strings.Contains(err.Error(), "No such image") { //log.Printf("[WARNING] Failed deploying %s from image %s: %s", identifier, image, err) log.Printf("[ERROR] Image doesn't exist. Shutting down") From 6c09375f33d56b45d49ca889ec3217603ba6c3e4 Mon Sep 17 00:00:00 2001 From: frikky Date: Wed, 10 Mar 2021 17:16:55 +0100 Subject: [PATCH 31/57] Workflow completion fixes --- backend/go-app/walkoff.go | 38 ++++++++++++++++-- docker-compose.yml | 8 ++-- frontend/src/views/Admin.jsx | 12 ++++-- frontend/src/views/AngularWorkflow.jsx | 6 +++ functions/onprem/orborus/build.sh | 2 +- functions/onprem/orborus/orborus.go | 2 +- functions/onprem/worker/Dockerfile | 10 ++--- functions/onprem/worker/build.sh | 2 +- functions/onprem/worker/worker.go | 55 +++++++++++++++----------- 9 files changed, 94 insertions(+), 41 deletions(-) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index c8acad84..8f28f72e 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -17,6 +17,9 @@ import ( "strings" "time" + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" + "cloud.google.com/go/datastore" scheduler "cloud.google.com/go/scheduler/apiv1" gyaml "github.com/ghodss/yaml" @@ -977,6 +980,21 @@ func validateNewWorkerExecution(body []byte) error { return errors.New(fmt.Sprintf("Bad length of trigger: %d (probably normal app)", len(execution.Workflow.Triggers))) } + if execution.Status == "EXECUTING" { + log.Printf("[INFO] Inside executing.") + extra := 0 + for _, trigger := range execution.Workflow.Triggers { + //log.Printf("Appname trigger (0): %s", trigger.AppName) + if trigger.AppName == "User Input" || trigger.AppName == "Shuffle Workflow" { + extra += 1 + } + } + + if len(execution.Workflow.Actions)+extra == len(execution.Results) { + execution.Status = "FINISHED" + } + } + // FIXME: Add extra here //executionLength := len(baseExecution.Workflow.Actions) //if executionLength != len(execution.Results) { @@ -986,7 +1004,7 @@ func validateNewWorkerExecution(body []byte) error { //log.Printf("\n\nSHOULD SET BACKEND DATA FOR EXEC \n\n") err = setWorkflowExecution(ctx, execution, true) if err == nil { - log.Printf("[INFO] Set workflowexecution based on new worker (>0.8.53) for execution %s", baseExecution.ExecutionId) + log.Printf("[INFO] Set workflowexecution based on new worker (>0.8.53) for execution %s. Actions: %d, Triggers: %d, Results: %d", execution.ExecutionId, len(execution.Workflow.Actions), len(execution.Workflow.Triggers), len(execution.Results)) //log.Printf("[INFO] Successfully set the execution to wait.") } else { log.Printf("[WARNING] Failed to set the execution to wait.") @@ -3106,7 +3124,7 @@ func cleanupExecutions(resp http.ResponseWriter, request *http.Request) { user, err := handleApiAuthentication(resp, request) if err != nil { - log.Printf("Api authentication failed in execute workflow: %s", err) + log.Printf("[INFO] Api authentication failed in cleanup executions: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "message": "Not authenticated"}`)) return @@ -3871,7 +3889,7 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { user, err := handleApiAuthentication(resp, request) if err != nil { - log.Printf("Api authentication failed in execute workflow: %s", err) + log.Printf("[INFO] Api authentication failed in execute workflow: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -6549,8 +6567,20 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin buildLaterFirst := []buildLaterStruct{} buildLaterList := []buildLaterStruct{} - // It's here to prevent getting them in every iteration ctx := context.Background() + if forceUpdate { + dockercli, err := client.NewEnvClient() + if err == nil { + _, err := dockercli.ImagePull(ctx, "frikky/shuffle:app_sdk", types.ImagePullOptions{}) + if err != nil { + log.Printf("[WARNING] Failed to download apps with the new App SDK: %s", err) + } + } else { + log.Printf("[WARNING] Failed to download apps with the new App SDK because of docker cli: %s", err) + } + } + + // It's here to prevent getting them in every iteration for _, file := range dir { if len(onlyname) > 0 && file.Name() != onlyname { continue diff --git a/docker-compose.yml b/docker-compose.yml index d50df814..ff2bf5d4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: frontend: #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.61 + image: ghcr.io/frikky/shuffle-frontend:0.8.62 container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -17,7 +17,7 @@ services: - backend backend: #build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.61 + image: ghcr.io/frikky/shuffle-backend:0.8.62 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: @@ -47,7 +47,7 @@ services: - database orborus: #build: ./functions/onprem/orborus - image: ghcr.io/frikky/shuffle-orborus:0.8.61 + image: ghcr.io/frikky/shuffle-orborus:0.8.62 container_name: shuffle-orborus hostname: shuffle-orborus networks: @@ -56,7 +56,7 @@ services: - /var/run/docker.sock:/var/run/docker.sock environment: - SHUFFLE_APP_SDK_VERSION=0.8.60 - - SHUFFLE_WORKER_VERSION=0.8.61 + - SHUFFLE_WORKER_VERSION=0.8.62 - ORG_ID=${ORG_ID} - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT} diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index 8b1d3ec5..7cee4bbe 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -377,6 +377,11 @@ const Admin = (props) => { } const handleGetOrg = (orgId) => { + if (orgId.length === 0) { + alert.error("Organization ID not defined. Please contact us on https://shuffler.io if this persists logout.") + return + } + // Just use this one? var baseurl = globalUrl const url = baseurl + '/api/v1/orgs/'+orgId @@ -870,7 +875,8 @@ const Admin = (props) => { }) .then((response) => { if (response.status !== 200) { - window.location.pathname = "/workflows" + // Ahh, this happens because they're not admin + // window.location.pathname = "/workflows" return } @@ -1359,14 +1365,14 @@ const Admin = (props) => { const cancelSubscriptions = (subscription_id) => { console.log(selectedOrganization) + const orgId = selectedOrganization.id const data = { "subscription_id": subscription_id, "action": "cancel", "org_id": selectedOrganization.id, } - - const url = globalUrl + `/api/v1/orgs/${selectedOrganization.id}`; + const url = globalUrl + `/api/v1/orgs/${orgId}`; fetch(url, { mode: 'cors', method: 'POST', diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index f328fbb7..3485f90c 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -382,6 +382,12 @@ const AngularWorkflow = (props) => { if (execution !== null && execution !== undefined) { setExecutionData(execution) setExecutionModalView(1) + start() + + setExecutionRequest({ + "execution_id": execution.execution_id, + "authorization": execution.authorization, + }) const newitem = removeParam("execution_id", cursearch) props.history.push(curpath+newitem) diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index 3156eee0..4c0b9450 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-orborus -VERSION=0.8.61 +VERSION=0.8.62 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 4323426b..faae0c38 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -259,7 +259,7 @@ func initializeImages() { log.Printf("[WARNING] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %s", appSdkVersion) } if workerVersion == "" { - workerVersion = "0.8.61" + workerVersion = "0.8.62" log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %s", workerVersion) } diff --git a/functions/onprem/worker/Dockerfile b/functions/onprem/worker/Dockerfile index a95b099e..3bf1541a 100644 --- a/functions/onprem/worker/Dockerfile +++ b/functions/onprem/worker/Dockerfile @@ -5,11 +5,11 @@ WORKDIR /app COPY worker.go /app/worker.go RUN go env -w GO111MODULE=auto -RUN go get -u github.com/docker/docker/api/types -RUN go get -u github.com/docker/docker/api/types/container -RUN go get -u github.com/docker/docker/client -RUN go get -u github.com/gorilla/mux -RUN go get -u github.com/patrickmn/go-cache +RUN go get github.com/docker/docker/api/types +RUN go get github.com/docker/docker/api/types/container +RUN go get github.com/docker/docker/client +RUN go get github.com/gorilla/mux +RUN go get github.com/patrickmn/go-cache RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker . diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh index 10eb7b03..3f2a483e 100644 --- a/functions/onprem/worker/build.sh +++ b/functions/onprem/worker/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-worker -VERSION=0.8.61 +VERSION=0.8.62 echo "Running docker build with $NAME:$VERSION" #CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin . diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 02718009..3e15c282 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -777,11 +777,11 @@ type AppExecutionExample struct { // removes every container except itself (worker) func shutdown(workflowExecution WorkflowExecution, nodeId string, reason string, handleResultSend bool) { - log.Printf("[INFO] Shutdown started with reason %s", reason) + log.Printf("[INFO] Shutdown (%s) started with reason %s", workflowExecution.Status, reason) //reason := "Error in execution" sleepDuration := 1 - if handleResultSend { + if handleResultSend && requestsSent < 2 { data, err := json.Marshal(workflowExecution) if err == nil { sendResult(workflowExecution, data) @@ -829,7 +829,7 @@ func shutdown(workflowExecution WorkflowExecution, nodeId string, reason string, //fmt.Println(url.QueryEscape(query)) fullUrl += path - log.Printf("Abort URL: %s", fullUrl) + log.Printf("[INFO] Abort URL: %s", fullUrl) req, err := http.NewRequest( "GET", @@ -976,7 +976,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] //log.Printf("[INFO] Info for container: %#v", stats) //log.Printf("%#v", stats.Config) //log.Printf("%#v", stats.ContainerJSONBase.State) - log.Printf("STATUS: %s", stats.ContainerJSONBase.State.Status) + log.Printf("[INFO] EXECUTION STATUS: %s", stats.ContainerJSONBase.State.Status) if stats.ContainerJSONBase.State.Status == "exited" { logOptions := types.ContainerLogsOptions{ ShowStdout: true, @@ -991,6 +991,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] io.Copy(buf, out) logs := buf.String() log.Printf("Logs: %s", logs) + //log.Printf(logs) // check errors /* @@ -2290,6 +2291,8 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting execution"}`))) return } + + log.Printf(`[INFO] Got result %s from %s`, actionResult.Status, actionResult.Action.ID) resultLength := len(workflowExecution.Results) dbSave := false setExecution := true @@ -2448,6 +2451,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl for index, item := range workflowExecution.Results { if item.Action.ID == actionResult.Action.ID { found = true + if item.Status == actionResult.Status { skip = true } @@ -2478,38 +2482,45 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl log.Printf("[INFO] Updating %s in workflow %s from %s to %s", actionResult.Action.ID, workflowExecution.ExecutionId, workflowExecution.Results[outerindex].Status, actionResult.Status) workflowExecution.Results[outerindex] = actionResult } else { - log.Printf("[INFO] Setting value of %s in workflow %s to %s", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status) workflowExecution.Results = append(workflowExecution.Results, actionResult) + log.Printf("[INFO] Setting value (1) of %s in execution %s to %s. New result length: %d", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status, len(workflowExecution.Results)) } } else { - log.Printf("[INFO] Setting value of %s in workflow %s to %s", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status) workflowExecution.Results = append(workflowExecution.Results, actionResult) + log.Printf("[INFO] Setting value (2) of %s in execution %s to %s. New result length: %d", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status, len(workflowExecution.Results)) } // FIXME: Have a check for skippednodes and their parents - for resultIndex, result := range workflowExecution.Results { - if result.Status != "SKIPPED" { - continue - } + /* + for resultIndex, result := range workflowExecution.Results { + if result.Status != "SKIPPED" { + continue + } - // Checks if all parents are skipped or failed. Otherwise removes them from the results - for _, branch := range workflowExecution.Workflow.Branches { - if branch.DestinationID == result.Action.ID { - for _, subresult := range workflowExecution.Results { - if subresult.Action.ID == branch.SourceID { - if subresult.Status != "SKIPPED" && subresult.Status != "FAILURE" { - log.Printf("SUBRESULT PARENT STATUS: %s", subresult.Status) - log.Printf("Should remove resultIndex: %d", resultIndex) + // Checks if all parents are skipped or failed. + // Otherwise removes them from the results + for _, branch := range workflowExecution.Workflow.Branches { + if branch.DestinationID == result.Action.ID { + for _, subresult := range workflowExecution.Results { + if subresult.Action.ID == branch.SourceID { + if subresult.Status != "SKIPPED" && subresult.Status != "FAILURE" { + //log.Printf("SUBRESULT PARENT STATUS: %s", subresult.Status) + //log.Printf("Should remove resultIndex: %d", resultIndex) - workflowExecution.Results = append(workflowExecution.Results[:resultIndex], workflowExecution.Results[resultIndex+1:]...) + // FIXME: Reinstate this? + //workflowExecution.Results = append(workflowExecution.Results[:resultIndex], workflowExecution.Results[resultIndex+1:]...) + _ = resultIndex - break + break + } } } } } } - } + + log.Printf("NEW LENGTH: %d", len(workflowExecution.Results)) + */ extraInputs := 0 for _, trigger := range workflowExecution.Workflow.Triggers { @@ -2697,7 +2708,7 @@ func sendResult(workflowExecution WorkflowExecution, data []byte) { } func validateFinished(workflowExecution WorkflowExecution) { - log.Printf("[INFO] Status: %s, Actions: %d, Extra: %d, Results: %d\n", workflowExecution.Status, len(workflowExecution.Workflow.Actions), extra, len(workflowExecution.Results)) + log.Printf("[INFO] VALIDATION. Status: %s, Actions: %d, Extra: %d, Results: %d\n", workflowExecution.Status, len(workflowExecution.Workflow.Actions), extra, len(workflowExecution.Results)) //if len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions)+extra { if (len(environments) == 1 && requestsSent == 0 && len(workflowExecution.Results) >= 1) || (len(workflowExecution.Results) >= len(workflowExecution.Workflow.Actions) && len(workflowExecution.Workflow.Actions) > 0) { From 61f51592354952989a619c5cafc4dfcca1dea751 Mon Sep 17 00:00:00 2001 From: frikky Date: Wed, 10 Mar 2021 17:33:17 +0100 Subject: [PATCH 32/57] Fixed a bug with Workers not getting backend access because of Kubernetes changes --- functions/onprem/orborus/orborus.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index faae0c38..015d2986 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -101,7 +101,7 @@ func getThisContainerId() { log.Printf("[INFO] Running containerized in Docker!") default: - fCol = "0" // for backward-compatibility with production + fCol = "3" // for backward-compatibility with production log.Printf("[WARNING] RUNNING_MODE not set - defaulting to Docker (NOT Kubernetes).") } @@ -119,14 +119,14 @@ func getThisContainerId() { //docker-76c537e9a4b7c7233011f5d70e6b7f2d600b6413ac58a96519b8dca7a3f7117a.scope } } else { - if fCol != "0" { + if fCol == "0" { containerId = "shuffle-orborus" log.Printf("[WARNING] Failed getting container ID: %s", err) } } } - log.Printf("Started with containerId %s", containerId) + log.Printf(`[INFO] Started with containerId "%s"`, containerId) } // Deploys the internal worker whenever something happens From 2a825dd1ea66e7482a68b1d837eccb57773a2d26 Mon Sep 17 00:00:00 2001 From: frikky Date: Thu, 11 Mar 2021 16:50:58 +0100 Subject: [PATCH 33/57] #293: Entire fix in a single button --- backend/app_sdk/app_base.py | 171 ++++++++++++++++- backend/app_sdk/build.sh | 2 +- backend/go-app/main.go | 245 +++++++++++++++++++++++++ backend/go-app/walkoff.go | 31 ++-- frontend/src/views/AngularWorkflow.jsx | 83 +++++++-- frontend/src/views/Workflows.jsx | 3 + functions/onprem/orborus/Dockerfile | 1 - functions/onprem/orborus/go.mod | 3 +- functions/onprem/orborus/go.sum | 7 + functions/onprem/worker/Dockerfile | 17 +- functions/onprem/worker/build.sh | 2 +- functions/onprem/worker/worker.go | 83 ++++++++- 12 files changed, 594 insertions(+), 54 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 1cb3dc80..c369533b 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -9,6 +9,7 @@ import requests import urllib.parse import http.client import urllib3 +import hashlib class AppBase: __version__ = None @@ -91,6 +92,135 @@ class AppBase: else: return {()} + # Handles unique fields by negoiating with the backend + def validate_unique_fields(self, params): + #print("IN THE UNIQUE FIELDS PLACE!") + + newlist = [params] + if isinstance(params, list): + #print("ITS A LIST!") + newlist = params + + #self.full_execution = os.getenv("FULL_EXECUTION", "") + #print(len(params)) + #print(params.items()) + #print(list(params.items())) + #print(f"PARAM: {params}") + #print(f"NEWLIST: {newlist}") + + # FIXME: Also handle MULTI PARAM + values = [] + param_names = [] + all_values = {} + index = 0 + for outerparam in newlist: + + #print(f"INNERTYPE: {type(outerparam)}") + #print(f"HANDLING PARAM {key}") + param_value = "" + for key, value in outerparam.items(): + #print("KEY: %s" % key) + #value = params[key] + for param in self.action["parameters"]: + try: + if param["name"] == key and param["unique_toggled"]: + print(f"FOUND: {key} with param {param}!") + if isinstance(value, dict) or isinstance(value, list): + try: + value = json.dumps(value) + except json.decoder.JSONDecodeError as e: + print(f"Error in json decode for param {value}: {e}") + continue + elif isinstance(value, int) or isinstance(value, float): + value = str(value) + elif value == False: + value = "False" + elif value == True: + value = "True" + + print(f"VALUE APPEND: {value}") + param_value += value + + if param["name"] not in param_names: + param_names.append(param["name"]) + except (KeyError, NameError) as e: + print(f"Key/NameError in param handler: {e}") + + print(f"OUTER VALUE: {param_value}") + if len(param_value) > 0: + md5 = hashlib.md5(param_value.encode('utf-8')).hexdigest() + values.append(md5) + all_values[md5] = { + "index": index, + } + + index += 1 + + # When in here, it means it should be unique + # Should this be done by the backend? E.g. ask it if the value is valid? + # 1. Check if it's unique towards key:value store in org for action + # 2. Check if COMBINATION is unique towards key:value store of action for org + # 3. Have a workflow configuration for unique ID's in unison or per field? E.g. if toggled, then send a hash of all fields together alphabetically, but if not, send one field at a time + + # org_id = full_execution["workflow"]["execution_org"]["id"] + + # USE ARRAY? + + new_params = [] + if len(values) > 0: + org_id = self.full_execution["workflow"]["execution_org"]["id"] + data = { + "append": True, + "workflow_check": False, + "authorization": self.authorization, + "execution_ref": self.current_execution_id, + "org_id": org_id, + "values": [{ + "app": self.action["app_name"], + "action": self.action["name"], + "parameternames": param_names, + "parametervalues": values, + }] + } + + #print(f"DATA: {data}") + # 1594869a676630b397bc34f7dc0951a3 + + #print(f"VALUE URL: {url}") + #print(f"RET: {ret.text}") + #print(f"ID: {ret.status_code}") + url = f"{self.url}/api/v1/orgs/{org_id}/validate_app_values" + ret = requests.post(url, json=data) + if ret.status_code == 200: + json_value = ret.json() + if len(json_value["found"]) > 0: + modifier = 0 + for item in json_value["found"]: + print(f"Should remove {item}") + + try: + print(f"FOUND: {all_values[item]}") + print(f"SHOULD REMOVE INDEX: {all_values[item]['index']}") + + try: + newlist.pop(all_values[item]["index"]-modifier) + modifier += 1 + except IndexError as e: + print(f"Error popping value from array: {e}") + except (NameError, KeyError) as e: + print(f"Failed removal: {e}") + + + #return False + else: + print("None of the items were found!") + return newlist + else: + print(f"[WARNING] Failed checking values with status code {ret.status_code}!") + + #return True + return newlist + # Returns a list of all the executions to be done in the inner loop # FIXME: Doesn't take into account whether you actually WANT to loop or not # Check if the last part of the value is #? @@ -347,6 +477,26 @@ class AppBase: ret = [] param_multiplier = await self.get_param_multipliers(newparams) + # FIXME: This does a deduplication of the data + new_params = self.validate_unique_fields(param_multiplier) + print(f"NEW PARAMS: {new_params}") + if len(new_params) == 0: + print(f"No ID's to handle for validation") + else: + #subparams = new_params + print(f"NEW PARAMS: {new_params}") + + #print("Returned with newparams of length %d", len(new_params)) + #if isinstance(new_params, list) and len(new_params) == 1: + # params = new_params[0] + #else: + # print("[WARNING] SHOULD STOP EXECUTION BECAUSE FIELDS AREN'T UNIQUE") + # action_result["status"] = "SKIPPED" + # action_result["result"] = f"A non-unique value was found" + # action_result["completed_at"] = int(time.time()) + # self.send_result(action_result, headers, stream_path) + # return + print("[INFO] Multiplier length: %d" % len(param_multiplier)) for subparams in param_multiplier: print(f"SUBPARAMS IN MULTI: {subparams}") @@ -1398,6 +1548,7 @@ class AppBase: return True, "" + # THE START IS ACTUALLY RIGHT HERE :O # Checks whether conditions are met, otherwise set branchcheck, tmpresult = check_branch_conditions(action, fullexecution) @@ -1848,14 +1999,19 @@ class AppBase: #print() if not multiexecution: - #newparams.append({ - # "name": val["key"], - # "value": val["value"], - # "variant": "STATIC_VALUE", - # "id": "body_replacement", - #}) + # Runs a single iteration here + new_params = self.validate_unique_fields(params) + print(f"Returned with newparams of length {len(new_params)}") + if isinstance(new_params, list) and len(new_params) == 1: + params = new_params[0] + else: + print("[WARNING] SHOULD STOP EXECUTION BECAUSE FIELDS AREN'T UNIQUE") + action_result["status"] = "SKIPPED" + action_result["result"] = f"A non-unique value was found" + action_result["completed_at"] = int(time.time()) + self.send_result(action_result, headers, stream_path) + return - #print("[INFO] APP_SDK DONE: Starting NORMAL execution of function") print("[INFO] Running normal execution\n") newres = await func(**params) print("\n[INFO] Returned from execution!")#, newres) @@ -2058,7 +2214,6 @@ class AppBase: print("Normal result - no list?") result = results - print("RESULT: %s" % result) action_result["status"] = "SUCCESS" action_result["result"] = str(result) if action_result["result"] == "": diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index ffed66c5..ae25a4a3 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,6 +1,6 @@ #!/bin/bash NAME=shuffle-app_sdk -VERSION=0.8.61 +VERSION=0.8.62 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/go-app/main.go b/backend/go-app/main.go index cc620eed..57d57e75 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -7722,6 +7722,247 @@ func handleStopCloudSync(syncUrl string, org Org) error { return nil } +func handleKeyValueCheck(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + body, err := ioutil.ReadAll(request.Body) + if err != nil { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`)) + return + } + + // Append: Checks if the value should be appended + // WorkflowCheck: Checks if the value should only check for workflow in org, or entire org + // Authorization: the Authorization to use + // ExecutionRef: Ref for the execution + // Values: The values to use + type DataValues struct { + App string + Actions string + ParameterNames []string + ParameterValues []string + } + + type ReturnData struct { + Append bool `json:"append"` + WorkflowCheck bool `json:"workflow_check"` + Authorization string `json:"authorization"` + ExecutionRef string `json:"execution_ref"` + OrgId string `json:"org_id"` + Values []DataValues `json:"values"` + } + + //for key, value := range data.Apps { + var fileId string + location := strings.Split(request.URL.String(), "/") + if location[1] == "api" { + if len(location) <= 4 { + log.Printf("Path too short: %d", len(location)) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + fileId = location[4] + } + + var tmpData ReturnData + err = json.Unmarshal(body, &tmpData) + if err != nil { + log.Printf("Failed unmarshalling test: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if tmpData.OrgId != fileId { + log.Printf("[INFO] OrgId %s and %s don't match", tmpData.OrgId, fileId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "Organization ID's don't match"}`)) + return + } + + ctx := context.Background() + + org, err := getOrg(ctx, tmpData.OrgId) + if err != nil { + log.Printf("[INFO] Organization doesn't exist: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + workflowExecution, err := getWorkflowExecution(ctx, tmpData.ExecutionRef) + if err != nil { + log.Printf("[INFO] User can't edit the org") + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "No permission to get execution"}`)) + return + } + + if workflowExecution.Authorization != tmpData.Authorization { + log.Printf("[INFO] Execution auth %s and %s don't match", workflowExecution.Authorization, tmpData.Authorization) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "Auth doesn't match"}`)) + return + } + + if workflowExecution.Status != "EXECUTING" { + log.Printf("[INFO] Workflow isn't executing and shouldn't be searching", workflowExecution.ExecutionId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "Workflow isn't executing"}`)) + return + } + + if workflowExecution.ExecutionOrg != org.Id { + log.Printf("[INFO] Org %s wasn't used to execute %s", org.Id, workflowExecution.ExecutionId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "Bad organization specified"}`)) + return + } + + // Prepared for the future~ + if len(tmpData.Values) != 1 { + log.Printf("[INFO] Filter data can only hande 1 value right now, not %d", len(tmpData.Values)) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "Can't handle multiple apps yet, just one"}`)) + return + } + + value := tmpData.Values[0] + + // FIXME: Alphabetically sort the parameternames + // FIXME: Add organization wide search, not just workflow based + + found := []string{} + notFound := []string{} + + dbKey := fmt.Sprintf("app_execution_values") + parameterNames := fmt.Sprintf("%s_%s", value.App, strings.Join(value.ParameterNames, "_")) + log.Printf("[INFO] PARAMNAME: %s", parameterNames) + if tmpData.WorkflowCheck { + //for _, item := range tmpData.Values { + // log.Printf("[INFO] Should validate if values %#v in app parameter %#v exists WITH WORKFLOW %s", item.ParameterValues, item.ParameterNames, workflowExecution.Workflow.ID) + + // FIXME: Make this alphabetical + + for _, value := range value.ParameterValues { + if len(value) == 0 { + log.Printf("Shouldn't have value of length 0!") + continue + } + + log.Printf("[INFO] Looking for value %s", value) + + q := datastore.NewQuery(dbKey).Filter("org_id =", org.Id).Filter("workflow_id =", workflowExecution.Workflow.ID).Filter("parameter_name =", parameterNames).Filter("value =", value) + foundCount, err := dbclient.Count(ctx, q) + if err != nil { + log.Printf("[WARNING] Failed getting key %s: %s", dbKey, err) + notFound = append(notFound, value) + //found = append(found, value) + continue + } + + if foundCount > 0 { + found = append(found, value) + } else { + log.Printf("[INFO] Found for %s: %d", dbKey, foundCount) + notFound = append(notFound, value) + } + } + } else { + log.Printf("[INFO] Should validate if value %s in app %s exists WITH ORG %s", workflowExecution.Workflow.ID) + + for _, value := range value.ParameterValues { + if len(value) == 0 { + log.Printf("Shouldn't have value of length 0!") + continue + } + + log.Printf("[INFO] Looking for value %s", value) + + q := datastore.NewQuery(dbKey).Filter("org_id =", org.Id).Filter("workflow_id =", "").Filter("parameter_name =", parameterNames).Filter("value =", value) + foundCount, err := dbclient.Count(ctx, q) + if err != nil { + log.Printf("[WARNING] Failed getting key %s: %s", dbKey, err) + notFound = append(notFound, value) + //found = append(found, value) + continue + } + + if foundCount > 0 { + found = append(found, value) + } else { + log.Printf("[INFO] Found for %s: %d", dbKey, foundCount) + notFound = append(notFound, value) + } + } + } + + //App string + //Actions string + //ParameterNames string + //ParamererValues []string + + appended := 0 + if tmpData.Append { + log.Printf("[INFO] Should append %d values!", len(notFound)) + dbKey := fmt.Sprintf("app_execution_values") + + //q := datastore.NewQuery(dbKey).Filter("org_id =", org.Id).Filter("workflow_id", workflowExecution.Workflow.ID).Filter("app_name =", parameterNames).Filter("value =", value) + key := datastore.NameKey(dbKey, "", nil) + type NewValue struct { + OrgId string `json:"org_id" datastore:"org_id"` + WorkflowId string `json:"workflow_id" datastore:"workflow_id"` + WorkflowExecutionId string `json:"workflow_execution_id" datastore:"workflow_execution_id"` + ParameterName string `json:"parameter_name" datastore:"parameter_name"` + Value string `json:"value" datastore:"value"` + } + + //parameterNames := strings.Join(value.ParameterNames, "_") + for _, notFoundValue := range notFound { + newRequest := NewValue{ + OrgId: org.Id, + WorkflowExecutionId: workflowExecution.ExecutionId, + ParameterName: parameterNames, + Value: notFoundValue, + } + + if tmpData.WorkflowCheck { + newRequest.WorkflowId = workflowExecution.Workflow.ID + } + + if _, err := dbclient.Put(ctx, key, &newRequest); err != nil { + log.Printf("Error adding %s to appvalue: %s", notFoundValue, err) + continue + } + + appended += 1 + log.Printf("[INFO] Added %s as new appvalue to datastore", notFoundValue) + } + } + + type returnStruct struct { + Success bool `json:"success"` + Appended int `json:"appended"` + Found []string `json:"found"` + } + + returnData := returnStruct{ + Success: true, + Appended: appended, + Found: found, + } + + b, _ := json.Marshal(returnData) + resp.WriteHeader(200) + resp.Write(b) +} + func handleEditOrg(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -8432,6 +8673,10 @@ func initHandlers() { r.HandleFunc("/api/v1/orgs/", handleGetOrgs).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}", handleGetOrg).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}", handleEditOrg).Methods("POST", "OPTIONS") + + // This is a new API that validates if a key has been seen before. + // Not sure what the best course of action is for it. + r.HandleFunc("/api/v1/orgs/{orgId}/validate_app_values", handleKeyValueCheck).Methods("POST", "OPTIONS") //r.HandleFunc("/api/v1/orgs/{orgId}", handleEditOrg).Methods("POST", "OPTIONS") // Docker orborus specific diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 8f28f72e..05990f8b 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -229,6 +229,7 @@ type WorkflowAppActionParameter struct { Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` SkipMulticheck bool `json:"skip_multicheck" datastore:"skip_multicheck" yaml:"skip_multicheck"` ValueReplace []Valuereplace `json:"value_replace" datastore:"value_replace,noindex" yaml:"value_replace,omitempty"` + UniqueToggled bool `json:"unique_toggled" datastore:"unique_toggled" yaml:"unique_toggled"` } type Valuereplace struct { @@ -1187,6 +1188,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // Find underlying nodes and add them } else { log.Printf("[WARNING] Actionresult is %s for node %s in %s. Continuing anyway because of workflow configuration.", actionResult.Status, actionResult.Action.ID, workflowExecution.ExecutionId) + // Finds ALL childnodes to set them to SKIPPED childNodes = findChildNodes(*workflowExecution, actionResult.Action.ID) @@ -6119,8 +6121,6 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("[INFO] Starting app hotloading") - cacheKey := fmt.Sprintf("workflowapps-sorted-100") requestCache.Delete(cacheKey) cacheKey = fmt.Sprintf("workflowapps-sorted-500") @@ -6149,7 +6149,7 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("[INFO] Hotloading from %s", location) + log.Printf("[INFO] Starting hotloading from %s", location) err = handleAppHotload(location, true) if err != nil { log.Printf("Failed app hotload: %s", err) @@ -6251,6 +6251,20 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) { } else { log.Printf("Updating apps with updates") } + + if tmpBody.ForceUpdate { + ctx := context.Background() + dockercli, err := client.NewEnvClient() + if err == nil { + _, err := dockercli.ImagePull(ctx, "frikky/shuffle:app_sdk", types.ImagePullOptions{}) + if err != nil { + log.Printf("[WARNING] Failed to download apps with the new App SDK: %s", err) + } + } else { + log.Printf("[WARNING] Failed to download apps with the new App SDK because of docker cli: %s", err) + } + } + iterateAppGithubFolders(fs, dir, "", "", tmpBody.ForceUpdate) } else if strings.Contains(tmpBody.URL, "s3") { @@ -6568,17 +6582,6 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin buildLaterList := []buildLaterStruct{} ctx := context.Background() - if forceUpdate { - dockercli, err := client.NewEnvClient() - if err == nil { - _, err := dockercli.ImagePull(ctx, "frikky/shuffle:app_sdk", types.ImagePullOptions{}) - if err != nil { - log.Printf("[WARNING] Failed to download apps with the new App SDK: %s", err) - } - } else { - log.Printf("[WARNING] Failed to download apps with the new App SDK because of docker cli: %s", err) - } - } // It's here to prevent getting them in every iteration for _, file := range dir { diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 3485f90c..45f44ba9 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1,5 +1,6 @@ import React, {useState, useEffect, useLayoutEffect} from 'react'; import { useInterval } from 'react-powerhooks'; +import { useTheme } from '@material-ui/core/styles'; import uuid from "uuid"; import {Link} from 'react-router-dom'; @@ -26,6 +27,7 @@ import { w3cwebsocket as W3CWebSocket } from "websocket"; import { useAlert } from "react-alert"; import { validateJson } from "./Workflows.jsx"; import { GetParsedPaths } from "./Apps.jsx"; +import ConfigureWorkflow from '../components/ConfigureWorkflow.jsx'; const surfaceColor = "#27292D" const inputColor = "#383B40" @@ -87,6 +89,7 @@ const AngularWorkflow = (props) => { const referenceUrl = globalUrl+"/api/v1/hooks/" const alert = useAlert() const borderRadius = 3 + const theme = useTheme(); const [bodyWidth, bodyHeight] = useWindowSize(); const appBarSize = 74 @@ -126,6 +129,7 @@ const AngularWorkflow = (props) => { const [rightSideBarOpen, setRightSideBarOpen] = React.useState(false) const [showSkippedActions, setShowSkippedActions] = React.useState(false) const [lastExecution, setLastExecution] = React.useState("") + const [configureWorkflowModalOpen, setConfigureWorkflowModalOpen] = React.useState(false) const [curpath, setCurpath] = useState(typeof window === 'undefined' || window.location === undefined ? "" : window.location.pathname) // 0 = normal, 1 = just done, 2 = normal @@ -639,7 +643,7 @@ const AngularWorkflow = (props) => { getWorkflowExecution(props.match.params.key, "") } else if (responseJson.status === "FINISHED") { - console.log("STOPPING BECAUSE ITS OVAH!") + //console.log("STOPPING BECAUSE ITS OVAH!") setExecutionRunning(false) stop() getWorkflowExecution(props.match.params.key, "") @@ -1403,7 +1407,7 @@ const AngularWorkflow = (props) => { // might just be confusing cy.nodes().some(function( ele ) { if (ele.id() !== workflow.start && ele.data()["label"] !== undefined) { - alert.success("Changed startnode to "+ele.data()["label"]) + //alert.success("Changed startnode to "+ele.data()["label"]) ele.data("isStartNode", true) workflow.start = ele.id() //throw BreakException @@ -3734,7 +3738,7 @@ const AngularWorkflow = (props) => { return (
-
+
{data.configuration === true ? @@ -3744,15 +3748,15 @@ const AngularWorkflow = (props) => { }}/> : -
+
} -
+
{tmpitem}
- {selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0 ? null : + {/*selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0 ? null :
{ @@ -3781,6 +3785,29 @@ const AngularWorkflow = (props) => {
+ */} + {selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0 && selectedActionParameters[count].required === true && selectedActionParameters[count].unique_toggled !== undefined ? null : +
+ +
{}}> + { + //console.log("CHECKED!: ", selectedActionParameters[count]) + selectedActionParameters[count].unique_toggled = !selectedActionParameters[count].unique_toggled + selectedAction.parameters[count].unique_toggled = selectedActionParameters[count].unique_toggled + setSelectedActionParameters(selectedActionParameters) + setSelectedAction(selectedAction) + setUpdate(Math.random()) + }} + name="requires_unique" + /> +
+ +
}
{datafield} @@ -4015,14 +4042,13 @@ const AngularWorkflow = (props) => { /> {selectedApp.name !== undefined && selectedAction.authentication !== undefined && selectedAction.authentication.length === 0 && requiresAuthentication ?
- Authenticate {selectedApp.name}: - +
@@ -6117,26 +6143,26 @@ const AngularWorkflow = (props) => { const topBarStyle= { position: "fixed", right: 0, - left: leftBarSize, - top: appBarSize, + left: leftBarSize+20, + top: appBarSize+20, + /* minWidth: cytoscapeViewWidths, maxWidth: cytoscapeViewWidths, - marginLeft: 20, - marginBottom: 20, + */ } const TopCytoscapeBar = () => { return (
-
+
-

+

Workflows

-

+

{workflow.name}

@@ -7472,6 +7498,24 @@ const AngularWorkflow = (props) => { ) } + const configureWorkflowModal = configureWorkflowModalOpen && apps.length !== 0 ? + { + setConfigureWorkflowModalOpen(false) + }} + PaperProps={{ + style: { + backgroundColor: surfaceColor, + color: "white", + minWidth: 600, + padding: 15, + }, + }} + > + + + : null // This whole part is redundant. Made it part of Arguments instead. @@ -7502,6 +7546,7 @@ const AngularWorkflow = (props) => { {conditionsModal} {authenticationModal} {codePopoutModal} + {configureWorkflowModal} { const [editingWorkflow, setEditingWorkflow] = React.useState({}) const [executionLoading, setExecutionLoading] = React.useState(false) const [isDropzone, setIsDropzone] = React.useState(false); + const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" const { start, stop } = useInterval({ duration: 5000, @@ -1230,11 +1231,13 @@ const Workflows = (props) => { : null} + {isCloud ? null : + } const WorkflowView = () => { diff --git a/functions/onprem/orborus/Dockerfile b/functions/onprem/orborus/Dockerfile index 3fe760c7..7c178fd8 100644 --- a/functions/onprem/orborus/Dockerfile +++ b/functions/onprem/orborus/Dockerfile @@ -2,7 +2,6 @@ FROM golang:1.16.0-buster as builder RUN mkdir /app WORKDIR /app -RUN go get github.com/docker/docker/api/types github.com/docker/docker/api/types/container github.com/docker/docker/client COPY orborus.go /app/orborus.go RUN go mod init orborus diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 47154cda..4e17a529 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -3,13 +3,14 @@ module orborus go 1.13 require ( + github.com/Microsoft/go-winio v0.4.16 // indirect github.com/containerd/containerd v1.4.3 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/docker v20.10.1+incompatible github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/gogo/protobuf v1.3.1 // indirect - github.com/mackerelio/go-osstat v0.1.0 // indirect + github.com/mackerelio/go-osstat v0.1.0 github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index 84c5d0ec..e8c720de 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -1,5 +1,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk= +github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -41,6 +43,7 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/mackerelio/go-osstat v0.1.0 h1:e57QHeHob8kKJ5FhcXGdzx5O6Ktuc5RHMDIkeqhgkFA= github.com/mackerelio/go-osstat v0.1.0/go.mod h1:1K3NeYLhMHPvzUu+ePYXtoB58wkaRpxZsGClZBJyIFw= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -53,9 +56,11 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -73,8 +78,10 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190410235845-0ad05ae3009d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/functions/onprem/worker/Dockerfile b/functions/onprem/worker/Dockerfile index 3bf1541a..a9df5c2b 100644 --- a/functions/onprem/worker/Dockerfile +++ b/functions/onprem/worker/Dockerfile @@ -1,18 +1,21 @@ FROM golang:1.16.0-buster as builder WORKDIR /app +RUN go get github.com/docker/docker/api/types github.com/docker/docker/api/types/container github.com/docker/docker/client +#RUN go env -w GO111MODULE=auto COPY worker.go /app/worker.go -RUN go env -w GO111MODULE=auto - -RUN go get github.com/docker/docker/api/types -RUN go get github.com/docker/docker/api/types/container -RUN go get github.com/docker/docker/client -RUN go get github.com/gorilla/mux -RUN go get github.com/patrickmn/go-cache +RUN go mod init worker +RUN go get github.com/docker/docker/api/types && \ + go get github.com/docker/docker/api/types/container && \ + go get github.com/docker/docker/client && \ + go get github.com/gorilla/mux && \ + go get github.com/patrickmn/go-cache +RUN go build RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker . +## ALPINE IMAGE FROM alpine:3.12 ENV SHUFFLE_BASE_IMAGE_REGISTRY=docker.io diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh index 3f2a483e..44d134c5 100644 --- a/functions/onprem/worker/build.sh +++ b/functions/onprem/worker/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-worker -VERSION=0.8.62 +VERSION=0.8.63 echo "Running docker build with $NAME:$VERSION" #CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin . diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 3e15c282..0594b142 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -524,7 +524,7 @@ type WorkflowAppActionParameter struct { Description string `json:"description" datastore:"description,noindex" yaml:"description"` ID string `json:"id" datastore:"id" yaml:"id,omitempty"` Name string `json:"name" datastore:"name" yaml:"name"` - Example string `json:"example" datastore:"example" yaml:"example"` + Example string `json:"example" datastore:"example,noindex" yaml:"example"` Value string `json:"value" datastore:"value,noindex" yaml:"value,omitempty"` Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` Options []string `json:"options" datastore:"options" yaml:"options"` @@ -536,6 +536,7 @@ type WorkflowAppActionParameter struct { Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` SkipMulticheck bool `json:"skip_multicheck" datastore:"skip_multicheck" yaml:"skip_multicheck"` ValueReplace []Valuereplace `json:"value_replace" datastore:"value_replace,noindex" yaml:"value_replace,omitempty"` + UniqueToggled bool `json:"unique_toggled" datastore:"unique_toggled" yaml:"unique_toggled"` } type Valuereplace struct { @@ -2335,9 +2336,9 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } else { log.Printf("[WARNING] Actionresult is %s for node %s in %s. Continuing anyway because of workflow configuration.", actionResult.Status, actionResult.Action.ID, workflowExecution.ExecutionId) // Finds ALL childnodes to set them to SKIPPED - childNodes = findChildNodes(*workflowExecution, actionResult.Action.ID) // Remove duplicates //log.Printf("CHILD NODES: %d", len(childNodes)) + childNodes = findChildNodes(*workflowExecution, actionResult.Action.ID) for _, nodeId := range childNodes { if nodeId == actionResult.Action.ID { continue @@ -2490,6 +2491,84 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl log.Printf("[INFO] Setting value (2) of %s in execution %s to %s. New result length: %d", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status, len(workflowExecution.Results)) } + if actionResult.Status == "SKIPPED" { + log.Printf("\n\n[INFO] Handling special case for SKIPPED!\n\n") + childNodes := findChildNodes(*workflowExecution, actionResult.Action.ID) + for _, nodeId := range childNodes { + if nodeId == actionResult.Action.ID { + continue + } + + // 1. Find the action itself + // 2. Create an actionresult + curAction := Action{ID: ""} + for _, action := range workflowExecution.Workflow.Actions { + if action.ID == nodeId { + curAction = action + break + } + } + + if len(curAction.ID) == 0 { + log.Printf("Couldn't find subnode %s", nodeId) + continue + } + + resultExists := false + for _, result := range workflowExecution.Results { + if result.Action.ID == curAction.ID { + resultExists = true + break + } + } + + if !resultExists { + // Check parents are done here. Only add it IF all parents are skipped + skipNodeAdd := false + for _, branch := range workflowExecution.Workflow.Branches { + if branch.DestinationID == nodeId { + // If the branch's source node is NOT in childNodes, it's not a skipped parent + sourceNodeFound := false + for _, item := range childNodes { + if item == branch.SourceID { + sourceNodeFound = true + break + } + } + + if !sourceNodeFound { + log.Printf("[INFO] Not setting node %s to SKIPPED", nodeId) + skipNodeAdd = true + break + } + } + } + + if !skipNodeAdd { + newAction := Action{ + AppName: curAction.AppName, + AppVersion: curAction.AppVersion, + Label: curAction.Label, + Name: curAction.Name, + ID: curAction.ID, + } + + newResult := ActionResult{ + Action: newAction, + ExecutionId: actionResult.ExecutionId, + Authorization: actionResult.Authorization, + Result: "Skipped because of previous node", + StartedAt: 0, + CompletedAt: 0, + Status: "SKIPPED", + } + + workflowExecution.Results = append(workflowExecution.Results, newResult) + } + } + } + } + // FIXME: Have a check for skippednodes and their parents /* for resultIndex, result := range workflowExecution.Results { From b6dcc01e8f511f1c10e85d892db2656d5dadc407 Mon Sep 17 00:00:00 2001 From: frikky Date: Thu, 11 Mar 2021 17:15:05 +0100 Subject: [PATCH 34/57] #293: Added extra configuration to 0.8.63 --- backend/app_sdk/build.sh | 2 +- docker-compose.yml | 4 ++-- functions/onprem/orborus/build.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index ae25a4a3..258a65db 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,6 +1,6 @@ #!/bin/bash NAME=shuffle-app_sdk -VERSION=0.8.62 +VERSION=0.8.63 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION diff --git a/docker-compose.yml b/docker-compose.yml index ff2bf5d4..c8d2d2ab 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,7 +47,7 @@ services: - database orborus: #build: ./functions/onprem/orborus - image: ghcr.io/frikky/shuffle-orborus:0.8.62 + image: ghcr.io/frikky/shuffle-orborus:0.8.63 container_name: shuffle-orborus hostname: shuffle-orborus networks: @@ -56,7 +56,7 @@ services: - /var/run/docker.sock:/var/run/docker.sock environment: - SHUFFLE_APP_SDK_VERSION=0.8.60 - - SHUFFLE_WORKER_VERSION=0.8.62 + - SHUFFLE_WORKER_VERSION=0.8.63 - ORG_ID=${ORG_ID} - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT} diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index 4c0b9450..1555b51f 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-orborus -VERSION=0.8.62 +VERSION=0.8.63 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force From 33ac163ef64ac93de73b928346034438d9091d80 Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 13 Mar 2021 06:47:04 +0100 Subject: [PATCH 35/57] Made saving a workflow fast again --- backend/app_sdk/app_base.py | 12 +++++- backend/go-app/codegen.go | 33 +++++++++++++-- backend/go-app/walkoff.go | 31 +++++++------- docker-compose.yml | 4 +- frontend/src/components/AlertTemplate.js | 1 + frontend/src/views/Admin.jsx | 2 +- frontend/src/views/AppCreator.jsx | 12 +++--- frontend/src/views/Apps.jsx | 53 +++++++++++++++++------- 8 files changed, 104 insertions(+), 44 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index c369533b..c2667fec 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -481,10 +481,17 @@ class AppBase: new_params = self.validate_unique_fields(param_multiplier) print(f"NEW PARAMS: {new_params}") if len(new_params) == 0: - print(f"No ID's to handle for validation") + print("[WARNING] SHOULD STOP MULTI-EXECUTION BECAUSE FIELDS AREN'T UNIQUE") + action_result["status"] = "SKIPPED" + action_result["result"] = f"All values were non-unique" + action_result["completed_at"] = int(time.time()) + self.send_result(action_result, headers, stream_path) + exit() + #return else: #subparams = new_params print(f"NEW PARAMS: {new_params}") + param_multiplier = new_params #print("Returned with newparams of length %d", len(new_params)) #if isinstance(new_params, list) and len(new_params) == 1: @@ -1821,14 +1828,15 @@ class AppBase: if len(itemlist) > curminlength: curminlength = len(itemlist) + except json.decoder.JSONDecodeError as e: print("JSON Error: %s in %s" % (e, actualitem)) replacements[to_be_replaced] = actualitem + #print("In second part of else: %s" % (len(itemlist))) # This is a result array for JUST this value.. # What if there are more? - print("LENGTH: %d. In second part of else: %s" % (len(itemlist), replacements)) resultarray = [] for i in range(0, curminlength): tmpitem = json.loads(json.dumps(parameter["value"])) diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index 4389219f..b0b9b14d 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -1105,13 +1105,16 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [ } } - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ Name: "ssl_verify", Description: "Check if you want to verify request", Multiline: false, Required: false, Example: "True", + Options: []string{ + "True", + "False", + }, Schema: SchemaDefinition{ Type: "string", }, @@ -1244,10 +1247,14 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ Name: "ssl_verify", - Description: "Check if you want to verify the SSL certificate request", + Description: "Check if you want to verify request", Multiline: false, Required: false, - Example: "False - default=True", + Example: "True", + Options: []string{ + "True", + "False", + }, Schema: SchemaDefinition{ Type: "string", }, @@ -1382,6 +1389,10 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo Multiline: false, Required: false, Example: "True", + Options: []string{ + "True", + "False", + }, Schema: SchemaDefinition{ Type: "string", }, @@ -1517,6 +1528,10 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [] Multiline: false, Required: false, Example: "True", + Options: []string{ + "True", + "False", + }, Schema: SchemaDefinition{ Type: "string", }, @@ -1684,6 +1699,10 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo Multiline: false, Required: false, Example: "True", + Options: []string{ + "True", + "False", + }, Schema: SchemaDefinition{ Type: "string", }, @@ -1823,6 +1842,10 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W Multiline: false, Required: false, Example: "True", + Options: []string{ + "True", + "False", + }, Schema: SchemaDefinition{ Type: "string", }, @@ -1958,6 +1981,10 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor Multiline: false, Required: false, Example: "True", + Options: []string{ + "True", + "False", + }, Schema: SchemaDefinition{ Type: "string", }, diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 05990f8b..3d9923b5 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -2224,6 +2224,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { return } + log.Printf("PRE BODY") body, err := ioutil.ReadAll(request.Body) if err != nil { log.Printf("Failed hook unmarshaling: %s", err) @@ -2267,6 +2268,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { allNodes := []string{} workflow.Categories = Categories{} + log.Printf("PRE APPS") workflowapps, apperr := getAllWorkflowApps(ctx, 500) //log.Printf("Action: %#v", action.Authentication) @@ -2299,6 +2301,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { newActions = append(newActions, action) } + log.Printf("PRE SAVECHECK") if !workflow.PreviouslySaved { log.Printf("[WORKFLOW INIT] NOT PREVIOUSLY SAVED - SET ACTION AUTH!") //AuthenticationId string `json:"authentication_id,omitempty" datastore:"authentication_id"` @@ -2434,6 +2437,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { workflow.PreviouslySaved = true } + log.Printf("PRE TRIGGERS") workflow.Actions = newActions newTriggers := []Trigger{} for _, trigger := range workflow.Triggers { @@ -2549,6 +2553,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { workflow.Triggers = newTriggers + log.Printf("PRE VARIABLES") for _, variable := range workflow.WorkflowVariables { if len(variable.Value) == 0 { log.Printf("Can't have an empty variable: %s", variable.Name) @@ -2596,6 +2601,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } // FIXME - append all nodes (actions, triggers etc) to one single array here + log.Printf("PRE VARIABLES") if len(foundNodes) != len(allNodes) || len(workflow.Actions) <= 0 { // This shit takes a few seconds lol if !workflow.IsValid { @@ -2637,18 +2643,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // Have to do it like this to add the user's apps //log.Println("Apps set starting") //log.Printf("EXIT ON ERROR: %#v", workflow.Configuration.ExitOnError) - workflowApps := []WorkflowApp{} - //memcacheName = "all_apps" - //if item, err := memcache.Get(ctx, memcacheName); err == memcache.ErrCacheMiss { - // // Not in cache - // log.Printf("Apps not in cache.") - workflowApps, err = getAllWorkflowApps(ctx, 100) - if err != nil { - log.Printf("Failed getting all workflow apps from database: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } + //workflowapps, apperr := getAllWorkflowApps(ctx, 500) // Started getting the single apps, but if it's weird, this is faster // 1. Check workflow.Start @@ -2680,6 +2675,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } // Check every app action and param to see whether they exist + log.Printf("PRE ACTIONS 2") newActions = []Action{} for _, action := range workflow.Actions { reservedApps := []string{ @@ -2731,7 +2727,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } else { curapp := WorkflowApp{} // FIXME - can this work with ONLY AppID? - for _, app := range workflowApps { + for _, app := range workflowapps { if app.ID == action.AppID { curapp = app break @@ -2860,10 +2856,11 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { Errors: workflow.Errors, } - cacheKey := fmt.Sprintf("workflowapps-sorted-100") - requestCache.Delete(cacheKey) - cacheKey = fmt.Sprintf("workflowapps-sorted-500") - requestCache.Delete(cacheKey) + // Really don't know why this was happening + //cacheKey := fmt.Sprintf("workflowapps-sorted-100") + //requestCache.Delete(cacheKey) + //cacheKey = fmt.Sprintf("workflowapps-sorted-500") + //requestCache.Delete(cacheKey) log.Printf("[INFO] Saved new version of workflow %s (%s) for org %s", workflow.Name, fileId, workflow.OrgId) resp.WriteHeader(200) diff --git a/docker-compose.yml b/docker-compose.yml index c8d2d2ab..b26f3c1a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: frontend: #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.62 + image: ghcr.io/frikky/shuffle-frontend:0.8.63 container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -17,7 +17,7 @@ services: - backend backend: #build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.62 + image: ghcr.io/frikky/shuffle-backend:0.8.63 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: diff --git a/frontend/src/components/AlertTemplate.js b/frontend/src/components/AlertTemplate.js index a476afc7..58ff46cc 100644 --- a/frontend/src/components/AlertTemplate.js +++ b/frontend/src/components/AlertTemplate.js @@ -18,6 +18,7 @@ const alertStyle = { width: 400, boxSizing: 'border-box', zIndex: 100001, + overflow: "hidden", } const buttonStyle = { diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index 7cee4bbe..c9f85d55 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -1958,7 +1958,7 @@ const Admin = (props) => { primary={new Date(file.created_at*1000).toISOString()} /> { } } - console.log(methodvalue["requestBody"]["content"]) + //console.log(methodvalue["requestBody"]["content"]) if (methodvalue["requestBody"]["content"]["multipart/form-data"] !== undefined) { if (methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"] !== undefined && methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"] !== null) { if (methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"]["type"] === "object") { @@ -1796,8 +1796,8 @@ const AppCreator = (props) => { ) })} - {actionBodyRequest.map(data => ( - + {actionBodyRequest.map((data, index) => ( + {data} ))} @@ -1833,8 +1833,8 @@ const AppCreator = (props) => { console.log("URL: ", parsedurl) if (parsedurl.includes("<") && parsedurl.includes(">")) { console.log("REPLACE") - parsedurl = parsedurl.replace("<", "{") - parsedurl = parsedurl.replace(">", "}") + parsedurl = parsedurl.replaceAll("<", "{") + parsedurl = parsedurl.replaceAll(">", "}") } if (parsedurl.startsWith("PUT ") || parsedurl.startsWith("GET ") ||parsedurl.startsWith("POST ") || parsedurl.startsWith("DELETE ") ||parsedurl.startsWith("PATCH ") || parsedurl.startsWith("CONNECT ")) { @@ -2208,8 +2208,10 @@ const AppCreator = (props) => { }}/> const zoomIn = () => { + console.log("ZOOOMING IN") setScale(scale+0.1); } + const zoomOut = () => { setScale(scale-0.1); } diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 04fb0a62..0b4cd611 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -202,14 +202,40 @@ const Apps = (props) => { .then((responseJson) => { //console.log("Apps: ", responseJson) //responseJson = sortByKey(responseJson, "large_image") - responseJson = sortByKey(responseJson, "generated") + //responseJson = sortByKey(responseJson, "is_valid") + //setFilteredApps(responseJson.filter(app => !internalIds.includes(app.name) && !(!app.activated && app.generated))) + + var privateapps = [] + var valid = [] + var invalid = [] + for (var key in responseJson) { + const app = responseJson[key] + if (app.is_valid && !(!app.activated && app.generated)) { + privateapps.push(app) + } else if (app.private_id !== undefined && app.private_id.length > 0) { + valid.push(app) + } else { + invalid.push(app) + } + } - setApps(responseJson) - setFilteredApps(responseJson) - if (responseJson.length > 0) { - setSelectedApp(responseJson[0]) - if (responseJson[0].actions !== null && responseJson[0].actions.length > 0) { - setSelectedAction(responseJson[0].actions[0]) + //console.log(privateapps) + //console.log(valid) + //console.log(invalid) + //console.log(privateapps) + //privateapps.reverse() + privateapps.push(...valid) + privateapps.push(...invalid) + + setApps(privateapps) + setFilteredApps(privateapps) + if (privateapps.length > 0) { + if (selectedApp.id === undefined || selectedApp.id === null) { + setSelectedApp(privateapps[0]) + } + + if (privateapps[0].actions !== null && privateapps[0].actions.length > 0) { + setSelectedAction(privateapps[0].actions[0]) } else { setSelectedAction({}) } @@ -358,8 +384,7 @@ const Apps = (props) => { {imageline} -
-
+
@@ -957,7 +982,7 @@ const Apps = (props) => { setValidation(true) setIsLoading(true) - start() + //start() const parsedData = { "url": url, @@ -989,10 +1014,10 @@ const Apps = (props) => { if (response.status === 200) { alert.success("Loaded existing apps!") } - setIsLoading(false) - stop() - setValidation(false) + //stop() + setIsLoading(false) + setValidation(false) return response.json() }) .then((responseJson) => { @@ -1005,7 +1030,7 @@ const Apps = (props) => { console.log("ERROR: ", error.toString()) alert.error(error.toString()) - stop() + //stop() setIsLoading(false) setValidation(false) }) From 49d9c63d88d42462780615516caa97cb014b2a92 Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 13 Mar 2021 07:36:08 +0100 Subject: [PATCH 36/57] Fixed loop-match regex for apps --- backend/app_sdk/app_base.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index c2667fec..b3cd2f7e 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -1299,6 +1299,7 @@ class AppBase: # Matches with space in the first part, but not in subsequent parts. # JSON / yaml etc shouldn't have spaces in their fields anyway. + #match = ".*?([$]{1}([a-zA-Z0-9 _-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,})[$/, ]?" match = ".*?([$]{1}([a-zA-Z0-9 _-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,})" # Regex to find all the things @@ -1699,7 +1700,9 @@ class AppBase: # Custom format for ${name[0,1,2,...]}$ #submatch = "([${]{2}([0-9a-zA-Z_-]+)(\[.*\])[}$]{2})" - submatch = "([${]{2}#?([0-9a-zA-Z_-]+)#?(\[.*\])[}$]{2})" + print(f"Returnedvalue: {value}") + #submatch = "([${]{2}#?([0-9a-zA-Z_-]+)#?(\[.*\])[}$]{2})" + submatch = "([${]{2}#?([0-9a-zA-Z_-]+)#?(\[.*?]}\$))" actualitem = re.findall(submatch, value, re.MULTILINE) try: if action["skip_multicheck"]: @@ -1806,7 +1809,7 @@ class AppBase: multi_execution_lists.append(new_replacement) #print("MULTI finished: %s" % json_replacement) else: - print("(2) Pre replacement (loop with variables). ") #% actualitem) + print(f"(2) Pre replacement (loop with variables). Variables: {actualitem}") #% actualitem) # This is here to handle for loops within variables.. kindof # 1. Find the length of the longest array # 2. Build an array with the base values based on parameter["value"] @@ -1830,7 +1833,7 @@ class AppBase: curminlength = len(itemlist) except json.decoder.JSONDecodeError as e: - print("JSON Error: %s in %s" % (e, actualitem)) + print("JSON Error (replace): %s in %s" % (e, actualitem)) replacements[to_be_replaced] = actualitem From 4305b8275d9b42cb651869948dd9588908736d16 Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 13 Mar 2021 18:39:33 +0100 Subject: [PATCH 37/57] Started re-adding oauth2 configurations --- backend/app_sdk/app_base.py | 98 ++++- backend/go-app/main.go | 558 +------------------------ backend/go-app/oauth2.go | 468 +++++++++++++++++++++ backend/go-app/walkoff.go | 16 +- frontend/src/views/AngularWorkflow.jsx | 114 ++--- 5 files changed, 633 insertions(+), 621 deletions(-) create mode 100644 backend/go-app/oauth2.go diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index b3cd2f7e..b9eee023 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -28,6 +28,7 @@ class AppBase: self.authorization = os.getenv("AUTHORIZATION", "") self.current_execution_id = os.getenv("EXECUTIONID", "") self.full_execution = os.getenv("FULL_EXECUTION", "") + self.start_time = int(time.time()) self.result_wrapper_count = 0 if isinstance(self.action, str): @@ -482,10 +483,17 @@ class AppBase: print(f"NEW PARAMS: {new_params}") if len(new_params) == 0: print("[WARNING] SHOULD STOP MULTI-EXECUTION BECAUSE FIELDS AREN'T UNIQUE") - action_result["status"] = "SKIPPED" - action_result["result"] = f"All values were non-unique" - action_result["completed_at"] = int(time.time()) - self.send_result(action_result, headers, stream_path) + action_result = { + "action": self.action, + "authorization": self.authorization, + "execution_id": self.current_execution_id, + "result": f"All {len(param_multiplier)} values were non-unique", + "started_at": self.start_time, + "status": "SKIPPED", + "completed_at": int(time.time()), + } + + self.send_result(action_result, {"Content-Type": "application/json", "Authorization": "Bearer %s" % self.authorization}, "/api/v1/streams") exit() #return else: @@ -906,6 +914,41 @@ class AppBase: return data.strip() if "split" in thistype: return data.split() + if "join" in thistype: + print(f"SHOULD JOIN: {data}") + try: + splitvalues = data.split(",") + if "," not in data: + return f"join({data})" + + if len(splitvalues) >= 2: + print(f"SPLITVALUE: {splitvalues[-1]}") + + # 1. Take the list and parse it from string + # 2. Take all the items and join them + # 3. Parse them back as string and return + values = ",".join(splitvalues[0:-1]) + print(f"VALUES: {values}") + tmp = json.loads(values) + print(f"TMP: {tmp}") + #tmp = tmp[1:-1] + #print(f"TMP2: {tmp}") + try: + newvalues = splitvalues[-1].join(str(item).strip() for item in tmp) + except TypeError: + newvalues = splitvalues[-1].join(json.dumps(item).strip() for item in tmp) + + print(f"new: {newvalues}") + return newvalues + else: + print("Returning default") + return f"join({data})" + + except (KeyError, IndexError) as e: + print(f"ERROR in join(): {e}") + except json.decoder.JSONDecodeError as e: + print(f"JSON ERROR in join(): {e}") + if "len" in thistype or "length" in thistype or "lenght" in thistype: tmp = "" try: @@ -919,9 +962,9 @@ class AppBase: pass if isinstance(tmp, list): - return len(tmp) + return str(len(tmp)) elif isinstance(tmp, object): - return len(tmp) + return str(len(tmp)) return str(len(data)) if "parse" in thistype: @@ -967,7 +1010,7 @@ class AppBase: #print("Running %s" % data) # Look for the INNER wrapper first, then move out - wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length", "lenght"] + wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length", "lenght", "join"] found = False for wrapper in wrappers: if wrapper not in data.lower(): @@ -982,8 +1025,8 @@ class AppBase: # Do stuff here. innervalue = parse_nested_param(data, maxDepth(data)-0) outervalue = parse_nested_param(data, maxDepth(data)-1) - #print("INNER: ", innervalue) - #print("OUTER: ", outervalue) + print("INNER: ", innervalue) + print("OUTER: ", outervalue) if outervalue != innervalue: #print("Outer: ", outervalue, " inner: ", innervalue) @@ -1224,10 +1267,10 @@ class AppBase: baseresult = variable["value"] break except KeyError as e: - print("KeyError wf variables: %s" % e) + print("[INFO] KeyError wf variables: %s" % e) pass except TypeError as e: - print("TypeError wf variables: %s" % e) + print("[INFO] TypeError wf variables: %s" % e) pass print("BEFORE EXECUTION VAR") @@ -1240,10 +1283,10 @@ class AppBase: baseresult = variable["value"] break except KeyError as e: - print("KeyError exec variables: %s" % e) + print("[INFO] KeyError exec variables: %s" % e) pass except TypeError as e: - print("TypeError exec variables: %s" % e) + print("[INFO] TypeError exec variables: %s" % e) pass except KeyError as error: @@ -1701,7 +1744,8 @@ class AppBase: # Custom format for ${name[0,1,2,...]}$ #submatch = "([${]{2}([0-9a-zA-Z_-]+)(\[.*\])[}$]{2})" print(f"Returnedvalue: {value}") - #submatch = "([${]{2}#?([0-9a-zA-Z_-]+)#?(\[.*\])[}$]{2})" + # OLD: Used until 13.03.2021: submatch = "([${]{2}#?([0-9a-zA-Z_-]+)#?(\[.*\])[}$]{2})" + # \${[0-9a-zA-Z_-]+#?(\[.*?]}\$) submatch = "([${]{2}#?([0-9a-zA-Z_-]+)#?(\[.*?]}\$))" actualitem = re.findall(submatch, value, re.MULTILINE) try: @@ -1722,18 +1766,24 @@ class AppBase: # Loop WITH variables go in else. print("Before first part in multiexec!") handled = False + + # Has a loop without a variable used inside if len(actualitem[0]) > 2 and actualitem[0][1] == "SHUFFLE_NO_SPLITTER": print("(1) Pre replacement: %s" % actualitem[0][2]) tmpitem = value - replacement = actualitem[0][2] + index = 0 + replacement = actualitem[index][2] + if replacement.endswith("}$"): + replacement = replacement[:-2] + if replacement.startswith("\"") and replacement.endswith("\""): replacement = replacement[1:len(replacement)-1] print("POST replacement: %s" % replacement) - #json_replacement = tmpitem.replace(actualitem[0][0], replacement, 1) + #json_replacement = tmpitem.replace(actualitem[index][0], replacement, 1) #print("AFTER POST replacement: %s" % json_replacement) #json_replacement = replacement try: @@ -1755,9 +1805,9 @@ class AppBase: for i in range(len(json_replacement)): if isinstance(json_replacement[i], dict) or isinstance(json_replacement[i], list): tmp_replacer = json.dumps(json_replacement[i]) - newvalue = tmpitem.replace(actualitem[0][0], tmp_replacer, 1) + newvalue = tmpitem.replace(actualitem[index][0], tmp_replacer, 1) else: - newvalue = tmpitem.replace(actualitem[0][0], json_replacement[i], 1) + newvalue = tmpitem.replace(actualitem[index][0], json_replacement[i], 1) try: newvalue = json.loads(newvalue) @@ -1770,7 +1820,7 @@ class AppBase: print("New replacement: %s" % new_replacement) # New - tmpitem = tmpitem.replace(actualitem[0][0], replacement, 1) + tmpitem = tmpitem.replace(actualitem[index][0], replacement, 1) # This code handles files. resultarray = [] @@ -1821,9 +1871,15 @@ class AppBase: try: to_be_replaced = replace[0] actualitem = replace[2] + if actualitem.endswith("}$"): + actualitem = actualitem[:-2] except IndexError: continue + #print(f"\n\nTMPITEM: {actualitem}\n\n") + #actualitem = parse_wrapper_start(actualitem) + #print(f"\n\nTMPITEM2: {actualitem}\n\n") + try: itemlist = json.loads(actualitem) if len(itemlist) > minlength: @@ -1837,6 +1893,10 @@ class AppBase: replacements[to_be_replaced] = actualitem + + # Parses the data as string with length, split etc. before moving on. + + #print("In second part of else: %s" % (len(itemlist))) # This is a result array for JUST this value.. # What if there are more? diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 57d57e75..7dc85deb 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -42,7 +42,6 @@ import ( */ "github.com/google/go-github/v28/github" - "golang.org/x/oauth2" "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" @@ -671,7 +670,7 @@ func handleApiAuthentication(resp http.ResponseWriter, request *http.Request) (U // Should basically never happen Userdata, err := getUser(ctx, session.Id) if err != nil { - log.Printf("Username %s doesn't exist (authcheck): %s", session.Username, err) + log.Printf("[INFO] Username %s doesn't exist (authcheck): %s", session.Username, err) return User{}, err } @@ -1770,90 +1769,6 @@ type passwordChange struct { Currentpassword string `json:"currentpassword"` } -func handlePasswordResetMail(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - log.Println("Handling password reset mail") - defaultMessage := "We have sent you an email :)" - - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Println("Failed reading body") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, defaultMessage))) - return - } - - type passwordReset struct { - Username string `json:"username"` - } - - var t passwordReset - err = json.Unmarshal(body, &t) - if err != nil { - log.Printf("Failed unmarshaling: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, defaultMessage))) - return - } - - ctx := context.Background() - Userdata, err := getUser(ctx, t.Username) - if err != nil { - log.Printf("Username %s doesn't exist (pw reset mail): %s", t.Username, err) - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) - return - } - - resetToken := uuid.NewV4() - // FIXME: - // Weakness with this system is that you can spam someone with password resets, - // and they would never be able to reset, as a new token is always generated - url := fmt.Sprintf("https://shuffler.io/passwordreset/%s", resetToken.String()) - - Userdata.ResetReference = resetToken.String() - Userdata.ResetTimeout = 0 - err = setUser(ctx, Userdata) - if err != nil { - log.Printf("Error patching User for mail %s: %s", Userdata.Username, err) - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) - return - } - - log.Printf("%#v", Userdata) - addr := t.Username - const confirmMessage = ` -Reset URL :) - -%s - ` - - msg := &mail.Message{ - Sender: "Shuffle ", - To: []string{addr}, - Subject: "Reset your password - Shuffle", - Body: fmt.Sprintf(confirmMessage, url), - } - - log.Println(msg.Body) - if err := mail.Send(ctx, msg); err != nil { - log.Printf("Couldn't send email: %v", err) - } - - // FIXME - // Generate an email to send - // Generate a reset code with a reset link - // Build frontend to handle reset link with "new password" etc. - - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, defaultMessage))) -} - func handlePasswordReset(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -4954,398 +4869,6 @@ func getDocs(resp http.ResponseWriter, request *http.Request) { resp.Write(b) } -type OutlookProfile struct { - OdataContext string `json:"@odata.context"` - BusinessPhones []string `json:"businessPhones"` - DisplayName string `json:"displayName"` - GivenName string `json:"givenName"` - JobTitle interface{} `json:"jobTitle"` - Mail string `json:"mail"` - MobilePhone interface{} `json:"mobilePhone"` - OfficeLocation interface{} `json:"officeLocation"` - PreferredLanguage interface{} `json:"preferredLanguage"` - Surname string `json:"surname"` - UserPrincipalName string `json:"userPrincipalName"` - ID string `json:"id"` -} - -type OutlookFolder struct { - ID string `json:"id"` - DisplayName string `json:"displayName"` - ParentFolderID string `json:"parentFolderId"` - ChildFolderCount int `json:"childFolderCount"` - UnreadItemCount int `json:"unreadItemCount"` - TotalItemCount int `json:"totalItemCount"` -} - -type OutlookFolders struct { - OdataContext string `json:"@odata.context"` - OdataNextLink string `json:"@odata.nextLink"` - Value []OutlookFolder `json:"value"` -} - -func getOutlookFolders(client *http.Client) (OutlookFolders, error) { - requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/users/frikky@shuffletest.onmicrosoft.com/mailfolders") - - ret, err := client.Get(requestUrl) - if err != nil { - log.Printf("FolderErr: %s", err) - return OutlookFolders{}, err - } - - if ret.StatusCode != 200 { - log.Printf("Status folders: %d", ret.StatusCode) - return OutlookFolders{}, err - } - - body, err := ioutil.ReadAll(ret.Body) - if err != nil { - log.Printf("Body: %s", err) - return OutlookFolders{}, err - } - - //log.Printf("Body: %s", string(body)) - - mailfolders := OutlookFolders{} - err = json.Unmarshal(body, &mailfolders) - if err != nil { - log.Printf("Unmarshal: %s", err) - return OutlookFolders{}, err - } - - //fmt.Printf("%#v", mailfolders) - // FIXME - recursion for subfolders - // Recursive struct - // folderEndpoint := fmt.Sprintf("%s/%s/childfolders?$top=40", requestUrl, parentId) - //for _, folder := range mailfolders.Value { - // log.Println(folder.DisplayName) - //} - - return mailfolders, nil -} - -func getOutlookProfile(client *http.Client) (OutlookProfile, error) { - requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/me?$select=mail") - - ret, err := client.Get(requestUrl) - if err != nil { - log.Printf("FolderErr: %s", err) - return OutlookProfile{}, err - } - - log.Printf("Status folders: %d", ret.StatusCode) - body, err := ioutil.ReadAll(ret.Body) - if err != nil { - log.Printf("Body: %s", err) - return OutlookProfile{}, err - } - - profile := OutlookProfile{} - err = json.Unmarshal(body, &profile) - if err != nil { - log.Printf("Unmarshal: %s", err) - return OutlookProfile{}, err - } - - return profile, nil -} - -func handleNewOutlookRegister(resp http.ResponseWriter, request *http.Request) { - code := request.URL.Query().Get("code") - if len(code) == 0 { - log.Println("No code") - resp.WriteHeader(401) - return - } - - url := fmt.Sprintf("http://%s%s", request.Host, request.URL.EscapedPath()) - log.Println(url) - ctx := context.Background() - client, accessToken, err := getOutlookClient(ctx, code, OauthToken{}, url) - if err != nil { - log.Printf("Oauth client failure - outlook register: %s", err) - resp.WriteHeader(401) - return - } - // This should be possible, and will also give the actual username - profile, err := getOutlookProfile(client) - if err != nil { - log.Printf("Outlook profile failure: %s", err) - resp.WriteHeader(401) - return - } - - // This is a state workaround, which should really be for CSRF checks lol - state := request.URL.Query().Get("state") - if len(state) == 0 { - log.Println("No state") - resp.WriteHeader(401) - return - } - - stateitems := strings.Split(state, "%26") - if len(stateitems) == 1 { - stateitems = strings.Split(state, "&") - } - - // FIXME - trigger auth - senderUser := "" - trigger := TriggerAuth{} - for _, item := range stateitems { - itemsplit := strings.Split(item, "%3D") - if len(itemsplit) == 1 { - itemsplit = strings.Split(item, "=") - } - - if len(itemsplit) != 2 { - continue - } - - // Do something here - if itemsplit[0] == "workflow_id" { - trigger.WorkflowId = itemsplit[1] - } else if itemsplit[0] == "trigger_id" { - trigger.Id = itemsplit[1] - } else if itemsplit[0] == "type" { - trigger.Type = itemsplit[1] - } else if itemsplit[0] == "username" { - trigger.Username = itemsplit[1] - trigger.Owner = itemsplit[1] - senderUser = itemsplit[1] - } - } - - // THis is an override based on the user in oauth return - trigger.Username = profile.Mail - trigger.Code = code - trigger.OauthToken = OauthToken{ - AccessToken: accessToken.AccessToken, - TokenType: accessToken.TokenType, - RefreshToken: accessToken.RefreshToken, - Expiry: accessToken.Expiry, - } - - //log.Printf("%#v", trigger) - if trigger.WorkflowId == "" || trigger.Id == "" || trigger.Username == "" || trigger.Type == "" { - log.Printf("All oauth items need to contain data to register a new state") - resp.WriteHeader(401) - return - } - - // Should also update the user - Userdata, err := getUser(ctx, senderUser) - if err != nil { - log.Printf("Username %s doesn't exist (oauth2): %s", trigger.Username, err) - resp.WriteHeader(401) - return - } - - Userdata.Authentication = append(Userdata.Authentication, UserAuth{ - Name: "Outlook", - Description: "oauth2", - Workflows: []string{trigger.WorkflowId}, - Username: trigger.Username, - Fields: []UserAuthField{ - UserAuthField{ - Key: "trigger_id", - Value: trigger.Id, - }, - UserAuthField{ - Key: "username", - Value: trigger.Username, - }, - UserAuthField{ - Key: "code", - Value: code, - }, - UserAuthField{ - Key: "type", - Value: trigger.Type, - }, - }, - }) - - // Set apikey for the user if they don't have one - if len(Userdata.ApiKey) == 0 { - newUser, err := generateApikey(ctx, *Userdata) - Userdata = &newUser - if err != nil { - log.Printf("Failed to generate apikey for user %s when creating outlook sub: %s", Userdata.Username, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": ""}`)) - return - } - } - - //err = setUser(Userdata) - //if err != nil { - // log.Printf("Failed setting user data for %s: %s", Userdata.Username, err) - // resp.WriteHeader(401) - // return - //} - - err = setTriggerAuth(ctx, trigger) - if err != nil { - log.Printf("Failed to set trigger auth for %s - %s", trigger.Username, err) - resp.WriteHeader(401) - return - } - - // FIXME - not sure if these are good at all :) - environmentVariables := map[string]string{ - "FUNCTION_APIKEY": Userdata.ApiKey, - "CALLBACKURL": "https://shuffler.io", - "WORKFLOW_ID": trigger.WorkflowId, - "TRIGGER_ID": trigger.Id, - } - - applocation := fmt.Sprintf("gs://%s/triggers/outlooktrigger.zip", bucketName) - hookname := fmt.Sprintf("outlooktrigger_%s", trigger.Id) - - err = deployCloudFunctionGo(ctx, hookname, defaultLocation, applocation, environmentVariables) - if err != nil { - log.Printf("Error deploying hook: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Issue with starting hook. Please wait a second and try again"}`))) - return - } - - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true}`)) -} - -type OauthToken struct { - AccessToken string `json:"AccessToken" datastore:"AccessToken,noindex"` - TokenType string `json:"TokenType" datastore:"TokenType,noindex"` - RefreshToken string `json:"RefreshToken" datastore:"RefreshToken,noindex"` - Expiry time.Time `json:"Expiry" datastore:"Expiry,noindex"` -} -type TriggerAuth struct { - Id string `json:"id" datastore:"id"` - SubscriptionId string `json:"subscriptionId" datastore:"subscriptionId"` - - Username string `json:"username" datastore:"username,noindex"` - WorkflowId string `json:"workflow_id" datastore:"workflow_id,noindex"` - Owner string `json:"owner" datastore:"owner"` - Type string `json:"type" datastore:"type"` - Code string `json:"code,omitempty" datastore:"code,noindex"` - OauthToken OauthToken `json:"oauth_token,omitempty" datastore:"oauth_token"` -} - -func getTriggerAuth(ctx context.Context, id string) (*TriggerAuth, error) { - key := datastore.NameKey("trigger_auth", strings.ToLower(id), nil) - triggerauth := &TriggerAuth{} - if err := dbclient.Get(ctx, key, triggerauth); err != nil { - return &TriggerAuth{}, err - } - - return triggerauth, nil -} - -func setTriggerAuth(ctx context.Context, trigger TriggerAuth) error { - key1 := datastore.NameKey("trigger_auth", strings.ToLower(trigger.Id), nil) - - // New struct, to not add body, author etc - if _, err := dbclient.Put(ctx, key1, &trigger); err != nil { - log.Printf("Error adding trigger auth: %s", err) - return err - } - - return nil -} - -// THis all of a sudden became really horrible.. fml -func getOutlookClient(ctx context.Context, code string, accessToken OauthToken, redirectUri string) (*http.Client, *oauth2.Token, error) { - - conf := &oauth2.Config{ - ClientID: "", - ClientSecret: "", - Scopes: []string{ - "Mail.Read", - "User.Read", - }, - RedirectURL: redirectUri, - Endpoint: oauth2.Endpoint{ - AuthURL: "https://login.microsoftonline.com/common/oauth2/authorize", - TokenURL: "https://login.microsoftonline.com/common/oauth2/token", - }, - } - - if len(code) > 0 { - access_token, err := conf.Exchange(ctx, code) - if err != nil { - log.Printf("Access_token issue: %s", err) - return &http.Client{}, access_token, err - } - - client := conf.Client(ctx, access_token) - return client, access_token, nil - } else { - // Manually recreate the oauthtoken - access_token := &oauth2.Token{ - AccessToken: accessToken.AccessToken, - TokenType: accessToken.TokenType, - RefreshToken: accessToken.RefreshToken, - Expiry: accessToken.Expiry, - } - - client := conf.Client(ctx, access_token) - return client, access_token, nil - } -} - -func handleGetOutlookFolders(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - // Exchange every time hmm - // FIXME - // Should really just get the code from the trigger that's being used OR the user - triggerId := request.URL.Query().Get("trigger_id") - if len(triggerId) == 0 { - log.Println("No trigger_id supplied") - resp.WriteHeader(401) - return - } - - ctx := context.Background() - trigger, err := getTriggerAuth(ctx, triggerId) - if err != nil { - log.Printf("Trigger %s doesn't exist - outlook folders.", triggerId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Trigger doesn't exist."}`)) - return - } - - // FIXME - should be shuffler in literally every case except testing lol - redirectDomain := "shuffler.io" - url := fmt.Sprintf("https://%s/functions/outlook/register", redirectDomain) - outlookClient, _, err := getOutlookClient(ctx, "", trigger.OauthToken, url) - if err != nil { - log.Printf("Oauth client failure - outlook folders: %s", err) - resp.WriteHeader(401) - return - } - - folders, err := getOutlookFolders(outlookClient) - if err != nil { - resp.WriteHeader(401) - return - } - - b, err := json.Marshal(folders.Value) - if err != nil { - log.Println("Failed to marshal folderdata") - resp.WriteHeader(401) - return - } - - resp.WriteHeader(200) - resp.Write(b) -} - func handleGetSpecificStats(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -5395,66 +4918,6 @@ func handleGetSpecificStats(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(b)) } -func handleGetSpecificTrigger(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in getting specific workflow: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - location := strings.Split(request.URL.String(), "/") - - var workflowId string - if location[1] == "api" { - if len(location) <= 4 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - workflowId = location[4] - } - - if strings.Contains(workflowId, "?") { - workflowId = strings.Split(workflowId, "?")[0] - } - - ctx := context.Background() - trigger, err := getTriggerAuth(ctx, workflowId) - if err != nil { - log.Printf("Trigger %s doesn't exist - specific trigger.", workflowId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": ""}`)) - return - } - - if user.Username != trigger.Owner && user.Role != "admin" { - log.Printf("Wrong user (%s) for trigger %s", user.Username, trigger.Id) - resp.WriteHeader(401) - return - } - - trigger.OauthToken = OauthToken{} - trigger.Code = "" - - b, err := json.Marshal(trigger) - if err != nil { - log.Println("Failed to marshal data") - resp.WriteHeader(401) - return - } - - resp.WriteHeader(200) - resp.Write(b) -} - func handleDeleteOutlookSub(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -7545,7 +7008,7 @@ func runInit(ctx context.Context) { if err != nil { log.Printf("Failed loading repo %s into memory: %s", apis, err) } else { - log.Printf("Finished git clone. Looking for updates to the repo.") + log.Printf("[INFO] Finished git clone. Looking for updates to the repo.") dir, err := fs.ReadDir("") if err != nil { log.Printf("Failed reading folder: %s", err) @@ -7856,7 +7319,7 @@ func handleKeyValueCheck(resp http.ResponseWriter, request *http.Request) { continue } - log.Printf("[INFO] Looking for value %s", value) + log.Printf("[INFO] Looking for value %s in Workflow %s of ORG %s", value, workflowExecution.Workflow.ID, org.Id) q := datastore.NewQuery(dbKey).Filter("org_id =", org.Id).Filter("workflow_id =", workflowExecution.Workflow.ID).Filter("parameter_name =", parameterNames).Filter("value =", value) foundCount, err := dbclient.Count(ctx, q) @@ -7875,7 +7338,7 @@ func handleKeyValueCheck(resp http.ResponseWriter, request *http.Request) { } } } else { - log.Printf("[INFO] Should validate if value %s in app %s exists WITH ORG %s", workflowExecution.Workflow.ID) + //log.Printf("[INFO] Should validate if value %s in app %s exists WITH ORG %s", workflowExecution.Workflow.ID) for _, value := range value.ParameterValues { if len(value) == 0 { @@ -7883,7 +7346,7 @@ func handleKeyValueCheck(resp http.ResponseWriter, request *http.Request) { continue } - log.Printf("[INFO] Looking for value %s", value) + log.Printf("[INFO] Looking for value %s in ORG %s", value, org.Id) q := datastore.NewQuery(dbKey).Filter("org_id =", org.Id).Filter("workflow_id =", "").Filter("parameter_name =", parameterNames).Filter("value =", value) foundCount, err := dbclient.Count(ctx, q) @@ -8653,11 +8116,6 @@ func initHandlers() { r.HandleFunc("/api/v1/hooks/{key}", handleWebhookCallback).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/hooks/{key}/delete", handleDeleteHook).Methods("DELETE", "OPTIONS") - // Trigger hmm - //r.HandleFunc("/api/v1/triggers/{key}", handleGetSpecificTrigger).Methods("GET", "OPTIONS") - - r.HandleFunc("/api/v1/stats/{key}", handleGetSpecificStats).Methods("GET", "OPTIONS") - // OpenAPI configuration r.HandleFunc("/api/v1/verify_swagger", verifySwagger).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/verify_openapi", verifySwagger).Methods("POST", "OPTIONS") @@ -8693,6 +8151,12 @@ func initHandlers() { r.HandleFunc("/api/v1/files/{fileId}", handleDeleteFile).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/files", handleGetFiles).Methods("GET", "OPTIONS") + // Trigger hmm + r.HandleFunc("/api/v1/triggers/outlook/register", handleNewOutlookRegister).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/triggers/outlook/getFolders", handleGetOutlookFolders).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/triggers/outlook/{key}", handleGetSpecificTrigger).Methods("GET", "OPTIONS") + //r.HandleFunc("/api/v1/stats/{key}", handleGetSpecificStats).Methods("GET", "OPTIONS") + http.Handle("/", r) } diff --git a/backend/go-app/oauth2.go b/backend/go-app/oauth2.go new file mode 100644 index 00000000..5f234e68 --- /dev/null +++ b/backend/go-app/oauth2.go @@ -0,0 +1,468 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "log" + "net/http" + "strings" + "time" + + "cloud.google.com/go/datastore" + "golang.org/x/oauth2" +) + +type OutlookProfile struct { + OdataContext string `json:"@odata.context"` + BusinessPhones []string `json:"businessPhones"` + DisplayName string `json:"displayName"` + GivenName string `json:"givenName"` + JobTitle interface{} `json:"jobTitle"` + Mail string `json:"mail"` + MobilePhone interface{} `json:"mobilePhone"` + OfficeLocation interface{} `json:"officeLocation"` + PreferredLanguage interface{} `json:"preferredLanguage"` + Surname string `json:"surname"` + UserPrincipalName string `json:"userPrincipalName"` + ID string `json:"id"` +} + +type OutlookFolder struct { + ID string `json:"id"` + DisplayName string `json:"displayName"` + ParentFolderID string `json:"parentFolderId"` + ChildFolderCount int `json:"childFolderCount"` + UnreadItemCount int `json:"unreadItemCount"` + TotalItemCount int `json:"totalItemCount"` +} + +type OutlookFolders struct { + OdataContext string `json:"@odata.context"` + OdataNextLink string `json:"@odata.nextLink"` + Value []OutlookFolder `json:"value"` +} + +func getOutlookFolders(client *http.Client) (OutlookFolders, error) { + requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/users/ec03b4f2-fccf-4c35-b0eb-be85a0f5dd43/mailFolders") + + ret, err := client.Get(requestUrl) + if err != nil { + log.Printf("[INFO] FolderErr: %s", err) + return OutlookFolders{}, err + } + + body, err := ioutil.ReadAll(ret.Body) + if err != nil { + log.Printf("[WARNING] Failed body decoding from mailfolders") + return OutlookFolders{}, err + } + + log.Printf("[INFO] Folder Body: %s", string(body)) + log.Printf("[INFO] Status folders: %d", ret.StatusCode) + if ret.StatusCode != 200 { + return OutlookFolders{}, err + } + + //log.Printf("Body: %s", string(body)) + + mailfolders := OutlookFolders{} + err = json.Unmarshal(body, &mailfolders) + if err != nil { + log.Printf("Unmarshal: %s", err) + return OutlookFolders{}, err + } + + //fmt.Printf("%#v", mailfolders) + // FIXME - recursion for subfolders + // Recursive struct + // folderEndpoint := fmt.Sprintf("%s/%s/childfolders?$top=40", requestUrl, parentId) + //for _, folder := range mailfolders.Value { + // log.Println(folder.DisplayName) + //} + + return mailfolders, nil +} + +func getOutlookProfile(client *http.Client) (OutlookProfile, error) { + requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/me?$select=mail") + + ret, err := client.Get(requestUrl) + if err != nil { + log.Printf("[INFO] Folder error: %s", err) + return OutlookProfile{}, err + } + + log.Printf("[INFO] Status profile: %d", ret.StatusCode) + body, err := ioutil.ReadAll(ret.Body) + if err != nil { + log.Printf("[INFO] Body: %s", err) + return OutlookProfile{}, err + } + + log.Printf("[INFO] BODY: %s", string(body)) + + profile := OutlookProfile{} + err = json.Unmarshal(body, &profile) + if err != nil { + log.Printf("Unmarshal: %s", err) + return OutlookProfile{}, err + } + + return profile, nil +} + +func handleNewOutlookRegister(resp http.ResponseWriter, request *http.Request) { + code := request.URL.Query().Get("code") + if len(code) == 0 { + log.Println("No code") + resp.WriteHeader(401) + return + } + + url := fmt.Sprintf("http://%s%s", request.Host, request.URL.EscapedPath()) + log.Println(url) + ctx := context.Background() + client, accessToken, err := getOutlookClient(ctx, code, OauthToken{}, url) + if err != nil { + log.Printf("Oauth client failure - outlook register: %s", err) + resp.WriteHeader(401) + return + } + + // This should be possible, and will also give the actual username + profile, err := getOutlookProfile(client) + if err != nil { + log.Printf("Outlook profile failure: %s", err) + resp.WriteHeader(401) + return + } + + // This is a state workaround, which should really be for CSRF checks lol + state := request.URL.Query().Get("state") + if len(state) == 0 { + log.Println("No state") + resp.WriteHeader(401) + return + } + + stateitems := strings.Split(state, "%26") + if len(stateitems) == 1 { + stateitems = strings.Split(state, "&") + } + + // FIXME - trigger auth + senderUser := "" + trigger := TriggerAuth{} + for _, item := range stateitems { + itemsplit := strings.Split(item, "%3D") + if len(itemsplit) == 1 { + itemsplit = strings.Split(item, "=") + } + + if len(itemsplit) != 2 { + continue + } + + // Do something here + if itemsplit[0] == "workflow_id" { + trigger.WorkflowId = itemsplit[1] + } else if itemsplit[0] == "trigger_id" { + trigger.Id = itemsplit[1] + } else if itemsplit[0] == "type" { + trigger.Type = itemsplit[1] + } else if itemsplit[0] == "username" { + trigger.Username = itemsplit[1] + trigger.Owner = itemsplit[1] + senderUser = itemsplit[1] + } + } + + // THis is an override based on the user in oauth return + trigger.Username = profile.Mail + trigger.Code = code + trigger.OauthToken = OauthToken{ + AccessToken: accessToken.AccessToken, + TokenType: accessToken.TokenType, + RefreshToken: accessToken.RefreshToken, + Expiry: accessToken.Expiry, + } + + //log.Printf("%#v", trigger) + log.Println(trigger.WorkflowId) + log.Println(trigger.Id) + log.Println(trigger.Username) + log.Println(trigger.Type) + if trigger.WorkflowId == "" || trigger.Id == "" || trigger.Username == "" || trigger.Type == "" { + log.Printf("[INFO] All oauth items need to contain data to register a new state") + resp.WriteHeader(401) + return + } + + // Should also update the user + log.Printf("[INFO] Attempting to set up outlook trigger for %s", senderUser) + Userdata, err := getUser(ctx, senderUser) + if err != nil { + log.Printf("[INFO] Username %s doesn't exist (oauth2): %s", trigger.Username, err) + resp.WriteHeader(401) + return + } + + Userdata.Authentication = append(Userdata.Authentication, UserAuth{ + Name: "Outlook", + Description: "oauth2", + Workflows: []string{trigger.WorkflowId}, + Username: trigger.Username, + Fields: []UserAuthField{ + UserAuthField{ + Key: "trigger_id", + Value: trigger.Id, + }, + UserAuthField{ + Key: "username", + Value: trigger.Username, + }, + UserAuthField{ + Key: "code", + Value: code, + }, + UserAuthField{ + Key: "type", + Value: trigger.Type, + }, + }, + }) + + // Set apikey for the user if they don't have one + err = setUser(ctx, Userdata) + if err != nil { + log.Printf("Failed setting user data for %s: %s", Userdata.Username, err) + resp.WriteHeader(401) + return + } + + err = setTriggerAuth(ctx, trigger) + if err != nil { + log.Printf("Failed to set trigger auth for %s - %s", trigger.Username, err) + resp.WriteHeader(401) + return + } + + resp.WriteHeader(200) + resp.Write([]byte(`{"success": true}`)) +} + +type OauthToken struct { + AccessToken string `json:"AccessToken" datastore:"AccessToken,noindex"` + TokenType string `json:"TokenType" datastore:"TokenType,noindex"` + RefreshToken string `json:"RefreshToken" datastore:"RefreshToken,noindex"` + Expiry time.Time `json:"Expiry" datastore:"Expiry,noindex"` +} +type TriggerAuth struct { + Id string `json:"id" datastore:"id"` + SubscriptionId string `json:"subscriptionId" datastore:"subscriptionId"` + + Username string `json:"username" datastore:"username,noindex"` + WorkflowId string `json:"workflow_id" datastore:"workflow_id,noindex"` + Owner string `json:"owner" datastore:"owner"` + Type string `json:"type" datastore:"type"` + Code string `json:"code,omitempty" datastore:"code,noindex"` + OauthToken OauthToken `json:"oauth_token,omitempty" datastore:"oauth_token"` +} + +func getTriggerAuth(ctx context.Context, id string) (*TriggerAuth, error) { + key := datastore.NameKey("trigger_auth", strings.ToLower(id), nil) + triggerauth := &TriggerAuth{} + if err := dbclient.Get(ctx, key, triggerauth); err != nil { + return &TriggerAuth{}, err + } + + return triggerauth, nil +} + +func setTriggerAuth(ctx context.Context, trigger TriggerAuth) error { + key1 := datastore.NameKey("trigger_auth", strings.ToLower(trigger.Id), nil) + + // New struct, to not add body, author etc + if _, err := dbclient.Put(ctx, key1, &trigger); err != nil { + log.Printf("Error adding trigger auth: %s", err) + return err + } + + return nil +} + +// THis all of a sudden became really horrible.. fml +func getOutlookClient(ctx context.Context, code string, accessToken OauthToken, redirectUri string) (*http.Client, *oauth2.Token, error) { + + conf := &oauth2.Config{ + ClientID: "fd55c175-aa30-4fa6-b303-09a29fb3f750", + ClientSecret: "14OBKgUpov.D7fe0~hp0z-cIQdP~SlYm.8", + Scopes: []string{ + "Mail.Read", + }, + RedirectURL: redirectUri, + Endpoint: oauth2.Endpoint{ + AuthURL: "https://login.microsoftonline.com/common/oauth2/authorize", + TokenURL: "https://login.microsoftonline.com/common/oauth2/token", + }, + } + + if len(code) > 0 { + access_token, err := conf.Exchange(ctx, code) + if err != nil { + log.Printf("Access_token issue: %s", err) + return &http.Client{}, access_token, err + } + + client := conf.Client(ctx, access_token) + return client, access_token, nil + } + + // Manually recreate the oauthtoken + access_token := &oauth2.Token{ + AccessToken: accessToken.AccessToken, + TokenType: accessToken.TokenType, + RefreshToken: accessToken.RefreshToken, + Expiry: accessToken.Expiry, + } + + client := conf.Client(ctx, access_token) + return client, access_token, nil +} + +func handleGetOutlookFolders(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + // Exchange every time hmm + // FIXME + // Should really just get the code from the trigger that's being used OR the user + triggerId := request.URL.Query().Get("trigger_id") + if len(triggerId) == 0 { + log.Println("No trigger_id supplied") + resp.WriteHeader(401) + return + } + + ctx := context.Background() + trigger, err := getTriggerAuth(ctx, triggerId) + if err != nil { + log.Printf("[INFO] Trigger %s doesn't exist - outlook folders.", triggerId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Trigger doesn't exist."}`)) + return + } + + //client, accessToken, err := getOutlookClient(ctx, code, OauthToken{}, url) + //if err != nil { + // log.Printf("Oauth client failure - outlook register: %s", err) + // resp.WriteHeader(401) + // return + //} + + // FIXME - should be shuffler in literally every case except testing lol + //log.Printf("TRIGGER: %#v", trigger) + redirectDomain := "localhost:5001" + url := fmt.Sprintf("http://%s/api/v1/triggers/outlook/register", redirectDomain) + outlookClient, _, err := getOutlookClient(ctx, "", trigger.OauthToken, url) + if err != nil { + log.Printf("[WARNING] Oauth client failure - outlook folders: %s", err) + resp.Write([]byte(`{"success": false, "reason": "Failed creating outlook client"}`)) + resp.WriteHeader(401) + return + } + + // This should be possible, and will also give the actual username + /* + profile, err := getOutlookProfile(outlookClient) + if err != nil { + log.Printf("Outlook profile failure: %s", err) + resp.WriteHeader(401) + return + } + log.Printf("PROFILE: %#v", profile) + */ + + folders, err := getOutlookFolders(outlookClient) + if err != nil { + log.Printf("[WARNING] Failed setting outlook folders: %s", err) + resp.Write([]byte(`{"success": false, "reason": "Failed getting outlook folders"}`)) + resp.WriteHeader(401) + return + } + + b, err := json.Marshal(folders.Value) + if err != nil { + log.Println("[INFO] Failed to marshal folderdata") + resp.Write([]byte(`{"success": false, "reason": "Failed decoding JSON"}`)) + resp.WriteHeader(401) + return + } + + resp.WriteHeader(200) + resp.Write(b) +} + +func handleGetSpecificTrigger(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + user, err := handleApiAuthentication(resp, request) + if err != nil { + log.Printf("Api authentication failed in getting specific workflow: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + location := strings.Split(request.URL.String(), "/") + + var workflowId string + if location[1] == "api" { + if len(location) <= 4 { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + workflowId = location[5] + } + + if strings.Contains(workflowId, "?") { + workflowId = strings.Split(workflowId, "?")[0] + } + + ctx := context.Background() + trigger, err := getTriggerAuth(ctx, workflowId) + if err != nil { + log.Printf("[INFO] Trigger %s doesn't exist - specific trigger.", workflowId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": ""}`)) + return + } + + if user.Username != trigger.Owner && user.Role != "admin" { + log.Printf("Wrong user (%s) for trigger %s", user.Username, trigger.Id) + resp.WriteHeader(401) + return + } + + trigger.OauthToken = OauthToken{} + trigger.Code = "" + + b, err := json.Marshal(trigger) + if err != nil { + log.Println("Failed to marshal data") + resp.WriteHeader(401) + return + } + + resp.WriteHeader(200) + resp.Write(b) +} diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 3d9923b5..ac3265de 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -982,7 +982,7 @@ func validateNewWorkerExecution(body []byte) error { } if execution.Status == "EXECUTING" { - log.Printf("[INFO] Inside executing.") + //log.Printf("[INFO] Inside executing.") extra := 0 for _, trigger := range execution.Workflow.Triggers { //log.Printf("Appname trigger (0): %s", trigger.AppName) @@ -3992,7 +3992,7 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() workflow, err := getWorkflow(ctx, fileId) if err != nil { - log.Printf("Failed getting the workflow locally (stop schedule): %s", err) + log.Printf("[WARNING] Failed getting the workflow locally (stop schedule): %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -4001,7 +4001,7 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { // FIXME - have a check for org etc too.. // FIXME - admin check like this? idk if user.Id != workflow.Owner && user.Role != "admin" && user.Role != "scheduler" { - log.Printf("Wrong user (%s) for workflow %s (stop schedule)", user.Username, workflow.ID) + log.Printf("[WARNING] Wrong user (%s) for workflow %s (stop schedule)", user.Username, workflow.ID) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -4009,7 +4009,7 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { schedule, err := getSchedule(ctx, scheduleId) if err != nil { - log.Printf("Failed finding schedule %s", scheduleId) + log.Printf("[WARNING] Failed finding schedule %s", scheduleId) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -4039,15 +4039,15 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { err = executeCloudAction(action, org.SyncConfig.Apikey) if err != nil { - log.Printf("Failed cloud action STOP schedule: %s", err) + log.Printf("[WARNING] Failed cloud action STOP schedule: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } else { - log.Printf("Successfully ran cloud action STOP schedule") + log.Printf("[INFO] Successfully ran cloud action STOP schedule") err = DeleteKey(ctx, "schedules", scheduleId) if err != nil { - log.Printf("Failed deleting cloud schedule onprem..") + log.Printf("[WARNING] Failed deleting cloud schedule onprem..") resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed deleting cloud schedule"}`))) return @@ -4061,7 +4061,7 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { err = deleteSchedule(ctx, scheduleId) if err != nil { - log.Printf("Failed deleting schedule: %s", err) + log.Printf("[WARNING] Failed deleting schedule: %s", err) if strings.Contains(err.Error(), "Job not found") { resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 45f44ba9..bde4fff9 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -2307,10 +2307,12 @@ const AngularWorkflow = (props) => { return } + /* if (data.is_valid === false) { alert.error(data.name+" is only available on https://shuffler.io so far") return } + */ const triggerLabel = getNextActionName(data.name) @@ -4929,7 +4931,7 @@ const AngularWorkflow = (props) => { } const setFolders = () => { - fetch(globalUrl+"/functions/outlook/getFolders?trigger_id="+selectedTrigger.id, { + fetch(globalUrl+"/api/v1/triggers/outlook/getFolders?trigger_id="+selectedTrigger.id, { method: "GET", headers: {"content-type": "application/json"}, credentials: "include", @@ -4942,7 +4944,10 @@ const AngularWorkflow = (props) => { return response.json() }) .then((responseJson) => { - setTriggerFolders(responseJson) + if (responseJson !== null && responseJson.success !== false) { + setTriggerFolders(responseJson) + } + if (workflow.triggers[selectedTriggerIndex].parameters.length === 0 && responseJson.length > 0) { workflow.triggers[selectedTriggerIndex].parameters = [{"value": responseJson[0].displayName, "name": "outlookfolder", "id": responseJson[0].id}] selectedTrigger.parameters = [{"value": responseJson[0].displayName, "name": "outlookfolder", "id": responseJson[0].id}] @@ -4956,7 +4961,7 @@ const AngularWorkflow = (props) => { } const getTriggerAuth = () => { - fetch(globalUrl+"/api/v1/triggers/"+selectedTrigger.id, { + fetch(globalUrl+"/api/v1/triggers/outlook/"+selectedTrigger.id, { method: "GET", headers: {"content-type": "application/json"}, credentials: "include", @@ -4987,15 +4992,22 @@ const AngularWorkflow = (props) => { const outlookButton =
{outlookButton} -
-
-
- Folders: (hold CTRL to select multiple) -
-
- } - key={selectedTrigger} - > - {triggerFolders.map(folder => { - var folderItem = - if (folder.childFolderCount > 0) { - // Here to handle subfolders sometime later - folderItem = - - } + {triggerFolders === undefined || triggerFolders === null ? null : + +
+
+
+ Folders: (hold CTRL to select multiple) +
+
+ } + key={selectedTrigger} + > + {triggerFolders.map(folder => { + var folderItem = + if (folder.childFolderCount > 0) { + // Here to handle subfolders sometime later + folderItem = + + } - return folderItem - })} - + return folderItem + })} + + + }
} else if (triggerAuthentication.type === "gmail") { triggerInfo = "SPECIAL GMAIL" @@ -5578,10 +5594,10 @@ const AngularWorkflow = (props) => { if (trigger.id === undefined) { return } - alert.info("Stopping trigger") + alert.info("Deleting mail trigger") fetch(globalUrl+"/api/v1/workflows/"+props.match.params.key+"/outlook/"+trigger.id, { - method: 'DELETE', + method: 'DELETE', headers: { 'Content-Type': 'application/json', 'Accept': 'application/json', @@ -5616,6 +5632,10 @@ const AngularWorkflow = (props) => { const startMailSub = (trigger, triggerindex) => { var folders = [] + if (triggerFolders === null || triggerFolders === undefined) { + return null + } + const splitItem = workflow.triggers[selectedTriggerIndex].parameters[0].value.split(splitter) for (var key in splitItem) { const item = splitItem[key] From 9d22a20489c4ce3344fd66a300cc2317897c7016 Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 13 Mar 2021 18:49:39 +0100 Subject: [PATCH 38/57] Minor updates --- backend/go-app/main.go | 268 ------------------------------------ backend/go-app/oauth2.go | 283 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 278 insertions(+), 273 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 7dc85deb..29e1432e 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5057,274 +5057,6 @@ func handleOutlookSubRemoval(ctx context.Context, workflowId, triggerId string) return nil } -// This sets up the sub with outlook itself -// Parses data from the workflow to see whether access is right to subscribe it -// Creates the cloud function for outlook return -// Wait for it to be available, then schedule a workflow to it -func createOutlookSub(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - location := strings.Split(request.URL.String(), "/") - - var workflowId string - if location[1] == "api" { - if len(location) <= 4 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - workflowId = location[4] - } - - ctx := context.Background() - workflow, err := getWorkflow(ctx, workflowId) - if err != nil { - log.Printf("Failed getting the workflow locally (outlook sub): %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in outlook deploy: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - // FIXME - have a check for org etc too.. - if user.Id != workflow.Owner && user.Role != "admin" { - log.Printf("Wrong user (%s) for workflow %s when deploying outlook", user.Username, workflow.ID) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - log.Println("Handle outlook subscription for trigger") - - // Should already be authorized at this point, as the workflow is shared - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Printf("Failed body read for workflow %s", workflow.ID) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - log.Println(string(body)) - - // Based on the input data from frontend - type CurTrigger struct { - Name string `json:"name"` - Folders []string `json:"folders"` - ID string `json:"id"` - } - - var curTrigger CurTrigger - err = json.Unmarshal(body, &curTrigger) - if err != nil { - log.Printf("Failed body read unmarshal for trigger %s", workflow.ID) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if len(curTrigger.Folders) == 0 { - log.Printf("Error for %s. Choosing folders is required, currently 0", workflow.ID) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - // Now that it's deployed - wait a few seconds before generating: - // 1. Oauth2 token thingies for outlook.office.com - // 2. Set the url to have the right mailboxes (probably ID?) ("https://outlook.office.com/api/v2.0/me/mailfolders('inbox')/messages") - // 3. Set the callback URL to be the new trigger - // 4. Run subscription test - // 5. Set the subscriptionId to the trigger object - - // First - lets regenerate an oauth token for outlook.office.com from the original items - trigger, err := getTriggerAuth(ctx, curTrigger.ID) - if err != nil { - log.Printf("Trigger %s doesn't exist - outlook sub.", curTrigger.ID) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": ""}`)) - return - } - - // url doesn't really matter here - url := fmt.Sprintf("https://shuffler.io") - outlookClient, _, err := getOutlookClient(ctx, "", trigger.OauthToken, url) - if err != nil { - log.Printf("Oauth client failure - triggerauth: %s", err) - resp.WriteHeader(401) - return - } - - // Location + - notificationURL := fmt.Sprintf("https://%s-%s.cloudfunctions.net/outlooktrigger_%s", defaultLocation, gceProject, curTrigger.ID) - log.Println(notificationURL) - - // This is here simply to let the function start - // Usually takes 10 attempts minimum :O - // 10 * 5 = 50 seconds. That's waaay too much :( - //notificationURL = "https://europe-west1-shuffler.cloudfunctions.net/outlooktrigger_e2ce43b0-997e-4980-9617-6eadbc68cf88" - //notificationURL = "https://de4fc12b.ngrok.io" - - curSubscriptions, err := getOutlookSubscriptions(outlookClient) - if err == nil { - for _, sub := range curSubscriptions.Value { - if sub.NotificationURL == notificationURL { - log.Printf("Removing existing subscription %s", sub.Id) - removeOutlookSubscription(outlookClient, sub.Id) - } - } - } else { - log.Printf("Failed to get subscriptions - need to overwrite") - } - - maxFails := 15 - failCnt := 0 - log.Println(curTrigger.Folders) - for { - subId, err := makeOutlookSubscription(outlookClient, curTrigger.Folders, notificationURL) - if err != nil { - failCnt += 1 - log.Printf("Failed making oauth subscription, retrying in 5 seconds: %s", err) - time.Sleep(5 * time.Second) - if failCnt == maxFails { - log.Printf("Failed to set up subscription %d times.", maxFails) - resp.WriteHeader(401) - return - } - - continue - } - - // Set the ID somewhere here - trigger.SubscriptionId = subId - err = setTriggerAuth(ctx, *trigger) - if err != nil { - log.Printf("Failed setting triggerauth: %s", err) - } - - break - } - - log.Printf("Successfully handled outlook subscription for trigger %s in workflow %s", curTrigger.ID, workflow.ID) - - //log.Printf("%#v", user) - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true}`)) -} - -// Lists the users current subscriptions -func getOutlookSubscriptions(outlookClient *http.Client) (SubscriptionsWrapper, error) { - fullUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/subscriptions") - req, err := http.NewRequest( - "GET", - fullUrl, - nil, - ) - req.Header.Add("Content-Type", "application/json") - res, err := outlookClient.Do(req) - if err != nil { - log.Printf("suberror Client: %s", err) - return SubscriptionsWrapper{}, err - } - - body, err := ioutil.ReadAll(res.Body) - if err != nil { - log.Printf("Suberror Body: %s", err) - return SubscriptionsWrapper{}, err - } - - newSubs := SubscriptionsWrapper{} - err = json.Unmarshal(body, &newSubs) - if err != nil { - return SubscriptionsWrapper{}, err - } - - return newSubs, nil -} - -type SubscriptionsWrapper struct { - OdataContext string `json:"@odata.context"` - Value []Subscription `json:"value"` -} - -type Subscription struct { - ChangeType string `json:"changeType"` - NotificationURL string `json:"notificationUrl"` - Resource string `json:"resource"` - ExpirationDateTime string `json:"expirationDateTime"` - ClientState string `json:"clientState"` - Id string `json:"id"` -} - -func makeOutlookSubscription(client *http.Client, folderIds []string, notificationURL string) (string, error) { - fullUrl := "https://graph.microsoft.com/v1.0/subscriptions" - - // FIXME - this expires rofl - t := time.Now().Local().Add(time.Minute * time.Duration(4300)) - timeFormat := fmt.Sprintf("%d-%02d-%02dT%02d:%02d:%02d.0000000Z", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second()) - log.Println(timeFormat) - - resource := fmt.Sprintf("me/mailfolders('%s')/messages", strings.Join(folderIds, "','")) - log.Println(resource) - sub := Subscription{ - ChangeType: "created", - NotificationURL: notificationURL, - ExpirationDateTime: timeFormat, - ClientState: "This is a test", - Resource: resource, - } - - data, err := json.Marshal(sub) - if err != nil { - log.Printf("Marshal: %s", err) - return "", err - } - - req, err := http.NewRequest( - "POST", - fullUrl, - bytes.NewBuffer(data), - ) - req.Header.Add("Content-Type", "application/json") - - res, err := client.Do(req) - if err != nil { - log.Printf("Client: %s", err) - return "", err - } - - log.Printf("Status: %d", res.StatusCode) - body, err := ioutil.ReadAll(res.Body) - if err != nil { - log.Printf("Body: %s", err) - return "", err - } - - if res.StatusCode != 200 && res.StatusCode != 201 { - return "", errors.New(fmt.Sprintf("Subscription failed: %s", string(body))) - } - - // Use data from body here to create thingy - newSub := Subscription{} - err = json.Unmarshal(body, &newSub) - if err != nil { - return "", err - } - - return newSub.Id, nil -} - func getOpenapi(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { diff --git a/backend/go-app/oauth2.go b/backend/go-app/oauth2.go index 5f234e68..598c7b27 100644 --- a/backend/go-app/oauth2.go +++ b/backend/go-app/oauth2.go @@ -1,8 +1,10 @@ package main import ( + "bytes" "context" "encoding/json" + "errors" "fmt" "io/ioutil" "log" @@ -45,7 +47,8 @@ type OutlookFolders struct { } func getOutlookFolders(client *http.Client) (OutlookFolders, error) { - requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/users/ec03b4f2-fccf-4c35-b0eb-be85a0f5dd43/mailFolders") + //requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/users/ec03b4f2-fccf-4c35-b0eb-be85a0f5dd43/mailFolders") + requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/me/mailFolders") ret, err := client.Get(requestUrl) if err != nil { @@ -59,7 +62,7 @@ func getOutlookFolders(client *http.Client) (OutlookFolders, error) { return OutlookFolders{}, err } - log.Printf("[INFO] Folder Body: %s", string(body)) + //log.Printf("[INFO] Folder Body: %s", string(body)) log.Printf("[INFO] Status folders: %d", ret.StatusCode) if ret.StatusCode != 200 { return OutlookFolders{}, err @@ -165,6 +168,8 @@ func handleNewOutlookRegister(resp http.ResponseWriter, request *http.Request) { continue } + log.Printf("ITEM: %#v", itemsplit) + // Do something here if itemsplit[0] == "workflow_id" { trigger.WorkflowId = itemsplit[1] @@ -192,16 +197,16 @@ func handleNewOutlookRegister(resp http.ResponseWriter, request *http.Request) { //log.Printf("%#v", trigger) log.Println(trigger.WorkflowId) log.Println(trigger.Id) - log.Println(trigger.Username) + log.Println(senderUser) log.Println(trigger.Type) - if trigger.WorkflowId == "" || trigger.Id == "" || trigger.Username == "" || trigger.Type == "" { + log.Printf("[INFO] Attempting to set up outlook trigger for %s", senderUser) + if trigger.WorkflowId == "" || trigger.Id == "" || senderUser == "" || trigger.Type == "" { log.Printf("[INFO] All oauth items need to contain data to register a new state") resp.WriteHeader(401) return } // Should also update the user - log.Printf("[INFO] Attempting to set up outlook trigger for %s", senderUser) Userdata, err := getUser(ctx, senderUser) if err != nil { log.Printf("[INFO] Username %s doesn't exist (oauth2): %s", trigger.Username, err) @@ -466,3 +471,271 @@ func handleGetSpecificTrigger(resp http.ResponseWriter, request *http.Request) { resp.WriteHeader(200) resp.Write(b) } + +// This sets up the sub with outlook itself +// Parses data from the workflow to see whether access is right to subscribe it +// Creates the cloud function for outlook return +// Wait for it to be available, then schedule a workflow to it +func createOutlookSub(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + location := strings.Split(request.URL.String(), "/") + + var workflowId string + if location[1] == "api" { + if len(location) <= 4 { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + workflowId = location[4] + } + + ctx := context.Background() + workflow, err := getWorkflow(ctx, workflowId) + if err != nil { + log.Printf("Failed getting the workflow locally (outlook sub): %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + user, err := handleApiAuthentication(resp, request) + if err != nil { + log.Printf("Api authentication failed in outlook deploy: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + // FIXME - have a check for org etc too.. + if user.Id != workflow.Owner && user.Role != "admin" { + log.Printf("Wrong user (%s) for workflow %s when deploying outlook", user.Username, workflow.ID) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + log.Println("Handle outlook subscription for trigger") + + // Should already be authorized at this point, as the workflow is shared + body, err := ioutil.ReadAll(request.Body) + if err != nil { + log.Printf("Failed body read for workflow %s", workflow.ID) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + log.Println(string(body)) + + // Based on the input data from frontend + type CurTrigger struct { + Name string `json:"name"` + Folders []string `json:"folders"` + ID string `json:"id"` + } + + var curTrigger CurTrigger + err = json.Unmarshal(body, &curTrigger) + if err != nil { + log.Printf("Failed body read unmarshal for trigger %s", workflow.ID) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if len(curTrigger.Folders) == 0 { + log.Printf("Error for %s. Choosing folders is required, currently 0", workflow.ID) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + // Now that it's deployed - wait a few seconds before generating: + // 1. Oauth2 token thingies for outlook.office.com + // 2. Set the url to have the right mailboxes (probably ID?) ("https://outlook.office.com/api/v2.0/me/mailfolders('inbox')/messages") + // 3. Set the callback URL to be the new trigger + // 4. Run subscription test + // 5. Set the subscriptionId to the trigger object + + // First - lets regenerate an oauth token for outlook.office.com from the original items + trigger, err := getTriggerAuth(ctx, curTrigger.ID) + if err != nil { + log.Printf("Trigger %s doesn't exist - outlook sub.", curTrigger.ID) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": ""}`)) + return + } + + // url doesn't really matter here + url := fmt.Sprintf("https://shuffler.io") + outlookClient, _, err := getOutlookClient(ctx, "", trigger.OauthToken, url) + if err != nil { + log.Printf("Oauth client failure - triggerauth: %s", err) + resp.WriteHeader(401) + return + } + + // Location + + notificationURL := fmt.Sprintf("https://%s-%s.cloudfunctions.net/outlooktrigger_%s", defaultLocation, gceProject, curTrigger.ID) + log.Println(notificationURL) + + // This is here simply to let the function start + // Usually takes 10 attempts minimum :O + // 10 * 5 = 50 seconds. That's waaay too much :( + //notificationURL = "https://europe-west1-shuffler.cloudfunctions.net/outlooktrigger_e2ce43b0-997e-4980-9617-6eadbc68cf88" + //notificationURL = "https://de4fc12b.ngrok.io" + + curSubscriptions, err := getOutlookSubscriptions(outlookClient) + if err == nil { + for _, sub := range curSubscriptions.Value { + if sub.NotificationURL == notificationURL { + log.Printf("Removing existing subscription %s", sub.Id) + removeOutlookSubscription(outlookClient, sub.Id) + } + } + } else { + log.Printf("Failed to get subscriptions - need to overwrite") + } + + maxFails := 15 + failCnt := 0 + log.Println(curTrigger.Folders) + for { + subId, err := makeOutlookSubscription(outlookClient, curTrigger.Folders, notificationURL) + if err != nil { + failCnt += 1 + log.Printf("Failed making oauth subscription, retrying in 5 seconds: %s", err) + time.Sleep(5 * time.Second) + if failCnt == maxFails { + log.Printf("Failed to set up subscription %d times.", maxFails) + resp.WriteHeader(401) + return + } + + continue + } + + // Set the ID somewhere here + trigger.SubscriptionId = subId + err = setTriggerAuth(ctx, *trigger) + if err != nil { + log.Printf("Failed setting triggerauth: %s", err) + } + + break + } + + log.Printf("Successfully handled outlook subscription for trigger %s in workflow %s", curTrigger.ID, workflow.ID) + + //log.Printf("%#v", user) + resp.WriteHeader(200) + resp.Write([]byte(`{"success": true}`)) +} + +// Lists the users current subscriptions +func getOutlookSubscriptions(outlookClient *http.Client) (SubscriptionsWrapper, error) { + fullUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/subscriptions") + req, err := http.NewRequest( + "GET", + fullUrl, + nil, + ) + req.Header.Add("Content-Type", "application/json") + res, err := outlookClient.Do(req) + if err != nil { + log.Printf("suberror Client: %s", err) + return SubscriptionsWrapper{}, err + } + + body, err := ioutil.ReadAll(res.Body) + if err != nil { + log.Printf("Suberror Body: %s", err) + return SubscriptionsWrapper{}, err + } + + newSubs := SubscriptionsWrapper{} + err = json.Unmarshal(body, &newSubs) + if err != nil { + return SubscriptionsWrapper{}, err + } + + return newSubs, nil +} + +type SubscriptionsWrapper struct { + OdataContext string `json:"@odata.context"` + Value []Subscription `json:"value"` +} + +type Subscription struct { + ChangeType string `json:"changeType"` + NotificationURL string `json:"notificationUrl"` + Resource string `json:"resource"` + ExpirationDateTime string `json:"expirationDateTime"` + ClientState string `json:"clientState"` + Id string `json:"id"` +} + +func makeOutlookSubscription(client *http.Client, folderIds []string, notificationURL string) (string, error) { + fullUrl := "https://graph.microsoft.com/v1.0/subscriptions" + + // FIXME - this expires rofl + t := time.Now().Local().Add(time.Minute * time.Duration(4300)) + timeFormat := fmt.Sprintf("%d-%02d-%02dT%02d:%02d:%02d.0000000Z", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second()) + log.Println(timeFormat) + + resource := fmt.Sprintf("me/mailfolders('%s')/messages", strings.Join(folderIds, "','")) + log.Println(resource) + sub := Subscription{ + ChangeType: "created", + NotificationURL: notificationURL, + ExpirationDateTime: timeFormat, + ClientState: "This is a test", + Resource: resource, + } + + data, err := json.Marshal(sub) + if err != nil { + log.Printf("Marshal: %s", err) + return "", err + } + + req, err := http.NewRequest( + "POST", + fullUrl, + bytes.NewBuffer(data), + ) + req.Header.Add("Content-Type", "application/json") + + res, err := client.Do(req) + if err != nil { + log.Printf("Client: %s", err) + return "", err + } + + log.Printf("Status: %d", res.StatusCode) + body, err := ioutil.ReadAll(res.Body) + if err != nil { + log.Printf("Body: %s", err) + return "", err + } + + if res.StatusCode != 200 && res.StatusCode != 201 { + return "", errors.New(fmt.Sprintf("Subscription failed: %s", string(body))) + } + + // Use data from body here to create thingy + newSub := Subscription{} + err = json.Unmarshal(body, &newSub) + if err != nil { + return "", err + } + + return newSub.Id, nil +} From d662e9d6aa564c50eba7b5cab97e2e378d44f74e Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 14 Mar 2021 17:20:41 +0100 Subject: [PATCH 39/57] Fixed last pieces for oauth and app integrations --- backend/app_sdk/app_base.py | 2 +- backend/app_sdk/build.sh | 2 +- backend/go-app/main.go | 210 +++------- backend/go-app/oauth2.go | 519 +++++++++++++++++++++++-- backend/go-app/walkoff.go | 12 +- frontend/src/views/AngularWorkflow.jsx | 138 ++++--- 6 files changed, 641 insertions(+), 242 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index b9eee023..4db578cb 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -1743,7 +1743,7 @@ class AppBase: # Custom format for ${name[0,1,2,...]}$ #submatch = "([${]{2}([0-9a-zA-Z_-]+)(\[.*\])[}$]{2})" - print(f"Returnedvalue: {value}") + #print(f"Returnedvalue: {value}") # OLD: Used until 13.03.2021: submatch = "([${]{2}#?([0-9a-zA-Z_-]+)#?(\[.*\])[}$]{2})" # \${[0-9a-zA-Z_-]+#?(\[.*?]}\$) submatch = "([${]{2}#?([0-9a-zA-Z_-]+)#?(\[.*?]}\$))" diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 258a65db..eaee980c 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,6 +1,6 @@ #!/bin/bash NAME=shuffle-app_sdk -VERSION=0.8.63 +VERSION=0.8.64 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 29e1432e..1b6be031 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -73,11 +73,13 @@ var bucketName = "shuffler.appspot.com" var baseAppPath = "/home/frikky/git/shaffuru/tmp/apps" var baseDockerName = "frikky/shuffle" var registryName = "registry.hub.docker.com" +var runningEnvironment = "onprem" -//var syncUrl = "http://192.168.102.54:5002" var syncUrl = "https://shuffler.io" +var syncSubUrl = "https://shuffler.io" //var syncUrl = "http://localhost:5002" +//var syncSubUrl = "https://050196912a9d.ngrok.io" var dbclient *datastore.Client var requestCache *cache.Cache @@ -3478,10 +3480,12 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { // OrgId: activeOrgs[0].Id, workflowExecution, executionResp, err := handleExecution(item, workflow, newRequest) if err == nil { - err = increaseStatisticsField(ctx, "total_webhooks_ran", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) - if err != nil { - log.Printf("Failed to increase total apps loaded stats: %s", err) - } + /* + err = increaseStatisticsField(ctx, "total_webhooks_ran", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) + if err != nil { + log.Printf("Failed to increase total apps loaded stats: %s", err) + } + */ resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s", "authorization": "%s"}`, workflowExecution.ExecutionId, workflowExecution.Authorization))) @@ -4918,145 +4922,6 @@ func handleGetSpecificStats(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(b)) } -func handleDeleteOutlookSub(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - location := strings.Split(request.URL.String(), "/") - - var workflowId string - var triggerId string - if location[1] == "api" { - if len(location) <= 6 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - workflowId = location[4] - triggerId = location[6] - } - - if len(workflowId) == 0 || len(triggerId) == 0 { - log.Printf("Ids can't be zero when deleting %s", workflowId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - ctx := context.Background() - workflow, err := getWorkflow(ctx, workflowId) - if err != nil { - log.Printf("Failed getting the workflow locally (delete outlook): %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in outlook deploy: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - // FIXME - have a check for org etc too.. - if user.Id != workflow.Owner && user.Role != "admin" { - log.Printf("Wrong user (%s) for workflow %s when deploying outlook", user.Username, workflow.ID) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - // Check what kind of sub it is - err = handleOutlookSubRemoval(ctx, workflowId, triggerId) - if err != nil { - log.Printf("Failed sub removal: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true}`)) -} - -func removeOutlookSubscription(outlookClient *http.Client, subscriptionId string) error { - // DELETE https://graph.microsoft.com/v1.0/subscriptions/{id} - fullUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/subscriptions/%s", subscriptionId) - req, err := http.NewRequest( - "DELETE", - fullUrl, - nil, - ) - req.Header.Add("Content-Type", "application/json") - res, err := outlookClient.Do(req) - if err != nil { - log.Printf("Client: %s", err) - return err - } - - if res.StatusCode != 200 && res.StatusCode != 201 && res.StatusCode != 204 { - return errors.New(fmt.Sprintf("Bad status code when deleting subscription: %d", res.StatusCode)) - } - - body, err := ioutil.ReadAll(res.Body) - if err != nil { - log.Printf("Body: %s", err) - return err - } - - _ = body - - return nil -} - -// Remove AUTH -// Remove function -// Remove subscription -func handleOutlookSubRemoval(ctx context.Context, workflowId, triggerId string) error { - // 1. Get the auth for trigger - // 2. Stop the subscription - // 3. Remove the function - // 4. Remove the database entry for auth - trigger, err := getTriggerAuth(ctx, triggerId) - if err != nil { - log.Printf("Trigger auth %s doesn't exist - outlook sub removal.", triggerId) - return err - } - - url := fmt.Sprintf("https://shuffler.io") - outlookClient, _, err := getOutlookClient(ctx, "", trigger.OauthToken, url) - if err != nil { - log.Printf("Oauth client failure - triggerauth sub removal: %s", err) - return err - } - - notificationURL := fmt.Sprintf("https://%s-%s.cloudfunctions.net/outlooktrigger_%s", defaultLocation, gceProject, trigger.Id) - curSubscriptions, err := getOutlookSubscriptions(outlookClient) - if err == nil { - for _, sub := range curSubscriptions.Value { - if sub.NotificationURL == notificationURL { - log.Printf("Removing existing subscription %s", sub.Id) - removeOutlookSubscription(outlookClient, sub.Id) - } - } - } else { - log.Printf("Failed to get subscriptions - need to overwrite") - } - - // FIXME - not removing the function, as the trigger still exists - //err = removeOutlookTriggerFunction(triggerId) - //if err != nil { - // return err - //} - - return nil -} - func getOpenapi(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -5971,8 +5836,58 @@ func handleCloudExecutionOnprem(workflowId, startNode, executionSource, executio func handleCloudJob(job CloudSyncJob) error { // May need authentication in all of these..? - log.Printf("Handle job with type %s and action %s", job.Type, job.Action) - if job.Type == "webhook" { + log.Printf("[INFO] Handle job with type %s and action %s", job.Type, job.Action) + if job.Type == "outlook" { + if job.Action == "execute" { + // FIXME: Get the email + ctx := context.Background() + maildata := MailData{} + err := json.Unmarshal([]byte(job.ThirdItem), &maildata) + if err != nil { + log.Printf("Maildata unmarshal error: %s", err) + return err + } + + hookId := job.Id + hook, err := getTriggerAuth(ctx, hookId) + if err != nil { + log.Printf("[INFO] Failed getting trigger %s (callback cloud): %s", hookId, err) + return err + } + + redirectDomain := "localhost:5001" + redirectUrl := fmt.Sprintf("http://%s/api/v1/triggers/outlook/register", redirectDomain) + outlookClient, _, err := getOutlookClient(ctx, "", hook.OauthToken, redirectUrl) + if err != nil { + log.Printf("Oauth client failure - triggerauth: %s", err) + return err + } + + emails, err := getOutlookEmail(outlookClient, maildata) + log.Printf("EMAILS: %d", len(emails)) + log.Printf("INSIDE GET OUTLOOK EMAIL!: %#v, %s", emails, err) + + //type FullEmail struct { + email := FullEmail{} + if len(emails) == 1 { + email = emails[0] + } + + emailBytes, err := json.Marshal(email) + if err != nil { + log.Printf("[INFO] Failed email marshaling: %s", err) + return err + } + + log.Printf("Should handle webhook for workflow %s with start node %s and data %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem) + err = handleCloudExecutionOnprem(job.PrimaryItemId, job.SecondaryItem, "outlook", string(emailBytes)) + if err != nil { + log.Printf("Failed executing workflow from cloud outlook hook: %s", err) + } else { + log.Printf("Successfully executed workflow from cloud outlook hook!") + } + } + } else if job.Type == "webhook" { if job.Action == "execute" { log.Printf("Should handle webhook for workflow %s with start node %s and data %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem) err := handleCloudExecutionOnprem(job.PrimaryItemId, job.SecondaryItem, "webhook", job.ThirdItem) @@ -6138,7 +6053,7 @@ func remoteOrgJobController(org Org, body []byte) error { } if len(responseData.Jobs) > 0 { - log.Printf("Remote JOB ret: %s", string(body)) + log.Printf("[INFO] Remote JOB ret: %s", string(body)) log.Printf("Got job with reason %s and %d job(s)", responseData.Reason, len(responseData.Jobs)) } @@ -7887,6 +7802,7 @@ func initHandlers() { r.HandleFunc("/api/v1/triggers/outlook/register", handleNewOutlookRegister).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/triggers/outlook/getFolders", handleGetOutlookFolders).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/triggers/outlook/{key}", handleGetSpecificTrigger).Methods("GET", "OPTIONS") + //r.HandleFunc("/api/v1/triggers/outlook/{key}/callback", handleOutlookCallback).Methods("POST", "OPTIONS") //r.HandleFunc("/api/v1/stats/{key}", handleGetSpecificStats).Methods("GET", "OPTIONS") http.Handle("/", r) diff --git a/backend/go-app/oauth2.go b/backend/go-app/oauth2.go index 598c7b27..04bb845c 100644 --- a/backend/go-app/oauth2.go +++ b/backend/go-app/oauth2.go @@ -9,6 +9,7 @@ import ( "io/ioutil" "log" "net/http" + "net/url" "strings" "time" @@ -16,6 +17,77 @@ import ( "golang.org/x/oauth2" ) +type FullEmail struct { + OdataContext string `json:"@odata.context"` + OdataEtag string `json:"@odata.etag"` + ID string `json:"id"` + Createddatetime time.Time `json:"createdDateTime"` + Lastmodifieddatetime time.Time `json:"lastModifiedDateTime"` + Changekey string `json:"changeKey"` + Categories []interface{} `json:"categories"` + Receiveddatetime time.Time `json:"receivedDateTime"` + Sentdatetime time.Time `json:"sentDateTime"` + Hasattachments bool `json:"hasAttachments"` + Internetmessageid string `json:"internetMessageId"` + Subject string `json:"subject"` + Bodypreview string `json:"bodyPreview"` + Importance string `json:"importance"` + Parentfolderid string `json:"parentFolderId"` + Conversationid string `json:"conversationId"` + Conversationindex string `json:"conversationIndex"` + Isdeliveryreceiptrequested interface{} `json:"isDeliveryReceiptRequested"` + Isreadreceiptrequested bool `json:"isReadReceiptRequested"` + Isread bool `json:"isRead"` + Isdraft bool `json:"isDraft"` + Weblink string `json:"webLink"` + Inferenceclassification string `json:"inferenceClassification"` + Body struct { + Contenttype string `json:"contentType"` + Content string `json:"content"` + } `json:"body"` + Sender struct { + Emailaddress struct { + Name string `json:"name"` + Address string `json:"address"` + } `json:"emailAddress"` + } `json:"sender"` + From struct { + Emailaddress struct { + Name string `json:"name"` + Address string `json:"address"` + } `json:"emailAddress"` + } `json:"from"` + Torecipients []struct { + Emailaddress struct { + Name string `json:"name"` + Address string `json:"address"` + } `json:"emailAddress"` + } `json:"toRecipients"` + Ccrecipients []interface{} `json:"ccRecipients"` + Bccrecipients []interface{} `json:"bccRecipients"` + Replyto []interface{} `json:"replyTo"` + Flag struct { + Flagstatus string `json:"flagStatus"` + } `json:"flag"` +} + +type MailData struct { + Value []struct { + Subscriptionid string `json:"subscriptionId"` + Subscriptionexpirationdatetime string `json:"subscriptionExpirationDateTime"` + Changetype string `json:"changeType"` + Resource string `json:"resource"` + Resourcedata struct { + OdataType string `json:"@odata.type"` + OdataID string `json:"@odata.id"` + OdataEtag string `json:"@odata.etag"` + ID string `json:"id"` + } `json:"resourceData"` + Clientstate string `json:"clientState"` + Tenantid string `json:"tenantId"` + } `json:"value"` +} + type OutlookProfile struct { OdataContext string `json:"@odata.context"` BusinessPhones []string `json:"businessPhones"` @@ -46,6 +118,50 @@ type OutlookFolders struct { Value []OutlookFolder `json:"value"` } +func getOutlookEmail(client *http.Client, maildata MailData) ([]FullEmail, error) { + //requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/users/ec03b4f2-fccf-4c35-b0eb-be85a0f5dd43/mailFolders") + + emails := []FullEmail{} + for _, email := range maildata.Value { + //messageId := email.Resourcedata.ID + //requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/me/%s", messageId) + requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/%s", email.Resource) + log.Printf("URL: %#v", requestUrl) + + ret, err := client.Get(requestUrl) + if err != nil { + log.Printf("[INFO] OutlookErr: %s", err) + return []FullEmail{}, err + } + + body, err := ioutil.ReadAll(ret.Body) + if err != nil { + log.Printf("[WARNING] Failed body decoding from outlook email") + return []FullEmail{}, err + } + + //type FullEmail struct { + log.Printf("[INFO] EMAIL Body: %s", string(body)) + log.Printf("[INFO] Status email: %d", ret.StatusCode) + if ret.StatusCode != 200 { + return []FullEmail{}, err + } + + //log.Printf("Body: %s", string(body)) + + parsedmail := FullEmail{} + err = json.Unmarshal(body, &parsedmail) + if err != nil { + log.Printf("[INFO] Email unmarshal error: %s", err) + return []FullEmail{}, err + } + + emails = append(emails, parsedmail) + } + + return emails, nil +} + func getOutlookFolders(client *http.Client) (OutlookFolders, error) { //requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/users/ec03b4f2-fccf-4c35-b0eb-be85a0f5dd43/mailFolders") requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/me/mailFolders") @@ -127,7 +243,7 @@ func handleNewOutlookRegister(resp http.ResponseWriter, request *http.Request) { url := fmt.Sprintf("http://%s%s", request.Host, request.URL.EscapedPath()) log.Println(url) ctx := context.Background() - client, accessToken, err := getOutlookClient(ctx, code, OauthToken{}, url) + _, accessToken, err := getOutlookClient(ctx, code, OauthToken{}, url) if err != nil { log.Printf("Oauth client failure - outlook register: %s", err) resp.WriteHeader(401) @@ -135,12 +251,15 @@ func handleNewOutlookRegister(resp http.ResponseWriter, request *http.Request) { } // This should be possible, and will also give the actual username - profile, err := getOutlookProfile(client) - if err != nil { - log.Printf("Outlook profile failure: %s", err) - resp.WriteHeader(401) - return - } + + /* + profile, err := getOutlookProfile(client) + if err != nil { + log.Printf("Outlook profile failure: %s", err) + resp.WriteHeader(401) + return + } + */ // This is a state workaround, which should really be for CSRF checks lol state := request.URL.Query().Get("state") @@ -168,7 +287,7 @@ func handleNewOutlookRegister(resp http.ResponseWriter, request *http.Request) { continue } - log.Printf("ITEM: %#v", itemsplit) + //log.Printf("ITEM: %#v", itemsplit) // Do something here if itemsplit[0] == "workflow_id" { @@ -177,6 +296,8 @@ func handleNewOutlookRegister(resp http.ResponseWriter, request *http.Request) { trigger.Id = itemsplit[1] } else if itemsplit[0] == "type" { trigger.Type = itemsplit[1] + } else if itemsplit[0] == "start" { + trigger.Start = itemsplit[1] } else if itemsplit[0] == "username" { trigger.Username = itemsplit[1] trigger.Owner = itemsplit[1] @@ -185,7 +306,12 @@ func handleNewOutlookRegister(resp http.ResponseWriter, request *http.Request) { } // THis is an override based on the user in oauth return - trigger.Username = profile.Mail + /* + if len(profile.Mail) > 0 { + trigger.Username = profile.Mail + } + */ + trigger.Code = code trigger.OauthToken = OauthToken{ AccessToken: accessToken.AccessToken, @@ -199,6 +325,7 @@ func handleNewOutlookRegister(resp http.ResponseWriter, request *http.Request) { log.Println(trigger.Id) log.Println(senderUser) log.Println(trigger.Type) + log.Printf("STARTNODE: %s", trigger.Start) log.Printf("[INFO] Attempting to set up outlook trigger for %s", senderUser) if trigger.WorkflowId == "" || trigger.Id == "" || senderUser == "" || trigger.Type == "" { log.Printf("[INFO] All oauth items need to contain data to register a new state") @@ -264,6 +391,7 @@ type OauthToken struct { RefreshToken string `json:"RefreshToken" datastore:"RefreshToken,noindex"` Expiry time.Time `json:"Expiry" datastore:"Expiry,noindex"` } + type TriggerAuth struct { Id string `json:"id" datastore:"id"` SubscriptionId string `json:"subscriptionId" datastore:"subscriptionId"` @@ -273,6 +401,7 @@ type TriggerAuth struct { Owner string `json:"owner" datastore:"owner"` Type string `json:"type" datastore:"type"` Code string `json:"code,omitempty" datastore:"code,noindex"` + Start string `json:"start" datastore:"start"` OauthToken OauthToken `json:"oauth_token,omitempty" datastore:"oauth_token"` } @@ -520,7 +649,7 @@ func createOutlookSub(resp http.ResponseWriter, request *http.Request) { return } - log.Println("Handle outlook subscription for trigger") + log.Println("[INFO] Handle outlook subscription for trigger") // Should already be authorized at this point, as the workflow is shared body, err := ioutil.ReadAll(request.Body) @@ -531,8 +660,6 @@ func createOutlookSub(resp http.ResponseWriter, request *http.Request) { return } - log.Println(string(body)) - // Based on the input data from frontend type CurTrigger struct { Name string `json:"name"` @@ -540,6 +667,7 @@ func createOutlookSub(resp http.ResponseWriter, request *http.Request) { ID string `json:"id"` } + //log.Println(string(body)) var curTrigger CurTrigger err = json.Unmarshal(body, &curTrigger) if err != nil { @@ -566,44 +694,74 @@ func createOutlookSub(resp http.ResponseWriter, request *http.Request) { // First - lets regenerate an oauth token for outlook.office.com from the original items trigger, err := getTriggerAuth(ctx, curTrigger.ID) if err != nil { - log.Printf("Trigger %s doesn't exist - outlook sub.", curTrigger.ID) + log.Printf("[INFO] Trigger %s doesn't exist - outlook sub.", curTrigger.ID) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": ""}`)) return } // url doesn't really matter here - url := fmt.Sprintf("https://shuffler.io") + //url := fmt.Sprintf("https://shuffler.io") + redirectDomain := "localhost:5001" + url := fmt.Sprintf("http://%s/api/v1/triggers/outlook/register", redirectDomain) outlookClient, _, err := getOutlookClient(ctx, "", trigger.OauthToken, url) if err != nil { log.Printf("Oauth client failure - triggerauth: %s", err) + resp.Write([]byte(`{"success": false, "reason": ""}`)) resp.WriteHeader(401) return } // Location + - notificationURL := fmt.Sprintf("https://%s-%s.cloudfunctions.net/outlooktrigger_%s", defaultLocation, gceProject, curTrigger.ID) - log.Println(notificationURL) // This is here simply to let the function start // Usually takes 10 attempts minimum :O // 10 * 5 = 50 seconds. That's waaay too much :( - //notificationURL = "https://europe-west1-shuffler.cloudfunctions.net/outlooktrigger_e2ce43b0-997e-4980-9617-6eadbc68cf88" - //notificationURL = "https://de4fc12b.ngrok.io" + if runningEnvironment != "cloud" { + org, err := getOrg(ctx, user.ActiveOrg.Id) + if err != nil { + log.Printf("Failed finding org %s: %s", org.Id, err) + return + } + log.Printf("[INFO] Starting cloud configuration TO STOP trigger %s in org %s", trigger.Id, org.Id) + + action := CloudSyncJob{ + Type: "outlook", + Action: "start", + OrgId: org.Id, + PrimaryItemId: trigger.Id, + SecondaryItem: trigger.Start, + ThirdItem: trigger.WorkflowId, + } + + err = executeCloudAction(action, org.SyncConfig.Apikey) + if err != nil { + log.Printf("[INFO] Failed cloud action START outlook execution: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + return + } else { + log.Printf("[INFO] Successfully set up cloud action trigger") + } + } else { + log.Printf("Should configure a running environment for CLOUD") + } + + notificationURL := fmt.Sprintf("%s/api/v1/hooks/webhook_%s", syncSubUrl, trigger.Id) curSubscriptions, err := getOutlookSubscriptions(outlookClient) if err == nil { for _, sub := range curSubscriptions.Value { if sub.NotificationURL == notificationURL { - log.Printf("Removing existing subscription %s", sub.Id) + log.Printf("[INFO] Removing existing subscription %s", sub.Id) removeOutlookSubscription(outlookClient, sub.Id) } } } else { - log.Printf("Failed to get subscriptions - need to overwrite") + log.Printf("[INFO] Failed to get subscriptions - need to overwrite") } - maxFails := 15 + maxFails := 5 failCnt := 0 log.Println(curTrigger.Folders) for { @@ -631,7 +789,7 @@ func createOutlookSub(resp http.ResponseWriter, request *http.Request) { break } - log.Printf("Successfully handled outlook subscription for trigger %s in workflow %s", curTrigger.ID, workflow.ID) + log.Printf("[INFO] Successfully handled outlook subscription for trigger %s in workflow %s", curTrigger.ID, workflow.ID) //log.Printf("%#v", user) resp.WriteHeader(200) @@ -686,19 +844,19 @@ func makeOutlookSubscription(client *http.Client, folderIds []string, notificati fullUrl := "https://graph.microsoft.com/v1.0/subscriptions" // FIXME - this expires rofl - t := time.Now().Local().Add(time.Minute * time.Duration(4300)) + t := time.Now().Local().Add(time.Minute * time.Duration(4200)) timeFormat := fmt.Sprintf("%d-%02d-%02dT%02d:%02d:%02d.0000000Z", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second()) - log.Println(timeFormat) resource := fmt.Sprintf("me/mailfolders('%s')/messages", strings.Join(folderIds, "','")) - log.Println(resource) + log.Printf("[INFO] Subscription resource to get(s): %s", resource) sub := Subscription{ ChangeType: "created", + ClientState: "Shuffle subscription", NotificationURL: notificationURL, ExpirationDateTime: timeFormat, - ClientState: "This is a test", Resource: resource, } + //ClientState: "This is a test", data, err := json.Marshal(sub) if err != nil { @@ -719,7 +877,7 @@ func makeOutlookSubscription(client *http.Client, folderIds []string, notificati return "", err } - log.Printf("Status: %d", res.StatusCode) + log.Printf("[INFO] Subscription Status: %d", res.StatusCode) body, err := ioutil.ReadAll(res.Body) if err != nil { log.Printf("Body: %s", err) @@ -739,3 +897,310 @@ func makeOutlookSubscription(client *http.Client, folderIds []string, notificati return newSub.Id, nil } + +// Basically the same as a webhook +func handleOutlookCallback(resp http.ResponseWriter, request *http.Request) { + path := strings.Split(request.URL.String(), "/") + if len(path) < 4 { + log.Printf("[INFO] Bad outlook callback URL: %s", path) + resp.WriteHeader(403) + resp.Write([]byte(`{"success": false}`)) + return + } + + // 1. Get config with hookId + //fmt.Sprintf("%s/api/v1/hooks/%s", callbackUrl, hookId) + ctx := context.Background() + location := strings.Split(request.URL.String(), "/") + + var hookId string + if location[1] == "api" { + if len(location) <= 4 { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + hookId = location[5] + } + + // ID: webhook_ + if len(hookId) != 36 { + log.Printf("[WARNING] Bad hook ID: %s (%d)", hookId, len(hookId)) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "message": "ID not valid"}`)) + return + } + + //func getTriggerAuth(ctx context.Context, id string) (*TriggerAuth, error) { + hook, err := getTriggerAuth(ctx, hookId) + if err != nil { + log.Printf("[INFO] Failed getting trigger %s (callback): %s", hookId, err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + body, err := ioutil.ReadAll(request.Body) + if err != nil { + log.Printf("[INFO] Body data error: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + //log.Printf("[INFO] BODY: %s. Len: %d", string(body), len(body)) + //key, ok := request.URL.Query()["validationToken"] + //if ok { + //} + token := request.URL.Query().Get("validationToken") + if len(body) == 0 && len(token) > 0 { + log.Printf("[INFO] Should handle trigger token %s", token) + resp.WriteHeader(200) + resp.Write([]byte(string(token))) + return + } + + // 1. Take the body and parse data -> Get the email itself + + maildata := MailData{} + err = json.Unmarshal(body, &maildata) + if err != nil { + log.Printf("Maildata unmarshal error: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + redirectDomain := "localhost:5001" + redirectUrl := fmt.Sprintf("http://%s/api/v1/triggers/outlook/register", redirectDomain) + outlookClient, _, err := getOutlookClient(ctx, "", hook.OauthToken, redirectUrl) + if err != nil { + log.Printf("Oauth client failure - triggerauth: %s", err) + resp.WriteHeader(401) + return + } + + emails, err := getOutlookEmail(outlookClient, maildata) + log.Printf("EMAILS: %d", len(emails)) + log.Printf("INSIDE GET OUTLOOK EMAIL!: %#v, %s", emails, err) + + //type FullEmail struct { + email := FullEmail{} + if len(emails) == 1 { + email = emails[0] + } + + emailBytes, err := json.Marshal(email) + if err != nil { + log.Printf("[INFO] Failed email marshaling: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + type ExecutionStruct struct { + Start string `json:"start"` + ExecutionSource string `json:"execution_source"` + ExecutionArgument string `json:"execution_argument"` + } + + newBody := ExecutionStruct{ + Start: hook.Start, + ExecutionSource: "outlook", + ExecutionArgument: string(emailBytes), + } + + b, err := json.Marshal(newBody) + if err != nil { + log.Printf("[INFO] Failed newBody marshaling: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + baseUrl := &url.URL{} + newRequest := &http.Request{ + URL: baseUrl, + Method: "POST", + Body: ioutil.NopCloser(bytes.NewReader(b)), + } + + workflow := Workflow{ + ID: "", + } + + // OrgId: activeOrgs[0].Id, + workflowExecution, executionResp, err := handleExecution(hook.WorkflowId, workflow, newRequest) + if err == nil { + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s", "authorization": "%s"}`, workflowExecution.ExecutionId, workflowExecution.Authorization))) + return + } + + resp.WriteHeader(500) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, executionResp))) +} + +func removeOutlookSubscription(outlookClient *http.Client, subscriptionId string) error { + // DELETE https://graph.microsoft.com/v1.0/subscriptions/{id} + fullUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/subscriptions/%s", subscriptionId) + req, err := http.NewRequest( + "DELETE", + fullUrl, + nil, + ) + req.Header.Add("Content-Type", "application/json") + res, err := outlookClient.Do(req) + if err != nil { + log.Printf("Client: %s", err) + return err + } + + if res.StatusCode != 200 && res.StatusCode != 201 && res.StatusCode != 204 { + return errors.New(fmt.Sprintf("Bad status code when deleting subscription: %d", res.StatusCode)) + } + + body, err := ioutil.ReadAll(res.Body) + if err != nil { + log.Printf("Body: %s", err) + return err + } + + _ = body + + return nil +} + +// Remove AUTH +// Remove function +// Remove subscription +func handleOutlookSubRemoval(ctx context.Context, user User, workflowId, triggerId string) error { + // 1. Get the auth for trigger + // 2. Stop the subscription + // 3. Remove the function + // 4. Remove the database entry for auth + trigger, err := getTriggerAuth(ctx, triggerId) + if err != nil { + log.Printf("Trigger auth %s doesn't exist - outlook sub removal.", triggerId) + return err + } + + if runningEnvironment != "cloud" { + log.Printf("[INFO] SHOULD STOP OUTLOOK SUB ONPREM SYNC WITH CLOUD") + org, err := getOrg(ctx, user.ActiveOrg.Id) + if err != nil { + log.Printf("[INFO] Failed finding org %s during outlook removal: %s", org.Id, err) + return err + } + + log.Printf("[INFO] Stopping cloud configuration for trigger %s in org %s", trigger.Id, org.Id) + action := CloudSyncJob{ + Type: "outlook", + Action: "stop", + OrgId: org.Id, + PrimaryItemId: trigger.Id, + SecondaryItem: trigger.Start, + ThirdItem: trigger.WorkflowId, + } + + err = executeCloudAction(action, org.SyncConfig.Apikey) + if err != nil { + log.Printf("[INFO] Failed cloud action STOP outlook execution: %s", err) + return err + } else { + log.Printf("[INFO] Successfully set STOPPED outlook execution trigger") + } + } else { + log.Printf("SHOULD STOP OUTLOOK SUB IN CLOUD") + } + + // Actually delete the thing + redirectDomain := "localhost:5001" + url := fmt.Sprintf("http://%s/api/v1/triggers/outlook/register", redirectDomain) + outlookClient, _, err := getOutlookClient(ctx, "", trigger.OauthToken, url) + if err != nil { + log.Printf("[WARNING] Oauth client failure - outlook folders: %s", err) + return err + } + notificationURL := fmt.Sprintf("%s/api/v1/hooks/webhook_%s", syncSubUrl, trigger.Id) + curSubscriptions, err := getOutlookSubscriptions(outlookClient) + if err == nil { + for _, sub := range curSubscriptions.Value { + if sub.NotificationURL == notificationURL { + log.Printf("[INFO] Removing subscription %s from o365", sub.Id) + removeOutlookSubscription(outlookClient, sub.Id) + } + } + } else { + log.Printf("Failed to get subscriptions - need to overwrite") + } + + return nil +} + +func handleDeleteOutlookSub(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + location := strings.Split(request.URL.String(), "/") + + var workflowId string + var triggerId string + if location[1] == "api" { + if len(location) <= 6 { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + workflowId = location[4] + triggerId = location[6] + } + + if len(workflowId) == 0 || len(triggerId) == 0 { + log.Printf("Ids can't be zero when deleting %s", workflowId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + ctx := context.Background() + workflow, err := getWorkflow(ctx, workflowId) + if err != nil { + log.Printf("Failed getting the workflow locally (delete outlook): %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + user, err := handleApiAuthentication(resp, request) + if err != nil { + log.Printf("Api authentication failed in outlook deploy: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + // FIXME - have a check for org etc too.. + if user.Id != workflow.Owner && user.Role != "admin" { + log.Printf("Wrong user (%s) for workflow %s when deploying outlook", user.Username, workflow.ID) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + // Check what kind of sub it is + err = handleOutlookSubRemoval(ctx, user, workflowId, triggerId) + if err != nil { + log.Printf("Failed sub removal: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + resp.WriteHeader(200) + resp.Write([]byte(`{"success": true}`)) +} diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index ac3265de..b89d079d 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -2034,7 +2034,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { // log.Printf("Failed to delete webhook: %s", err) //} } else if item.TriggerType == "EMAIL" { - err = handleOutlookSubRemoval(ctx, workflow.ID, item.ID) + err = handleOutlookSubRemoval(ctx, user, workflow.ID, item.ID) if err != nil { log.Printf("Failed to delete email sub: %s", err) } @@ -2224,7 +2224,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("PRE BODY") + //log.Printf("PRE BODY") body, err := ioutil.ReadAll(request.Body) if err != nil { log.Printf("Failed hook unmarshaling: %s", err) @@ -2268,7 +2268,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { allNodes := []string{} workflow.Categories = Categories{} - log.Printf("PRE APPS") + //log.Printf("PRE APPS") workflowapps, apperr := getAllWorkflowApps(ctx, 500) //log.Printf("Action: %#v", action.Authentication) @@ -2553,7 +2553,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { workflow.Triggers = newTriggers - log.Printf("PRE VARIABLES") + //log.Printf("PRE VARIABLES") for _, variable := range workflow.WorkflowVariables { if len(variable.Value) == 0 { log.Printf("Can't have an empty variable: %s", variable.Name) @@ -2601,7 +2601,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } // FIXME - append all nodes (actions, triggers etc) to one single array here - log.Printf("PRE VARIABLES") + //log.Printf("PRE VARIABLES") if len(foundNodes) != len(allNodes) || len(workflow.Actions) <= 0 { // This shit takes a few seconds lol if !workflow.IsValid { @@ -2675,7 +2675,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } // Check every app action and param to see whether they exist - log.Printf("PRE ACTIONS 2") + //log.Printf("PRE ACTIONS 2") newActions = []Action{} for _, action := range workflow.Actions { reservedApps := []string{ diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index bde4fff9..321d664f 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -2221,7 +2221,7 @@ const AngularWorkflow = (props) => { "description": "Add your email provider", "trigger_type": "EMAIL", "errors": null, - "is_valid": true ? false : cloudSyncEnabled, + "is_valid": cloudSyncEnabled || isCloud ? true : false, "label": "Email", "environment": "cloud", "large_image": 'data:image/png;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4QAYRXhpZgAASUkqAAgAAAAAAAAAAAAAAP/hAytodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuMy1jMDExIDY2LjE0NTY2MSwgMjAxMi8wMi8wNi0xNDo1NjoyNyAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RSZWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZVJlZiMiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENTNiAoV2luZG93cykiIHhtcE1NOkluc3RhbmNlSUQ9InhtcC5paWQ6Nzg4QTJBMjVEMDI1MTFFN0EwQUVDODc5QjYyQkFCMUQiIHhtcE1NOkRvY3VtZW50SUQ9InhtcC5kaWQ6Nzg4QTJBMjZEMDI1MTFFN0EwQUVDODc5QjYyQkFCMUQiPiA8eG1wTU06RGVyaXZlZEZyb20gc3RSZWY6aW5zdGFuY2VJRD0ieG1wLmlpZDo3ODhBMkEyM0QwMjUxMUU3QTBBRUM4NzlCNjJCQUIxRCIgc3RSZWY6ZG9jdW1lbnRJRD0ieG1wLmRpZDo3ODhBMkEyNEQwMjUxMUU3QTBBRUM4NzlCNjJCQUIxRCIvPiA8L3JkZjpEZXNjcmlwdGlvbj4gPC9yZGY6UkRGPiA8L3g6eG1wbWV0YT4gPD94cGFja2V0IGVuZD0iciI/Pv/bAEMAAwICAgICAwICAgMDAwMEBgQEBAQECAYGBQYJCAoKCQgJCQoMDwwKCw4LCQkNEQ0ODxAQERAKDBITEhATDxAQEP/AAAsIAGQAZAEBEQD/xAAeAAABAwUBAQAAAAAAAAAAAAAAAQgJAgQFBwoGA//EAEoQAAECBAMEBwMDEQgDAAAAAAECAwAEBREGBxIIITFRCRMiMkFSYRRicSNCQxUWGBk2U1dYc3WBlJWzwdLTJDNjZXKDkeEmgqH/2gAIAQEAAD8Ak8JKipSlBwuDSpSeDw8qeRguQQrUAQNAV4JH3s+sA7OnT8n1fcv9Bfzc7wWAAToIAOsJ8Uq++H0gI1XSUlYWdSkji6fMnkBAdS9SidesWUpI3OjyjkYXtXCgbEDSFW3JT5D6+sIOzp0/J9X3NX0F/NzvBYABOggA6wnxSr74fSAjVdJSVhZ1KSOLp8yeQEBJUSVKCysaVKHB0eVPIwXIIVqAIGgK8Ej72fWFS640kNtzjcukcGli6k/GENwVagAQO0EcEjmj1g33AATe1wD3SnmffgG/Tp337mv535T+EaB2k9tzInZilVyuMq+up4iUjWxh+mFLs8s23dbv0stnmsgnwBiNPOHpddozHExMSmWsrSsA0tZIaMs0JudCfV50FKf/AFQPjDYsT7S+0LjJ8zGJc68aTqiSrSqtPoQD6ISoJH6BGFkM5c3qW8Jim5qYvlXArUFNVuZSb89y43Tlv0ju15ltMtKZzUmsRSbZGuSxC0mebcHIrV8r/wALEP02c+l2ywzAmZXDGeND+saqvEIRVWXFP0x1Z3AOE9thPx1JHioQ/un1Kn1eQZqtLn2ZuSmkJdamZVwOIWlQuktKTuUg8xFybgnUACB2gjgkc0e9BvuAAm9rgHulPM+/CpDik3bal1p8FPd8/GEtp7Ojq+r36OPUe96wWv2dF79vR5/8T/qI/ukM6RMZMGcyWyUqDMzjZ5vRWKwmy26UlQ3IQOBmLHx3IFibncIeqvWKtiCqTVbrtSmahUJ11T0zNTLqnHXnFG5UpSiSSeZi0ggggh2GxRt8Y92X69K4cr83N1zLuadCZumLXrcp4Ue0/KXPZPiW+6r0O+JxsF4zwvmFhSl42wXV5eo0Sqy6ZySmWFakJbUO/wDHiCk7wQQd4jN2v2dF79vR5v8AE/6g6rrflPYfar/S69Or9EIAAE6QoAHshfFJ5r9Ibpt3bTrOzBkbPYhpb7f11V5aqZh9le8iZUm65i3i20ntcirQPGIA6nU6jWqlNVirzr05PTzy5iZmHllTjrqyVKWoneSSSSYtoIIIIIIkI6J/axmsA4+Rs84yqZ+tvFb5VQ1vL7EjVCNyN/Bt4C1vOEn5xiYndYghVr3IHeKuY9yKVJbJu63MrV4qY7h+EVA6rEKKwvclSuLp8quQiDnpWM5ZnMrafn8HS04pykZfy6KMw2D2BNEByZUPXWQj/aEM0h3uwvk5PYmoeLM4HcnqHmth/CU1KytdwrNy5M+uUdQtZmZBYIu83oN2z3wbcbWk7ym2Zej6zuwXJ49y5yXwbUqXNgpUPZlpelnh32XmyrU24k7ik7/0WMey+wI2OPxesJfqyv5oPsCNjj8XrCX6sr+aD7AjY4/F6wl+rq/mhs2b2SGzXjPGc3s9bKuzngiq4zZGjEWJ3pRTlKwkyrcVOKCrOzVr6GRex73AiIe65TvqRWqhSet632Kadl9enTq0LKb28L24R86ZUp6jVKUrFMmVy85IvtzMu8g2U24hQUlQPMEAx0h7O+abGdWR+DM0mnAF16lMuzRTxamgNDzQHIOJWI2IpxDZ0Lm3ZdQ4tti6U/CEdeDaHJhxYWAklahwdAF9KeRjmXzRxJMYxzLxXiyacUt2sVqdnlFRuflHlq/jHmIlv6D37is1T/mlM/cvQ5zNnZ7xtl/jWc2htlFUtIYrfs7iXCLy+rpeLGk7zcDczN2vpdFrnvcSTsrIPaHwNtBYcmKlhsv02t0h4ydfw7UE9XUKPOJJC2Xmzv3KBAWNyrbvEDaDjjbTa3XVpQhAKlKUbBIHEk+ENLxdnDj/AGr8T1LJ7Zgra6NgymPmSxhmU0LhB+kkaV4OPkGynu6gG4PAlwGUuT2AMjsDy2BMuqGin06XBcdWTrfm3j3333D2nHFHeVH/AOCwjmnxr92Ve/Oc1+9VGGibDogMVP1vZWmKI+4b4dxJOybS1G4S06ht7QPipxf/ADD4kurbGhE21LpHBtwXUn4xbVNpb9OnGAAFrl3EkI4JukgFHrHMFWpdyUrE/KvAhxmZdbUDxBCyDFnEuHQej/wjNU/5rTP3L0PQ2hs1cR4f+pOUWU/VTGZeOtbFK1jW3SZNO6YqkwPBtlJ7IPfcKUi++NdYh2H5LB1CoWLtnXFD2Fs1sLS6rV+ZUXG8SqWouPtVVP0yXnCo6+8gqFtwAGAbkdpzbFcTgXNLB1Qyay9pBEri1iXm9VQxPNo/vJeVdT/dyJ3XcG9YNgTvt6TFODKZsY4nlc2MsaEmRypn25em45oMi2eqpiUANsVllA8gsiYtvUiyzcpJhz8rOylSkGqjITTUzKzTKXmHmlhSHG1C6VJI3EEEEGOXnGv3ZV785zX71UYaJjehfk3mdn/GM4oHRM4sWEBfcsiUZ1Eeu+JBUhxSbttS60+Cnu+fjCAaDbR1fV9rRx6n3vW8c5+2Bl1MZV7TGYmDXmlIaZrkxNyhIsFy0wrr2lD00OJjT0SfdE5mphvJnIrOXHmJutdalatSmZSSlxqmKhNuNOpYlWU8VOOLISAOdzuBiQLZ5yrxJQTVs382Q0/mVjrQ9VAk6m6RJp3y9Llz4NtA9ojvuFSjfdG54N8fGekZOpyUxTajKtTMrNNLYfYdSFIdbULKSpJ3EEEgiG4ZXz07s0Zis7OuJpp1zAmJFvP5cVSYWSJVYut2hurPzkC62Ce83dHFFogGxr92Ve/Oc1+9VGGiezo0MupnLzY/wezPy5bm8RqmMROsqFiUvr+SWf8AaQ2besOl6rrflPYfar/S69Or9EIAAE6QoAHshfFJ5r9Ii76Y7Z4mJgUHaSw7IqWhlCKHiLQN6RcmWmP9Nypsn8mIizj3GWGdOY2T9Xka1gSuJlH6bO/VKWbflm5hlubDam0v9U4lSC4lClBKiLp1G1iY3v8AbSdtr8LTP7Ekf6UL9tJ22vwss/sOR/pQfbSdtr8LLP7Dkf6UH20nba/C0z+w5H+lHmMxOkB2qc1MOKwrjjMNmfkPaGZxrTSZRl1iYZWFtPNOobC21pULhSSDx8DDe5uamJ6aenZt1Tr8w4p11auKlqNyT8SY2ZszZH1raIzqw1ldSG1hqozSXKlMAHTKyLZCn3VHwsi4HvKSPGOjSj0im0CjyVBpMqJen06XalZZhG7Q22kJQE+4AAIulJbJu63MrV4qY7h+EVA6rEKKwvclSuLp8quQjB45wVhrMbB9YwNjGnIn6JW5VyQnWVjihYtoTysbEKHAgGOf7a72U8abKeZszhStMOzVAnVreoNXCfk5uXvuSojcHUAgLTz3jcRGi4IIIIIuqVSqnXKnK0ajSD89PzzyJeWlpdsrcecUbJQlI3kkkAAROd0eGxqjZiy7XiLF8sy5j/FjaFVEiyhJMDeiSB9D2lkbiqw4JEO58CrUQAdJV4pPkHu+sIpxDZ0Lm3ZdQ4tti6U/CFJKiVKUFle5Sk8HR5U8jBcghWoAgaArwSPIfe9Y8PnJkvl1nzgScy7zMoDVQpMyLtlXZekHfmvNucULHgR8DcEiIZtqro1s58gZucxFg6QmsbYJQVOonpFkqnJNrw9pYTcgAfSJuk8Tp4Qz9SSklKgQQbEHwgggjYeTOz9m7n/iFGHMq8Fz1YdCgJiZSjRKSiT8955XYQB6m58AYmN2LejtwJsyoYxri52XxVmC43unA3/ZpAEb0ygVvv4F09ojgEgm7wSSq5Kgsr3KUODo8qeRguQQrUAQNAV4JHkPvesKl1bY0Im2pdI4NuC6k/GENwVagAQO0EcEjmj1g33AATe1wD3SnmffgG/Tp337mv535T+EG4i4KiCbAnvE8j7kaHzf2Hdl/O1+YqONcraexU3jd6p0i8jNFfPU1ZLnxWlUNjxL0LOTs6+tzC+bWLKSkHUWpmXl5xKR4BJAbJjD0/oTcEoeH1Uz5rbzfe0sUZlolH+pTigFelo3Nlr0UuyZgSYZn6vQ6zjKaQQpr6uz3yBI462WQhNvRVxDscMYVwvgujMYfwfh+n0Wly/ZZlJCVQw2k8tCABp9YypsL6iQAe0U8Unkj3YDcE6gAQO0EcEjmj3oN9wAE3tcA90p5n34VIcUm7bUutPgp7vn4wOpS05MNtiyZdAW0PKo+MASkuIbIulbPXKHNfOEa+V9m6zf7Vq633rcIpSoqbQ6T2lvdQo80coVxRbRMLRuVLuBts+VJ4iKnEhtb6ECwl0BxseVR8YAlJcQ2RdK2euUOa+cI18r7N1m/wBq1db71uEUpUVNodJ7S3uoUeaOUK4otomFo3Kl3A22fKk8RFTiQ2t9CBYS6A42PKo+MASkuIbIulbPXKHNfOPtKSkvNy6JiYaC3F71KJO+P//Z', @@ -4990,50 +4990,59 @@ const AngularWorkflow = (props) => { } const outlookButton = - @@ -5044,20 +5053,25 @@ const AngularWorkflow = (props) => { if (triggerAuthentication.type === "outlook") { triggerInfo =
-
-
-
- Login: -
-
- {outlookButton} + {selectedTrigger.status === "running" ? null : + +
+
+
+ Login +
+
+ {outlookButton} + + } - {triggerFolders === undefined || triggerFolders === null ? null : + {triggerFolders === undefined || triggerFolders === null ? + null :
- Folders: (hold CTRL to select multiple) + Select a folder
upload = ref} onChange={importFiles} /> + {workflows.length > 0 ? + + + + : null} + + + + + + const WorkflowView = () => { + if (workflows.length === 0) { + return ( +
+ +
+

Welcome to Shuffle

+
+
+

+ Shuffle is a flexible, easy to use, automation platform allowing users to integrate their services and devices freely. It's made to significantly reduce the amount of manual labor, and is focused on security applications. Click here to learn more. +

+
+
+ If you want to jump straight into it, click here to create your first workflow: +
+
+ + + + ..OR + + {workflowButtons} + +
+ +
+ ) + } + + return ( +
+
+
+
+

Workflows

+

THIS IS SOME WORKFLOW INFORMATION WHY ISN’T IT GROWING THE CORRECT WAY.

+
+
+ {workflowButtons} +
+
+ +
+
+
+
+
+
{workflows.length}
+
ACTIVE WORKFLOWS
+
+
+
+
+
+
+
+
{workflows.length}
+
AVAILABE WORKFLOWS
+
+
+
+
+
+
+
+
{workflows.length}
+
NOTIFICATIONS
+
+
+
+
+ +
+ {workflows.map((data, index) => { + return ( + + ) + })} +
+
+
+ ) + } + + const importWorkflowsFromUrl = (url) => { + console.log("IMPORT WORKFLOWS FROM ", downloadUrl) + + const parsedData = { + "url": url, + "field_3": downloadBranch || 'master' + } + + if (field1.length > 0) { + parsedData["field_1"] = field1 + } + + if (field2.length > 0) { + parsedData["field_2"] = field2 + } + + alert.success("Getting specific workflows from your URL.") + var cors = "cors" + fetch(globalUrl+"/api/v1/workflows/download_remote", { + method: "POST", + mode: "cors", + headers: { + 'Accept': 'application/json', + }, + body: JSON.stringify(parsedData), + credentials: "include", + }) + .then((response) => { + if (response.status === 200) { + alert.success("Successfully loaded workflows from "+downloadUrl) + getAvailableWorkflows() + } + + return response.json() + }) + .then((responseJson) => { + console.log("DATA: ", responseJson) + if (!responseJson.success) { + if (responseJson.reason !== undefined) { + alert.error("Failed loading: "+responseJson.reason) + } else { + alert.error("Failed loading") + } + } + }) + .catch(error => { + alert.error(error.toString()) + }) + } + + const handleGithubValidation = () => { + importWorkflowsFromUrl(downloadUrl) + setLoadWorkflowsModalOpen(false) + } + + const workflowDownloadModalOpen = loadWorkflowsModalOpen ? + { + }} + PaperProps={{ + style: { + backgroundColor: surfaceColor, + color: "white", + minWidth: "800px", + minHeight: "320px", + }, + }} + > + +
+ Load workflows from github repo +
+ + + +
+
+ + + Repository (supported: github, gitlab, bitbucket) + 0 ? userdata.active_org.defaults.workflow_download_repo : downloadUrl} + InputProps={{ + style:{ + color: "white", + height: "50px", + fontSize: "1em", + }, + }} + onChange={e => setDownloadUrl(e.target.value)} + placeholder="https://github.com/frikky/shuffle-apps" + fullWidth + /> + + Branch (default value is "master"): +
+ 0 ? userdata.active_org.defaults.workflow_download_branch : downloadBranch} + InputProps={{ + style:{ + color: "white", + height: "50px", + fontSize: "1em", + }, + }} + onChange={e => setDownloadBranch(e.target.value)} + placeholder="master" + fullWidth + /> +
+ + Authentication (optional - private repos etc): +
+ setField1(e.target.value)} + type="username" + placeholder="Username / APIkey (optional)" + fullWidth + /> + setField2(e.target.value)} + type="password" + placeholder="Password (optional)" + fullWidth + /> +
+ + + + + + + : null + + const loadedCheck = isLoaded && isLoggedIn && workflowDone ? +
+ + {modalView} + {deleteModal} + {workflowDownloadModalOpen} +
+ : +
+ + + Loading Workflows + +
+ + + // Maybe use gridview or something, idk + return ( +
+ {loadedCheck} +
+ ) +} + +export default MyView From 49564cdc445b2cc8d8ba04dbe3b2118ab4d66aa0 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 16 Mar 2021 01:39:47 +0100 Subject: [PATCH 42/57] Updated compose to 0.8.64 --- backend/Dockerfile | 1 + backend/go-app/main.go | 14 +-- backend/go-app/oauth2.go | 121 +++++++++++++++++++++++++ backend/go-app/walkoff.go | 64 +++++++++---- docker-compose.yml | 4 +- frontend/src/views/AngularWorkflow.jsx | 4 +- 6 files changed, 178 insertions(+), 30 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 40bb50ea..ea93625e 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -9,6 +9,7 @@ ADD ./go-app/walkoff.go /app ADD ./go-app/docker.go /app ADD ./go-app/codegen.go /app ADD ./go-app/files.go /app +ADD ./go-app/oauth2.go /app ADD ./go-app/go.mod /app diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 737e8b0e..0d855263 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5882,19 +5882,19 @@ func handleCloudJob(job CloudSyncJob) error { log.Printf("[INFO] Should handle outlook webhook for workflow %s with start node %s and data of length %d", job.PrimaryItemId, job.SecondaryItem, len(job.ThirdItem)) err = handleCloudExecutionOnprem(job.PrimaryItemId, job.SecondaryItem, "outlook", string(emailBytes)) if err != nil { - log.Printf("Failed executing workflow from cloud outlook hook: %s", err) + log.Printf("[WARNING] Failed executing workflow from cloud outlook hook: %s", err) } else { - log.Printf("Successfully executed workflow from cloud outlook hook!") + log.Printf("[INFO] Successfully executed workflow from cloud outlook hook!") } } } else if job.Type == "webhook" { if job.Action == "execute" { - log.Printf("Should handle normal webhook for workflow %s with start node %s and data %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem) + log.Printf("[INFO] Should handle normal webhook for workflow %s with start node %s and data %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem) err := handleCloudExecutionOnprem(job.PrimaryItemId, job.SecondaryItem, "webhook", job.ThirdItem) if err != nil { - log.Printf("Failed executing workflow from cloud hook: %s", err) + log.Printf("[INFO] Failed executing workflow from cloud hook: %s", err) } else { - log.Printf("Successfully executed workflow from cloud hook!") + log.Printf("[INFO] Successfully executed workflow from cloud hook!") } } @@ -5903,9 +5903,9 @@ func handleCloudJob(job CloudSyncJob) error { log.Printf("Should handle schedule for workflow %s with start node %s and data %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem) err := handleCloudExecutionOnprem(job.PrimaryItemId, job.SecondaryItem, "schedule", job.ThirdItem) if err != nil { - log.Printf("Failed executing workflow from cloud schedule: %s", err) + log.Printf("[INFO] Failed executing workflow from cloud schedule: %s", err) } else { - log.Printf("Successfully executed workflow from cloud schedule") + log.Printf("[INFO] Successfully executed workflow from cloud schedule") } } } else if job.Type == "email_trigger" { diff --git a/backend/go-app/oauth2.go b/backend/go-app/oauth2.go index 8c4a355f..b00f6c7a 100644 --- a/backend/go-app/oauth2.go +++ b/backend/go-app/oauth2.go @@ -17,6 +17,41 @@ import ( "golang.org/x/oauth2" ) +// This is what the structure should be when it's sent into a workflow +type ParsedShuffleMail struct { + Body struct { + URI []string `json:"uri"` + Email []string `json:"email"` + Domain []string `json:"domain"` + ContentHeader struct { + } `json:"content_header"` + Content string `json:"content"` + ContentType string `json:"content_type"` + Hash string `json:"hash"` + RawBody string `json:"raw_body"` + } `json:"body"` + Header struct { + Subject string `json:"subject"` + From string `json:"from"` + To []string `json:"to"` + Date string `json:"date"` + Received []struct { + Src string `json:"src"` + From []string `json:"from"` + By []string `json:"by"` + With string `json:"with"` + Date string `json:"date"` + } `json:"received"` + ReceivedDomain []string `json:"received_domain"` + ReceivedIP []string `json:"received_ip"` + Header struct { + } `json:"header"` + } `json:"header"` + MessageID string `json:"message_id"` + EmailFileid string `json:"email_fileid"` + AttachmentUids []string `json:"attachment_uids"` +} + type FullEmail struct { OdataContext string `json:"@odata.context"` OdataEtag string `json:"@odata.etag"` @@ -69,6 +104,19 @@ type FullEmail struct { Flag struct { Flagstatus string `json:"flagStatus"` } `json:"flag"` + Attachments []struct { + OdataType string `json:"@odata.type"` + OdataMediacontenttype string `json:"@odata.mediaContentType"` + ID string `json:"id"` + Lastmodifieddatetime time.Time `json:"lastModifiedDateTime"` + Name string `json:"name"` + Contenttype string `json:"contentType"` + Size int `json:"size"` + Isinline bool `json:"isInline"` + Contentid interface{} `json:"contentId"` + Contentlocation interface{} `json:"contentLocation"` + Contentbytes string `json:"contentBytes"` + } } type MailData struct { @@ -118,6 +166,47 @@ type OutlookFolders struct { Value []OutlookFolder `json:"value"` } +func getOutlookAttachment(client *http.Client, emailId, attachmentId string) ([]FullEmail, error) { + //requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/users/ec03b4f2-fccf-4c35-b0eb-be85a0f5dd43/mailFolders") + + requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/%s/attachments/%s", emailId, attachmentId) + //log.Printf("Outlook email URL: %#v", requestUrl) + + ret, err := client.Get(requestUrl) + if err != nil { + log.Printf("[INFO] OutlookErr: %s", err) + return []FullEmail{}, err + } + + body, err := ioutil.ReadAll(ret.Body) + if err != nil { + log.Printf("[WARNING] Failed body decoding from outlook email") + return []FullEmail{}, err + } + + //type FullEmail struct { + log.Printf("[INFO] Attachment Body: %s", string(body)) + log.Printf("[INFO] Status email: %d", ret.StatusCode) + if ret.StatusCode != 200 { + return []FullEmail{}, err + } + + //log.Printf("Body: %s", string(body)) + + /* + parsedmail := FullEmail{} + err = json.Unmarshal(body, &parsedmail) + if err != nil { + log.Printf("[INFO] Email unmarshal error: %s", err) + return []FullEmail{}, err + } + + emails = append(emails, parsedmail) + */ + + return []FullEmail{}, nil +} + func getOutlookEmail(client *http.Client, maildata MailData) ([]FullEmail, error) { //requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/users/ec03b4f2-fccf-4c35-b0eb-be85a0f5dd43/mailFolders") @@ -991,6 +1080,38 @@ func handleOutlookCallback(resp http.ResponseWriter, request *http.Request) { email = emails[0] } + // Parse indicators (domains, emails, ips, domains etc)! + newEmail := ParsedShuffleMail{} + newEmail.Body.ContentType = email.Body.Contenttype + newEmail.Body.Content = email.Body.Content + newEmail.Body.RawBody = email.Body.Content + + newEmail.Header.Subject = email.Subject + newEmail.Header.From = email.From.Emailaddress.Address + for _, to := range email.Torecipients { + newEmail.Header.To = append(newEmail.Header.To, to.Emailaddress.Address) + } + newEmail.Header.Date = email.Receiveddatetime.String() + + newEmail.MessageID = email.ID + + if email.Hasattachments { + log.Printf("SHOULD HANDLE ATTACHMENTS FOR EMAIL!") + + for _, attachment := range email.Attachments { + parsedAttachment, err := getOutlookAttachment(outlookClient, email.ID, attachment.ID) + if err != nil { + log.Printf("Failed attachment %s: %s", attachment.ID, err) + continue + } + + log.Printf("ATTACHMENT: %#v", parsedAttachment) + } + //log.Printf("%#v", attachments) + //log.Printf("%s", err) + //GET /users/{id | userPrincipalName}/events/{id}/attachments/{id} + } + emailBytes, err := json.Marshal(email) if err != nil { log.Printf("[INFO] Failed email marshaling: %s", err) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 16e1a653..7a20bec6 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -2565,6 +2565,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // Check every app action and param to see whether they exist //log.Printf("PRE ACTIONS 2") + allAuths, autherr := getAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) newActions = []Action{} for _, action := range workflow.Actions { reservedApps := []string{ @@ -2655,34 +2656,60 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // Check to see if the action is valid if curappaction.Name != action.Name { log.Printf("[ERROR] Action %s in app %s doesn't exist.", action.Name, curapp.Name) - if workflow.PreviouslySaved { - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Action %s in app %s doesn't exist"}`, action.Name, curapp.Name))) - return - } + thisError := fmt.Sprintf("%s: Action %s in app %s doesn't exist", action.Label, action.Name, action.AppName) + workflow.Errors = append(workflow.Errors, thisError) + workflow.IsValid = false + action.Errors = append(action.Errors, thisError) + action.IsValid = false + //if workflow.PreviouslySaved { + // resp.WriteHeader(401) + // resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Action %s in app %s doesn't exist"}`, action.Name, curapp.Name))) + // return + //} } // FIXME - check all parameters to see if they're valid // Includes checking required fields + selectedAuth := AppAuthenticationStorage{} + if len(action.AuthenticationId) > 0 && autherr == nil { + for _, auth := range allAuths { + if auth.Id == action.AuthenticationId { + selectedAuth = auth + break + } + } + } + newParams := []WorkflowAppActionParameter{} for _, param := range curappaction.Parameters { - found := false + paramFound := false // Handles check for parameter exists + value not empty in used fields for _, actionParam := range action.Parameters { if actionParam.Name == param.Name { - found = true + paramFound = true if actionParam.Value == "" && actionParam.Variant == "STATIC_VALUE" && actionParam.Required == true { - log.Printf("[WARNING] Appaction %s with required param '%s' is empty. Can't save.", action.Name, param.Name) - //if workflow.PreviouslySaved { - // resp.WriteHeader(401) - // resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s in app %s with required param '%s' is empty.", "node_id": "%s"}`, action.Name, action.AppName, param.Name, action.ID))) - // return - //} else { + // Validating if the field is an authentication field + if len(selectedAuth.Id) > 0 { + authFound := false + for _, field := range selectedAuth.Fields { + if field.Key == actionParam.Name { + authFound = true + //log.Printf("FOUND REQUIRED KEY %s IN AUTH", field.Key) + break + } + } - thisError := fmt.Sprintf("Missing parameter %s", param.Name) + if authFound { + newParams = append(newParams, actionParam) + continue + } + } + + log.Printf("[WARNING] Appaction %s with required param '%s' is empty. Can't save.", action.Name, param.Name) + thisError := fmt.Sprintf("%s is missing reqired parameter %s", action.Label, param.Name) action.Errors = append(action.Errors, thisError) workflow.Errors = append(workflow.Errors, thisError) action.IsValid = false @@ -2698,7 +2725,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } // Handles check for required params - if !found && param.Required { + if !paramFound && param.Required { log.Printf("Appaction %s with required param %s doesn't exist.", action.Name, param.Name) thisError := fmt.Sprintf("Parameter %s is required", param.Name) action.Errors = append(action.Errors, thisError) @@ -2719,13 +2746,10 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } } - //log.Printf("PRE SAVECHECK") if !workflow.PreviouslySaved { log.Printf("[WORKFLOW INIT] NOT PREVIOUSLY SAVED - SET ACTION AUTH!") - //AuthenticationId string `json:"authentication_id,omitempty" datastore:"authentication_id"` - allAuths, err := getAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) - if err == nil && len(workflowapps) > 0 && apperr == nil { + if autherr == nil && len(workflowapps) > 0 && apperr == nil { //log.Printf("Setting actions") actionFixing := []Action{} appsAdded := []string{} @@ -5389,7 +5413,7 @@ func getAppAuthentication(resp http.ResponseWriter, request *http.Request) { return } - // Cleanup for frontend + // Cleanup for frontend usage. User shouldn't be able to get the data. newAuth := []AppAuthenticationStorage{} for _, auth := range allAuths { newAuthField := auth diff --git a/docker-compose.yml b/docker-compose.yml index b26f3c1a..b96b2441 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: frontend: #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.63 + image: ghcr.io/frikky/shuffle-frontend:0.8.64 container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -17,7 +17,7 @@ services: - backend backend: #build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.63 + image: ghcr.io/frikky/shuffle-backend:0.8.64 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 9bb3ae21..40f9b948 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -204,8 +204,10 @@ const AngularWorkflow = (props) => { }) const [elements, setElements] = useState([]) + // No point going as fast, as the nodes aren't realtime anymore, but bulk updated. + // Set it from 2500 to 6000 to reduce overall load const { start, stop } = useInterval({ - duration: 2500, + duration: 6000, startImmediate: false, callback: () => { fetchUpdates() From 8148eb924c9a73c0cbf512c0639a4f3554e17cf4 Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 20 Mar 2021 16:16:53 +0100 Subject: [PATCH 43/57] Moved worker and orborus to run with shuffle-shared --- backend/app_sdk/app_base.py | 4 +- backend/go-app/main.go | 2 +- backend/go-app/walkoff.go | 14 +- frontend/src/defaultCytoscapeStyle.js | 9 +- functions/onprem/orborus/Dockerfile | 3 +- functions/onprem/orborus/build.sh | 2 +- functions/onprem/orborus/go.mod | 2 +- functions/onprem/orborus/go.sum | 383 ++++++++++++++++++++++++++ functions/onprem/orborus/orborus.go | 21 +- functions/onprem/worker/Dockerfile | 3 +- functions/onprem/worker/build.sh | 2 +- functions/onprem/worker/worker.go | 262 +++++++++--------- 12 files changed, 553 insertions(+), 154 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 4db578cb..c18be280 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -1208,7 +1208,9 @@ class AppBase: outercnt += 1 except KeyError as e: - print("Lower keyerror: %s" % e) + print("[INFO] Lower keyerror: %s" % e) + return "", False + #return basejson #return "KeyError: Couldn't find key: %s" % e diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 0d855263..abb13c8b 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3695,7 +3695,7 @@ func handleNewHook(resp http.ResponseWriter, request *http.Request) { log.Printf("[INFO] Failed to increase total workflows: %s", err) } - log.Printf("Set up a new hook with ID %s and environment %s", newId, hook.Environment) + log.Printf("[INFO] Set up a new hook with ID %s and environment %s", newId, hook.Environment) resp.WriteHeader(200) resp.Write([]byte(`{"success": true}`)) } diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 7a20bec6..456ecb4f 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -1158,6 +1158,13 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl resultLength := len(workflowExecution.Results) dbSave := false setExecution := true + + if actionResult.Action.ID == "" { + //log.Printf("[ERROR] Failed handling EMPTY action %#v", actionResult) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't handle empty action"}`))) + return + } //tx, err := dbclient.NewTransaction(ctx) //if err != nil { // log.Printf("client.NewTransaction: %v", err) @@ -1309,6 +1316,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // FIXME rebuild to be like this or something // workflowExecution/ExecutionId/Nodes/NodeId // Find the appropriate action + //log.Printf("[INFO] Setting value of %s in workflow %s to %s (1)", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status) if len(workflowExecution.Results) > 0 { // FIXME skip := false @@ -1344,14 +1352,14 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } } - log.Printf("[INFO] Updating %s in workflow %s from %s to %s", actionResult.Action.ID, workflowExecution.ExecutionId, workflowExecution.Results[outerindex].Status, actionResult.Status) + log.Printf("[INFO] Updating %s in workflow %s from %s to %s (3)", actionResult.Action.ID, workflowExecution.ExecutionId, workflowExecution.Results[outerindex].Status, actionResult.Status) workflowExecution.Results[outerindex] = actionResult } else { - log.Printf("[INFO] Setting value of %s in workflow %s to %s", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status) + log.Printf("[INFO] Setting value of %s in workflow %s to %s (1)", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status) workflowExecution.Results = append(workflowExecution.Results, actionResult) } } else { - log.Printf("[INFO] Setting value of %s in workflow %s to %s", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status) + log.Printf("[INFO] Setting value of %s in workflow %s to %s (2)", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status) workflowExecution.Results = append(workflowExecution.Results, actionResult) } diff --git a/frontend/src/defaultCytoscapeStyle.js b/frontend/src/defaultCytoscapeStyle.js index ab8fae1e..90c7aae8 100644 --- a/frontend/src/defaultCytoscapeStyle.js +++ b/frontend/src/defaultCytoscapeStyle.js @@ -6,7 +6,7 @@ const data = [{ 'font-family': 'Segoe UI, Tahoma, Geneva, Verdana, sans-serif, sans-serif', 'font-weight': 'lighter', 'margin-right': '10px', - 'font-size': '15px', + 'font-size': '18px', 'width': '80px', 'height': '80px', 'color': 'white', @@ -20,14 +20,15 @@ const data = [{ selector: 'edge', css: { 'target-arrow-shape': 'triangle', - 'target-arrow-color': 'yellow', + 'target-arrow-color': 'grey', 'curve-style': 'unbundled-bezier', 'label': 'data(label)', 'text-margin-y': '-15px', + 'width': '2px', "color": "white", "line-fill": "linear-gradient", - "line-gradient-stop-colors": ["cyan", "yellow"], "line-gradient-stop-positions": ["0.0", "100"], + "line-gradient-stop-colors": ["grey", "grey"], }, }, { @@ -183,7 +184,7 @@ const data = [{ css: { 'background-color': "#f85a3e", 'border-color': '#f85a3e', - 'border-width': '8px', + 'border-width': '12px', 'transition-property': 'border-width', 'transition-duration': '0.25s', }, diff --git a/functions/onprem/orborus/Dockerfile b/functions/onprem/orborus/Dockerfile index 7c178fd8..6479c09e 100644 --- a/functions/onprem/orborus/Dockerfile +++ b/functions/onprem/orborus/Dockerfile @@ -10,7 +10,8 @@ RUN go get github.com/docker/docker/api/types && \ go get github.com/docker/docker/client && \ go get github.com/mackerelio/go-osstat/cpu && \ go get github.com/mackerelio/go-osstat/memory && \ - go get github.com/satori/go.uuid + go get github.com/satori/go.uuid && \ + go get github.com/frikky/shuffle-shared RUN go build RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o orborus . diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index 1555b51f..2ea4d555 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-orborus -VERSION=0.8.63 +VERSION=0.8.64 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 4e17a529..f7d50e57 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -9,6 +9,7 @@ require ( github.com/docker/docker v20.10.1+incompatible github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect + github.com/frikky/shuffle-shared v0.0.12 // indirect github.com/gogo/protobuf v1.3.1 // indirect github.com/mackerelio/go-osstat v0.1.0 github.com/opencontainers/go-digest v1.0.0 // indirect @@ -16,5 +17,4 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/satori/go.uuid v1.2.0 github.com/sirupsen/logrus v1.7.0 // indirect - google.golang.org/grpc v1.34.0 // indirect ) diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index e8c720de..d3ea7e29 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -1,9 +1,54 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.66.0/go.mod h1:dgqGAjKCDxyhGTtC9dAREQGUJpkceNm1yt590Qno0Ko= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.75.0 h1:XgtDnVJRCPEUG21gjFiRPz4zI1Mjg16R+NYQjfmU4XY= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/datastore v1.4.0 h1:CFDJm15RpYXeEblQ0TMDUrYtqmBmbAWTy536nA8JIc8= +cloud.google.com/go/datastore v1.4.0/go.mod h1:d18825/a9bICdAIJy2EkHs9joU4RlIZ1t6l8WDdbdY0= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.12.0 h1:4y3gHptW1EHVtcPAVE0eBBlFuGqEejTTG3KdIE0lUX4= +cloud.google.com/go/storage v1.12.0/go.mod h1:fFLk2dp2oAhDz8QFKwqrjdJvxSp/W2g7nillojlL5Ho= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk= github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/containerd/containerd v1.4.3 h1:ijQT13JedHSHrQGWFcGEwzcNKrAGIiZ+jSD5QQG07SY= github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= @@ -19,14 +64,42 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/frikky/kin-openapi v0.38.0 h1:V7ttwIJS8Vks4KL+mZVj1ZSqhIcQtgaG8akeqXEQgsE= +github.com/frikky/kin-openapi v0.38.0/go.mod h1:Fr28TtCHL4K0kIqtqui8HWxN1LG5uAh3z/tDfFyiA1s= +github.com/frikky/shuffle-shared v0.0.12 h1:+0EIfThmK47Po+LogPYZR4XjbS4Ds19WNMFu2YUSjhw= +github.com/frikky/shuffle-shared v0.0.12/go.mod h1:SEY432/xs4oBkOUnGwxiYKCZWZLRaheGInhAoW7N8ww= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -35,17 +108,56 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200905233945-acf8798be1f7/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mackerelio/go-osstat v0.1.0 h1:e57QHeHob8kKJ5FhcXGdzx5O6Ktuc5RHMDIkeqhgkFA= github.com/mackerelio/go-osstat v0.1.0/go.mod h1:1K3NeYLhMHPvzUu+ePYXtoB58wkaRpxZsGClZBJyIFw= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= @@ -54,6 +166,7 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= @@ -62,47 +175,301 @@ github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b h1:iFwSg7t5GZmB/Q5TjiEAsdoLDrdJRC1RiF2WhuV29Qw= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210113160501-8b1d76fa0423 h1:/hEknzWkMPCjTo7StMHRrBRa8YBbXuBWfck8680k3RE= +golang.org/x/oauth2 v0.0.0-20210113160501-8b1d76fa0423/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190410235845-0ad05ae3009d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3 h1:kzM6+9dur93BcC2kVlYl34cHU+TYZLanmpSJHVMmL64= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20200915173823-2db8f0ff891c/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/tools v0.0.0-20200918232735-d647fc253266/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963 h1:K+NlvTLy0oONtRtkl1jRD9xIhnItbG2PiE7YOdjPb+k= +golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.31.0/go.mod h1:CL+9IBCa2WWU6gRuBWaKqGWLFFwbEUXkfeMkHLQWYWo= +google.golang.org/api v0.32.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0 h1:l2Nfbl2GPXdWorv+dT2XfinX2jOOw4zv1VhLstx+6rE= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200831141814-d751682dd103/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200914193844-75d14daec038/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200921151605-7abf4a1a14d5/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210113195801-ae06605f4595 h1:x7nk+/4+SvuTDI4wnzQUlhvi+DTpyfncXBo3QWTFs7U= +google.golang.org/genproto v0.0.0-20210113195801-ae06605f4595/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0 h1:raiipEjMOIC/TO2AvyTxP25XFdLxNIBwzDh3FM3XztI= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.34.1 h1:ugq+9++ZQPFzM2pKUMCIK8gj9M0pFyuUWO9Q8kwEDQw= +google.golang.org/grpc v1.34.1/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -111,9 +478,25 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 015d2986..ad508e8a 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -5,6 +5,8 @@ package main */ import ( + "github.com/frikky/shuffle-shared" + "bytes" "context" "encoding/json" @@ -55,19 +57,6 @@ var runningMode = strings.ToLower(os.Getenv("RUNNING_MODE")) var cleanupEnv = strings.ToLower(os.Getenv("CLEANUP")) var executionIds = []string{} -type ExecutionRequestWrapper struct { - Data []ExecutionRequest `json:"data"` -} - -type ExecutionRequest struct { - ExecutionId string `json:"execution_id"` - ExecutionArgument string `json:"execution_argument"` - WorkflowId string `json:"workflow_id"` - Authorization string `json:"authorization"` - Status string `json:"status"` - Type string `json:"type"` -} - var dockercli *dockerclient.Client var containerId string @@ -259,7 +248,7 @@ func initializeImages() { log.Printf("[WARNING] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %s", appSdkVersion) } if workerVersion == "" { - workerVersion = "0.8.62" + workerVersion = "0.8.64" log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %s", workerVersion) } @@ -477,7 +466,7 @@ func main() { continue } - var executionRequests ExecutionRequestWrapper + var executionRequests shuffle.ExecutionRequestWrapper err = json.Unmarshal(body, &executionRequests) if err != nil { log.Printf("[WARNING] Failed executionrequest in queue unmarshaling: %s", err) @@ -523,7 +512,7 @@ func main() { } // New, abortable version. Should check executionid and remove everything else - var toBeRemoved ExecutionRequestWrapper + var toBeRemoved shuffle.ExecutionRequestWrapper for _, execution := range executionRequests.Data { if len(execution.ExecutionArgument) > 0 { log.Printf("[INFO] Argument: %#v", execution.ExecutionArgument) diff --git a/functions/onprem/worker/Dockerfile b/functions/onprem/worker/Dockerfile index a9df5c2b..62aa2d0c 100644 --- a/functions/onprem/worker/Dockerfile +++ b/functions/onprem/worker/Dockerfile @@ -10,7 +10,8 @@ RUN go get github.com/docker/docker/api/types && \ go get github.com/docker/docker/api/types/container && \ go get github.com/docker/docker/client && \ go get github.com/gorilla/mux && \ - go get github.com/patrickmn/go-cache + go get github.com/patrickmn/go-cache && \ + go get github.com/frikky/shuffle-shared RUN go build RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker . diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh index 44d134c5..424b6207 100644 --- a/functions/onprem/worker/build.sh +++ b/functions/onprem/worker/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-worker -VERSION=0.8.63 +VERSION=0.8.64 echo "Running docker build with $NAME:$VERSION" #CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin . diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 0594b142..9966785a 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -1,6 +1,8 @@ package main import ( + "github.com/frikky/shuffle-shared" + "bytes" "context" "encoding/json" @@ -83,6 +85,7 @@ func init() { } } +/* type Userapi struct { Username string `datastore:"username"` ApiKey string `datastore:"apikey"` @@ -90,12 +93,12 @@ type Userapi struct { type ExecutionInfo struct { TotalApiUsage int64 `json:"total_api_usage" datastore:"total_api_usage"` - TotalWorkflowExecutions int64 `json:"total_workflow_executions" datastore:"total_workflow_executions"` + Totalshuffle.WorkflowExecutions int64 `json:"total_workflow_executions" datastore:"total_workflow_executions"` TotalAppExecutions int64 `json:"total_app_executions" datastore:"total_app_executions"` TotalCloudExecutions int64 `json:"total_cloud_executions" datastore:"total_cloud_executions"` TotalOnpremExecutions int64 `json:"total_onprem_executions" datastore:"total_onprem_executions"` DailyApiUsage int64 `json:"daily_api_usage" datastore:"daily_api_usage"` - DailyWorkflowExecutions int64 `json:"daily_workflow_executions" datastore:"daily_workflow_executions"` + Dailyshuffle.WorkflowExecutions int64 `json:"daily_workflow_executions" datastore:"daily_workflow_executions"` DailyAppExecutions int64 `json:"daily_app_executions" datastore:"daily_app_executions"` DailyCloudExecutions int64 `json:"daily_cloud_executions" datastore:"daily_cloud_executions"` DailyOnpremExecutions int64 `json:"daily_onprem_executions" datastore:"daily_onprem_executions"` @@ -112,6 +115,7 @@ type StatisticsItem struct { Fieldname string `json:"field_name" datastore:"field_name"` Data []StatisticsData `json:"data" datastore:"data"` } +*/ // "Execution by status" // Execution history @@ -130,6 +134,7 @@ type StatisticsItem struct { // Baseline map[string]int64 `json:"baseline" datastore:"baseline"` //} +/* type ParsedOpenApi struct { Body string `datastore:"body,noindex" json:"body"` ID string `datastore:"id" json:"id"` @@ -139,7 +144,7 @@ type ParsedOpenApi struct { // Limits set for a user so that they can't do a shitload type UserLimits struct { DailyApiUsage int64 `json:"daily_api_usage" datastore:"daily_api_usage"` - DailyWorkflowExecutions int64 `json:"daily_workflow_executions" datastore:"daily_workflow_executions"` + Dailyshuffle.WorkflowExecutions int64 `json:"daily_workflow_executions" datastore:"daily_workflow_executions"` DailyCloudExecutions int64 `json:"daily_cloud_executions" datastore:"daily_cloud_executions"` DailyTriggers int64 `json:"daily_triggers" datastore:"daily_triggers"` DailyMailUsage int64 `json:"daily_mail_usage" datastore:"daily_mail_usage"` @@ -380,7 +385,7 @@ type Info struct { Description string `json:"description" datastore:"description,noindex"` } -// Actions to be done by webhooks etc +// shuffle.Actions to be done by webhooks etc // Field is the actual field to use from json type HookAction struct { Type string `json:"type" datastore:"type"` @@ -425,7 +430,7 @@ type SyncFeatures struct { Notifications SyncData `json:"notifications" datastore:"notifications"` EmailTrigger SyncData `json:"email_trigger" datastore:"email_trigger"` AppExecutions SyncData `json:"app_executions" datastore:"app_executions"` - WorkflowExecutions SyncData `json:"workflow_executions" datastore:"workflow_executions"` + shuffle.WorkflowExecutions SyncData `json:"workflow_executions" datastore:"workflow_executions"` Apps SyncData `json:"apps" datastore:"apps"` Workflows SyncData `json:"workflows" datastore:"workflows"` Autocomplete SyncData `json:"autocomplete" datastore:"autocomplete"` @@ -520,7 +525,7 @@ type WorkflowApp struct { LastRuntime int64 `json:"last_runtime" datastore:"last_runtime"` } -type WorkflowAppActionParameter struct { +type shuffle.WorkflowAppActionParameter struct { Description string `json:"description" datastore:"description,noindex" yaml:"description"` ID string `json:"id" datastore:"id" yaml:"id,omitempty"` Name string `json:"name" datastore:"name" yaml:"name"` @@ -561,7 +566,7 @@ type WorkflowAppAction struct { Tags []string `json:"tags" datastore:"tags" yaml:"tags"` Authentication []AuthenticationStore `json:"authentication" datastore:"authentication,noindex" yaml:"authentication,omitempty"` Tested bool `json:"tested" datastore:"tested" yaml:"tested"` - Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters"` + Parameters []shuffle.WorkflowAppActionParameter `json:"parameters" datastore: "parameters"` ExecutionVariable struct { Description string `json:"description" datastore:"description,noindex"` ID string `json:"id" datastore:"id"` @@ -579,7 +584,7 @@ type WorkflowAppAction struct { AuthNotRequired bool `json:"auth_not_required" datastore:"auth_not_required" yaml:"auth_not_required"` } -type WorkflowExecution struct { +type shuffle.WorkflowExecution struct { Type string `json:"type" datastore:"type"` Status string `json:"status" datastore:"status"` Start string `json:"start" datastore:"start"` @@ -607,7 +612,7 @@ type WorkflowExecution struct { OrgId string `json:"org_id" datastore:"org_id"` } -type Action struct { +type shuffle.Action struct { AppName string `json:"app_name,omitempty" datastore:"app_name"` AppVersion string `json:"app_version,omitempty" datastore:"app_version"` AppID string `json:"app_id,omitempty" datastore:"app_id"` @@ -622,7 +627,7 @@ type Action struct { LargeImage string `json:"large_image,omitempty" datastore:"large_image,noindex" yaml:"large_image" required:false` Environment string `json:"environment,omitempty" datastore:"environment"` Name string `json:"name,omitempty" datastore:"name"` - Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"` + Parameters []shuffle.WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"` ExecutionVariable struct { Description string `json:"description,omitempty" datastore:"description,noindex"` ID string `json:"id,omitempty" datastore:"id"` @@ -640,7 +645,7 @@ type Action struct { } // Added environment for location to execute -type Trigger struct { +type shuffle.Trigger struct { AppName string `json:"app_name" datastore:"app_name"` Description string `json:"description" datastore:"description,noindex"` LongDescription string `json:"long_description" datastore:"long_description"` @@ -657,7 +662,7 @@ type Trigger struct { TriggerType string `json:"trigger_type" datastore:"trigger_type"` Name string `json:"name" datastore:"name"` Tags []string `json:"tags" datastore:"tags" yaml:"tags"` - Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"` + Parameters []shuffle.WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"` Position struct { X float64 `json:"x" datastore:"x"` Y float64 `json:"y" datastore:"y"` @@ -676,9 +681,9 @@ type Branch struct { // Same format for a lot of stuff type Condition struct { - Condition WorkflowAppActionParameter `json:"condition" datastore:"condition"` - Source WorkflowAppActionParameter `json:"source" datastore:"source"` - Destination WorkflowAppActionParameter `json:"destination" datastore:"destination"` + Condition shuffle.WorkflowAppActionParameter `json:"condition" datastore:"condition"` + Source shuffle.WorkflowAppActionParameter `json:"source" datastore:"source"` + Destination shuffle.WorkflowAppActionParameter `json:"destination" datastore:"destination"` } type Schedule struct { @@ -729,8 +734,8 @@ type Workflow struct { ExecutionEnvironment string `json:"execution_environment" datastore:"execution_environment"` } -type ActionResult struct { - Action Action `json:"action" datastore:"action,noindex"` +type shuffle.ActionResult struct { + Action shuffle.Action `json:"action" datastore:"action,noindex"` ExecutionId string `json:"execution_id" datastore:"execution_id"` Authorization string `json:"authorization" datastore:"authorization"` Result string `json:"result" datastore:"result,noindex"` @@ -775,9 +780,10 @@ type AppExecutionExample struct { SuccessExamples []string `json:"success_examples" datastore:"success_examples,noindex"` FailureExamples []string `json:"failure_examples" datastore:"failure_examples,noindex"` } +*/ // removes every container except itself (worker) -func shutdown(workflowExecution WorkflowExecution, nodeId string, reason string, handleResultSend bool) { +func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason string, handleResultSend bool) { log.Printf("[INFO] Shutdown (%s) started with reason %s", workflowExecution.Status, reason) //reason := "Error in execution" @@ -881,7 +887,7 @@ func shutdown(workflowExecution WorkflowExecution, nodeId string, reason string, } // Deploys the internal worker whenever something happens -func deployApp(cli *dockerclient.Client, image string, identifier string, env []string) error { +func deployApp(cli *dockerclient.Client, image string, identifier string, env []string, workflowExecution shuffle.WorkflowExecution) error { // form basic hostConfig ctx := context.Background() hostConfig := &container.HostConfig{ @@ -968,47 +974,51 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] log.Printf("[INFO] Container %s was created for %s", cont.ID, identifier) // Waiting to see if it exits.. Stupid, but stable(r) - time.Sleep(2 * time.Second) + if workflowExecution.ExecutionSource != "default" { + log.Printf("Handling NON-default execution source %s - NOT waiting and validating!", workflowExecution.ExecutionSource) + } else if workflowExecution.ExecutionSource == "default" { + time.Sleep(2 * time.Second) - stats, err := cli.ContainerInspect(ctx, cont.ID) - if err != nil { - log.Printf("[ERROR] Failed getting container stats") - } else { - //log.Printf("[INFO] Info for container: %#v", stats) - //log.Printf("%#v", stats.Config) - //log.Printf("%#v", stats.ContainerJSONBase.State) - log.Printf("[INFO] EXECUTION STATUS: %s", stats.ContainerJSONBase.State.Status) - if stats.ContainerJSONBase.State.Status == "exited" { - logOptions := types.ContainerLogsOptions{ - ShowStdout: true, + stats, err := cli.ContainerInspect(ctx, cont.ID) + if err != nil { + log.Printf("[ERROR] Failed getting container stats") + } else { + //log.Printf("[INFO] Info for container: %#v", stats) + //log.Printf("%#v", stats.Config) + //log.Printf("%#v", stats.ContainerJSONBase.State) + log.Printf("[INFO] EXECUTION STATUS: %s", stats.ContainerJSONBase.State.Status) + if stats.ContainerJSONBase.State.Status == "exited" { + logOptions := types.ContainerLogsOptions{ + ShowStdout: true, + } + + out, err := cli.ContainerLogs(ctx, cont.ID, logOptions) + if err != nil { + log.Printf("[INFO] Failed getting logs: %s", err) + } else { + log.Printf("IN ELSE FOR DEPLOY") + buf := new(strings.Builder) + io.Copy(buf, out) + logs := buf.String() + log.Printf("Logs: %s", logs) + + //log.Printf(logs) + // check errors + /* + if strings.Contains(logs, "Error") { + log.Printf("ERROR IN %s?", cont.ID) + log.Println(logs) + //return errors.New(fmt.Sprintf("ERROR FROM CONTAINER %s", cont.ID)) + } else { + log.Printf("NORMAL EXEC OF %s?", cont.ID) + } + */ + } + + log.Printf("ERROR IN CONTAINER DEPLOYMENT - ITS EXITED!") + + return errors.New(fmt.Sprintf(`{"success": false, "reason": "Container %s exited prematurely.","debug": "docker logs -f %s"}`, cont.ID, cont.ID)) } - - out, err := cli.ContainerLogs(ctx, cont.ID, logOptions) - if err != nil { - log.Printf("[INFO] Failed getting logs: %s", err) - } else { - log.Printf("IN ELSE FOR DEPLOY") - buf := new(strings.Builder) - io.Copy(buf, out) - logs := buf.String() - log.Printf("Logs: %s", logs) - - //log.Printf(logs) - // check errors - /* - if strings.Contains(logs, "Error") { - log.Printf("ERROR IN %s?", cont.ID) - log.Println(logs) - //return errors.New(fmt.Sprintf("ERROR FROM CONTAINER %s", cont.ID)) - } else { - log.Printf("NORMAL EXEC OF %s?", cont.ID) - } - */ - } - - log.Printf("ERROR IN CONTAINER DEPLOYMENT - ITS EXITED!") - - return errors.New(fmt.Sprintf(`{"success": false, "reason": "Container %s exited prematurely.","debug": "docker logs -f %s"}`, cont.ID, cont.ID)) } } @@ -1072,7 +1082,7 @@ func removeContainer(containername string) error { return nil } -func runFilter(workflowExecution WorkflowExecution, action Action) { +func runFilter(workflowExecution shuffle.WorkflowExecution, action shuffle.Action) { // 1. Get the parameter $.#.id if action.Label == "filter_cases" && len(action.Parameters) > 0 { if action.Parameters[0].Variant == "ACTION_RESULT" { @@ -1088,7 +1098,7 @@ func runFilter(workflowExecution WorkflowExecution, action Action) { } -func handleSubworkflowExecution(client *http.Client, workflowExecution WorkflowExecution, action Trigger, baseAction Action) error { +func handleSubworkflowExecution(client *http.Client, workflowExecution shuffle.WorkflowExecution, action shuffle.Trigger, baseAction shuffle.Action) error { apikey := "" workflowId := "" executionArgument := "" @@ -1140,14 +1150,14 @@ func handleSubworkflowExecution(client *http.Client, workflowExecution WorkflowE } timeNow := time.Now().Unix() - //curaction := Action{ + //curaction := shuffle.Action{ // AppName: baseAction.AppName, // AppVersion: baseAction.AppVersion, // Label: baseAction.Label, // Name: baseAction.Name, // ID: baseAction.ID, //} - result := ActionResult{ + result := shuffle.ActionResult{ Action: baseAction, ExecutionId: workflowExecution.ExecutionId, Authorization: workflowExecution.Authorization, @@ -1200,7 +1210,7 @@ func removeIndex(s []string, i int) []string { return s[:len(s)-1] } -func handleExecutionResult(workflowExecution WorkflowExecution) { +func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { if len(startAction) == 0 { startAction = workflowExecution.Start if len(startAction) == 0 { @@ -1284,7 +1294,7 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { // care if it gets stuck in a loop. // FIXME: Force killing a worker should result in a notification somewhere if len(nextActions) == 0 { - log.Printf("[INFO] No next action. Finished? Result vs Actions: %d - %d", len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)) + log.Printf("[INFO] No next action. Finished? Result vs shuffle.Actions: %d - %d", len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)) exit := true for _, item := range workflowExecution.Results { if item.Status == "EXECUTING" { @@ -1401,7 +1411,7 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { //visited = append(visited, action.ID) //executed = append(executed, action.ID) - trigger := Trigger{} + trigger := shuffle.Trigger{} for _, innertrigger := range workflowExecution.Workflow.Triggers { if innertrigger.ID == action.ID { trigger = innertrigger @@ -1410,18 +1420,18 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { } // FIXME: Add startnode from frontend - action.Parameters = []WorkflowAppActionParameter{} + action.Parameters = []shuffle.WorkflowAppActionParameter{} for _, parameter := range trigger.Parameters { parameter.Variant = "STATIC_VALUE" action.Parameters = append(action.Parameters, parameter) } - action.Parameters = append(action.Parameters, WorkflowAppActionParameter{ + action.Parameters = append(action.Parameters, shuffle.WorkflowAppActionParameter{ Name: "source_workflow", Value: workflowExecution.Workflow.ID, }) - action.Parameters = append(action.Parameters, WorkflowAppActionParameter{ + action.Parameters = append(action.Parameters, shuffle.WorkflowAppActionParameter{ Name: "source_execution", Value: workflowExecution.ExecutionId, }) @@ -1444,7 +1454,7 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { continue } else { log.Printf("Should stop after this iteration because it's user-input based. %#v", action) - trigger := Trigger{} + trigger := shuffle.Trigger{} for _, innertrigger := range workflowExecution.Workflow.Triggers { if innertrigger.ID == action.ID { trigger = innertrigger @@ -1573,7 +1583,7 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { } if len(action.Parameters) == 0 { - action.Parameters = []WorkflowAppActionParameter{} + action.Parameters = []shuffle.WorkflowAppActionParameter{} } if len(action.Errors) == 0 { @@ -1639,7 +1649,7 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { // If cleanup is set, it should run for efficiency pullOptions := types.ImagePullOptions{} if cleanupEnv == "true" { - err = deployApp(dockercli, images[0], identifier, env) + err = deployApp(dockercli, images[0], identifier, env, workflowExecution) if err != nil { if strings.Contains(err.Error(), "exited prematurely") { shutdown(workflowExecution, action.ID, err.Error(), true) @@ -1666,7 +1676,7 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { log.Printf("[INFO] Successfully downloaded %s", image) } - err = deployApp(dockercli, image, identifier, env) + err = deployApp(dockercli, image, identifier, env, workflowExecution) if err != nil { log.Printf("[ERROR] Failed deploying image for the FOURTH time. Aborting if the image doesn't exist") @@ -1683,7 +1693,7 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { } } else { - err = deployApp(dockercli, image, identifier, env) + err = deployApp(dockercli, image, identifier, env, workflowExecution) if err != nil { if strings.Contains(err.Error(), "exited prematurely") { shutdown(workflowExecution, action.ID, err.Error(), true) @@ -1696,7 +1706,7 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { image = strings.ReplaceAll(image, " ", "-") } - err = deployApp(dockercli, image, identifier, env) + err = deployApp(dockercli, image, identifier, env, workflowExecution) if err != nil { if strings.Contains(err.Error(), "exited prematurely") { shutdown(workflowExecution, action.ID, err.Error(), true) @@ -1707,7 +1717,7 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { image = strings.ReplaceAll(image, " ", "-") } - err = deployApp(dockercli, image, identifier, env) + err = deployApp(dockercli, image, identifier, env, workflowExecution) if err != nil { if strings.Contains(err.Error(), "exited prematurely") { shutdown(workflowExecution, action.ID, err.Error(), true) @@ -1734,7 +1744,7 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { log.Printf("[INFO] Successfully downloaded %s", image) } - err = deployApp(dockercli, image, identifier, env) + err = deployApp(dockercli, image, identifier, env, workflowExecution) if err != nil { log.Printf("[ERROR] Failed deploying image for the FOURTH time. Aborting if the image doesn't exist") if strings.Contains(err.Error(), "exited prematurely") { @@ -1792,7 +1802,7 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { return } -func executionInit(workflowExecution WorkflowExecution) error { +func executionInit(workflowExecution shuffle.WorkflowExecution) error { parents = map[string][]string{} children = map[string][]string{} @@ -1834,10 +1844,10 @@ func executionInit(workflowExecution WorkflowExecution) error { } if trigger.ID == branch.SourceID { - log.Printf("[INFO] Trigger %s is the source!", trigger.AppName) + log.Printf("[INFO] shuffle.Trigger %s is the source!", trigger.AppName) sourceFound = true } else if trigger.ID == branch.DestinationID { - log.Printf("[INFO] Trigger %s is the destination!", trigger.AppName) + log.Printf("[INFO] shuffle.Trigger %s is the destination!", trigger.AppName) destinationFound = true } } @@ -1862,7 +1872,7 @@ func executionInit(workflowExecution WorkflowExecution) error { log.Printf("[INFO] NEXT ACTIONS: %#v\n\n", nextActions) */ - log.Printf("[INFO] Actions: %d + Special Triggers: %d", len(workflowExecution.Workflow.Actions), extra) + log.Printf("[INFO] shuffle.Actions: %d + Special shuffle.Triggers: %d", len(workflowExecution.Workflow.Actions), extra) onpremApps := []string{} toExecuteOnprem := []string{} for _, action := range workflowExecution.Workflow.Actions { @@ -1913,7 +1923,7 @@ func executionInit(workflowExecution WorkflowExecution) error { return nil } -func handleExecution(client *http.Client, req *http.Request, workflowExecution WorkflowExecution) error { +func handleExecution(client *http.Client, req *http.Request, workflowExecution shuffle.WorkflowExecution) error { // if no onprem runs (shouldn't happen, but extra check), exit // if there are some, load the images ASAP for the app @@ -1999,17 +2009,17 @@ func arrayContains(visited []string, id string) bool { return found } -func getResult(workflowExecution WorkflowExecution, id string) ActionResult { +func getResult(workflowExecution shuffle.WorkflowExecution, id string) shuffle.ActionResult { for _, actionResult := range workflowExecution.Results { if actionResult.Action.ID == id { return actionResult } } - return ActionResult{} + return shuffle.ActionResult{} } -func getAction(workflowExecution WorkflowExecution, id, environment string) Action { +func getAction(workflowExecution shuffle.WorkflowExecution, id, environment string) shuffle.Action { for _, action := range workflowExecution.Workflow.Actions { if action.ID == id { return action @@ -2018,7 +2028,7 @@ func getAction(workflowExecution WorkflowExecution, id, environment string) Acti for _, trigger := range workflowExecution.Workflow.Triggers { if trigger.ID == id { - return Action{ + return shuffle.Action{ ID: trigger.ID, AppName: trigger.AppName, Name: trigger.AppName, @@ -2029,12 +2039,12 @@ func getAction(workflowExecution WorkflowExecution, id, environment string) Acti } } - return Action{} + return shuffle.Action{} } -func runUserInput(client *http.Client, action Action, workflowId, workflowExecutionId, authorization string, configuration string) error { +func runUserInput(client *http.Client, action shuffle.Action, workflowId, workflowExecutionId, authorization string, configuration string) error { timeNow := time.Now().Unix() - result := ActionResult{ + result := shuffle.ActionResult{ Action: action, ExecutionId: workflowExecutionId, Authorization: authorization, @@ -2104,7 +2114,7 @@ func runTestExecution(client *http.Client, workflowId, apikey string) (string, s } log.Printf("[INFO] Test Body: %s", string(body)) - var workflowExecution WorkflowExecution + var workflowExecution shuffle.WorkflowExecution err = json.Unmarshal(body, &workflowExecution) if err != nil { log.Printf("Failed workflowExecution unmarshal: %s", err) @@ -2124,17 +2134,17 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { } //log.Printf("Got result: %s", string(body)) - var actionResult ActionResult + var actionResult shuffle.ActionResult err = json.Unmarshal(body, &actionResult) if err != nil { - log.Printf("Failed ActionResult unmarshaling: %s", err) + log.Printf("Failed shuffle.ActionResult unmarshaling: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } - // 1. Get the WorkflowExecution(ExecutionId) from the database - // 2. if ActionResult.Authentication != WorkflowExecution.Authentication -> exit + // 1. Get the shuffle.WorkflowExecution(ExecutionId) from the database + // 2. if shuffle.ActionResult.Authentication != shuffle.WorkflowExecution.Authentication -> exit // 3. Add to and update actionResult in workflowExecution // 4. Push to db // IF FAIL: Set executionstatus: abort or cancel @@ -2179,7 +2189,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { //if actionResult.Status == "WAITING" && actionResult.Action.AppName == "User Input" { // log.Printf("SHOULD WAIT A BIT AND RUN CLOUD STUFF WITH USER INPUT! WAITING!") - // var trigger Trigger + // var trigger shuffle.Trigger // err = json.Unmarshal([]byte(actionResult.Result), &trigger) // if err != nil { // log.Printf("Failed unmarshaling actionresult for user input: %s", err) @@ -2199,7 +2209,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { // actionResult.Result = fmt.Sprintf("Cloud error: %s", err) // workflowExecution.Results = append(workflowExecution.Results, actionResult) // workflowExecution.Status = "ABORTED" - // err = setWorkflowExecution(ctx, *workflowExecution, true) + // err = setshuffle.WorkflowExecution(ctx, *workflowExecution, true) // if err != nil { // log.Printf("Failed ") // } else { @@ -2217,7 +2227,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { // workflowExecution.Results = append(workflowExecution.Results, actionResult) // workflowExecution.Status = actionResult.Status - // err = setWorkflowExecution(ctx, *workflowExecution, true) + // err = setshuffle.WorkflowExecution(ctx, *workflowExecution, true) // if err != nil { // log.Printf("Failed ") // } else { @@ -2234,7 +2244,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { } -func findChildNodes(workflowExecution WorkflowExecution, nodeId string) []string { +func findChildNodes(workflowExecution shuffle.WorkflowExecution, nodeId string) []string { //log.Printf("\nNODE TO FIX: %s\n\n", nodeId) allChildren := []string{nodeId} @@ -2282,7 +2292,7 @@ func findChildNodes(workflowExecution WorkflowExecution, nodeId string) []string } // Will make sure transactions are always ran for an execution. This is recursive if it fails. Allowed to fail up to 5 times -func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workflowExecutionId string, actionResult ActionResult, resp http.ResponseWriter) { +func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workflowExecutionId string, actionResult shuffle.ActionResult, resp http.ResponseWriter) { //log.Printf("IN WORKFLOWEXECUTION SUB!") // Should start a tx for the execution here workflowExecution, err := getWorkflowExecution(ctx, workflowExecutionId) @@ -2306,7 +2316,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl //} //key := datastore.NameKey("workflowexecution", workflowExecutionId, nil) - //workflowExecution := &WorkflowExecution{} + //workflowExecution := &shuffle.WorkflowExecution{} //if err := tx.Get(key, workflowExecution); err != nil { // log.Printf("[ERROR] tx.Get bug: %v", err) // tx.Rollback() @@ -2314,7 +2324,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting the workflow key"}`))) // return //} - actionResult.Action = Action{ + actionResult.Action = shuffle.Action{ AppName: actionResult.Action.AppName, AppVersion: actionResult.Action.AppVersion, Label: actionResult.Action.Label, @@ -2326,15 +2336,15 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl if actionResult.Status == "ABORTED" || actionResult.Status == "FAILURE" { //dbSave = true - newResults := []ActionResult{} + newResults := []shuffle.ActionResult{} childNodes := []string{} if workflowExecution.Workflow.Configuration.ExitOnError { - log.Printf("[WARNING] Actionresult is %s for node %s in %s. Should set workflowExecution and exit all running functions", actionResult.Status, actionResult.Action.ID, workflowExecution.ExecutionId) + log.Printf("[WARNING] shuffle.Actionresult is %s for node %s in %s. Should set workflowExecution and exit all running functions", actionResult.Status, actionResult.Action.ID, workflowExecution.ExecutionId) workflowExecution.Status = actionResult.Status workflowExecution.LastNode = actionResult.Action.ID // Find underlying nodes and add them } else { - log.Printf("[WARNING] Actionresult is %s for node %s in %s. Continuing anyway because of workflow configuration.", actionResult.Status, actionResult.Action.ID, workflowExecution.ExecutionId) + log.Printf("[WARNING] shuffle.Actionresult is %s for node %s in %s. Continuing anyway because of workflow configuration.", actionResult.Status, actionResult.Action.ID, workflowExecution.ExecutionId) // Finds ALL childnodes to set them to SKIPPED // Remove duplicates //log.Printf("CHILD NODES: %d", len(childNodes)) @@ -2346,7 +2356,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // 1. Find the action itself // 2. Create an actionresult - curAction := Action{ID: ""} + curAction := shuffle.Action{ID: ""} for _, action := range workflowExecution.Workflow.Actions { if action.ID == nodeId { curAction = action @@ -2396,7 +2406,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } if !skipNodeAdd { - newResult := ActionResult{ + newResult := shuffle.ActionResult{ Action: curAction, ExecutionId: actionResult.ExecutionId, Authorization: actionResult.Authorization, @@ -2419,7 +2429,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // Cleans up aborted, and always gives a result lastResult := "" - // type ActionResult struct { + // type shuffle.ActionResult struct { for _, result := range workflowExecution.Results { if actionResult.Action.ID == result.Action.ID { continue @@ -2501,7 +2511,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // 1. Find the action itself // 2. Create an actionresult - curAction := Action{ID: ""} + curAction := shuffle.Action{ID: ""} for _, action := range workflowExecution.Workflow.Actions { if action.ID == nodeId { curAction = action @@ -2545,7 +2555,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } if !skipNodeAdd { - newAction := Action{ + newAction := shuffle.Action{ AppName: curAction.AppName, AppVersion: curAction.AppVersion, Label: curAction.Label, @@ -2553,7 +2563,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl ID: curAction.ID, } - newResult := ActionResult{ + newResult := shuffle.ActionResult{ Action: newAction, ExecutionId: actionResult.ExecutionId, Authorization: actionResult.Authorization, @@ -2688,7 +2698,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // Result string `json:"result" datastore:"result,noindex"` // Arbitrary reduction size maxSize := 500000 - newResults := []ActionResult{} + newResults := []shuffle.ActionResult{} for _, item := range workflowExecution.Results { if len(item.Result) > maxSize { item.Result = "[ERROR] Result too large to handle (https://github.com/frikky/shuffle/issues/171)" @@ -2705,7 +2715,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // Handled using cachhing, so actually pretty fast cacheKey := fmt.Sprintf("workflowexecution-%s", workflowExecution.ExecutionId) if value, found := requestCache.Get(cacheKey); found { - parsedValue := value.(*WorkflowExecution) + parsedValue := value.(*shuffle.WorkflowExecution) if len(parsedValue.Results) > 0 && len(parsedValue.Results) != resultLength { setExecution = false if attempts > 5 { @@ -2731,7 +2741,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl log.Printf("[INFO] Skipping setexec with status %s", workflowExecution.Status) // Just in case. Should MAYBE validate finishing another time as well. - // This fixes issues with e.g. Action -> Trigger -> Action. + // This fixes issues with e.g. shuffle.Action -> shuffle.Trigger -> shuffle.Action. handleExecutionResult(*workflowExecution) //validateFinished(workflowExecution) } @@ -2744,21 +2754,21 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl //resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } -func getWorkflowExecution(ctx context.Context, id string) (*WorkflowExecution, error) { +func getWorkflowExecution(ctx context.Context, id string) (*shuffle.WorkflowExecution, error) { //log.Printf("IN GET WORKFLOW EXEC!") cacheKey := fmt.Sprintf("workflowexecution-%s", id) if value, found := requestCache.Get(cacheKey); found { - parsedValue := value.(*WorkflowExecution) + parsedValue := value.(*shuffle.WorkflowExecution) //log.Printf("Found execution for id %s with %d results", parsedValue.ExecutionId, len(parsedValue.Results)) //validateFinished(*parsedValue) return parsedValue, nil } - return &WorkflowExecution{}, errors.New("No workflowexecution defined yet") + return &shuffle.WorkflowExecution{}, errors.New("No workflowexecution defined yet") } -func sendResult(workflowExecution WorkflowExecution, data []byte) { +func sendResult(workflowExecution shuffle.WorkflowExecution, data []byte) { fullUrl := fmt.Sprintf("%s/api/v1/streams", baseUrl) req, err := http.NewRequest( "POST", @@ -2786,8 +2796,8 @@ func sendResult(workflowExecution WorkflowExecution, data []byte) { } } -func validateFinished(workflowExecution WorkflowExecution) { - log.Printf("[INFO] VALIDATION. Status: %s, Actions: %d, Extra: %d, Results: %d\n", workflowExecution.Status, len(workflowExecution.Workflow.Actions), extra, len(workflowExecution.Results)) +func validateFinished(workflowExecution shuffle.WorkflowExecution) { + log.Printf("[INFO] VALIDATION. Status: %s, shuffle.Actions: %d, Extra: %d, Results: %d\n", workflowExecution.Status, len(workflowExecution.Workflow.Actions), extra, len(workflowExecution.Results)) //if len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions)+extra { if (len(environments) == 1 && requestsSent == 0 && len(workflowExecution.Results) >= 1) || (len(workflowExecution.Results) >= len(workflowExecution.Workflow.Actions) && len(workflowExecution.Workflow.Actions) > 0) { @@ -2814,10 +2824,10 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { return } - var actionResult ActionResult + var actionResult shuffle.ActionResult err = json.Unmarshal(body, &actionResult) if err != nil { - log.Printf("Failed ActionResult unmarshaling: %s", err) + log.Printf("Failed shuffle.ActionResult unmarshaling: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return @@ -2852,7 +2862,7 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { } -func setWorkflowExecution(ctx context.Context, workflowExecution WorkflowExecution, dbSave bool) error { +func setWorkflowExecution(ctx context.Context, workflowExecution shuffle.WorkflowExecution, dbSave bool) error { //log.Printf("IN SET WORKFLOW EXEC!") //log.Printf("\n\n\nRESULT: %s\n\n\n", workflowExecution.Status) if len(workflowExecution.ExecutionId) == 0 { @@ -2901,7 +2911,7 @@ func getAvailablePort() (net.Listener, error) { //return fmt.Sprintf(":%d", port) } -func webserverSetup(workflowExecution WorkflowExecution) net.Listener { +func webserverSetup(workflowExecution shuffle.WorkflowExecution) net.Listener { hostname := getLocalIP() // FIXME: This MAY not work because of speed between first @@ -2972,7 +2982,7 @@ func main() { log.Printf("[INFO] Running normal execution with auth %s and ID %s", authorization, executionId) } - workflowExecution := WorkflowExecution{ + workflowExecution := shuffle.WorkflowExecution{ ExecutionId: executionId, } if len(authorization) == 0 { @@ -3052,7 +3062,8 @@ func main() { } log.Printf("Environments: %s. 1 = webserver, 0 or >1 = default", environments) - if len(environments) == 1 { //&& len(workflowExecution.Actions)+len(workflowExecution.Triggers) > 1 { + if len(environments) == 1 { //&& workflowExecution.ExecutionSource != "default" { + log.Printf("[INFO] Running OPTIMIZED execution (not manual)") listener := webserverSetup(workflowExecution) err := executionInit(workflowExecution) if err != nil { @@ -3070,6 +3081,9 @@ func main() { //wg := sync.WaitGroup{} //wg.Add(1) //wg.Wait() + } else { + log.Printf("[INFO] Running NON-OPTIMIZED execution for type %s with %d environments", workflowExecution.ExecutionSource, len(environments)) + } } From 4f49a87a21da4d023a511406c71896dda0e360cc Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 20 Mar 2021 16:17:28 +0100 Subject: [PATCH 44/57] Removed unused structs from worker --- functions/onprem/worker/worker.go | 697 ------------------------------ 1 file changed, 697 deletions(-) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 9966785a..c2b5e5d9 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -85,703 +85,6 @@ func init() { } } -/* -type Userapi struct { - Username string `datastore:"username"` - ApiKey string `datastore:"apikey"` -} - -type ExecutionInfo struct { - TotalApiUsage int64 `json:"total_api_usage" datastore:"total_api_usage"` - Totalshuffle.WorkflowExecutions int64 `json:"total_workflow_executions" datastore:"total_workflow_executions"` - TotalAppExecutions int64 `json:"total_app_executions" datastore:"total_app_executions"` - TotalCloudExecutions int64 `json:"total_cloud_executions" datastore:"total_cloud_executions"` - TotalOnpremExecutions int64 `json:"total_onprem_executions" datastore:"total_onprem_executions"` - DailyApiUsage int64 `json:"daily_api_usage" datastore:"daily_api_usage"` - Dailyshuffle.WorkflowExecutions int64 `json:"daily_workflow_executions" datastore:"daily_workflow_executions"` - DailyAppExecutions int64 `json:"daily_app_executions" datastore:"daily_app_executions"` - DailyCloudExecutions int64 `json:"daily_cloud_executions" datastore:"daily_cloud_executions"` - DailyOnpremExecutions int64 `json:"daily_onprem_executions" datastore:"daily_onprem_executions"` -} - -type StatisticsData struct { - Timestamp int64 `json:"timestamp" datastore:"timestamp"` - Id string `json:"id" datastore:"id"` - Amount int64 `json:"amount" datastore:"amount"` -} - -type StatisticsItem struct { - Total int64 `json:"total" datastore:"total"` - Fieldname string `json:"field_name" datastore:"field_name"` - Data []StatisticsData `json:"data" datastore:"data"` -} -*/ - -// "Execution by status" -// Execution history -//type GlobalStatistics struct { -// BackendExecutions int64 `json:"backend_executions" datastore:"backend_executions"` -// WorkflowCount int64 `json:"workflow_count" datastore:"workflow_count"` -// ExecutionCount int64 `json:"execution_count" datastore:"execution_count"` -// ExecutionSuccessCount int64 `json:"execution_success_count" datastore:"execution_success_count"` -// ExecutionAbortCount int64 `json:"execution_abort_count" datastore:"execution_abort_count"` -// ExecutionFailureCount int64 `json:"execution_failure_count" datastore:"execution_failure_count"` -// ExecutionPendingCount int64 `json:"execution_pending_count" datastore:"execution_pending_count"` -// AppUsageCount int64 `json:"app_usage_count" datastore:"app_usage_count"` -// TotalAppsCount int64 `json:"total_apps_count" datastore:"total_apps_count"` -// SelfMadeAppCount int64 `json:"self_made_app_count" datastore:"self_made_app_count"` -// WebhookUsageCount int64 `json:"webhook_usage_count" datastore:"webhook_usage_count"` -// Baseline map[string]int64 `json:"baseline" datastore:"baseline"` -//} - -/* -type ParsedOpenApi struct { - Body string `datastore:"body,noindex" json:"body"` - ID string `datastore:"id" json:"id"` - Success bool `datastore:"success,omitempty" json:"success,omitempty"` -} - -// Limits set for a user so that they can't do a shitload -type UserLimits struct { - DailyApiUsage int64 `json:"daily_api_usage" datastore:"daily_api_usage"` - Dailyshuffle.WorkflowExecutions int64 `json:"daily_workflow_executions" datastore:"daily_workflow_executions"` - DailyCloudExecutions int64 `json:"daily_cloud_executions" datastore:"daily_cloud_executions"` - DailyTriggers int64 `json:"daily_triggers" datastore:"daily_triggers"` - DailyMailUsage int64 `json:"daily_mail_usage" datastore:"daily_mail_usage"` - MaxTriggers int64 `json:"max_triggers" datastore:"max_triggers"` - MaxWorkflows int64 `json:"max_workflows" datastore:"max_workflows"` -} - -type retStruct struct { - Success bool `json:"success"` - SyncFeatures SyncFeatures `json:"sync_features"` - SessionKey string `json:"session_key"` - IntervalSeconds int64 `json:"interval_seconds"` - Reason string `json:"reason"` -} - -// Saves some data, not sure what to have here lol -type UserAuth struct { - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - Name string `json:"name" datastore:"name" yaml:"name"` - Workflows []string `json:"workflows" datastore:"workflows"` - Username string `json:"username" datastore:"username"` - Fields []UserAuthField `json:"fields" datastore:"fields"` -} - -type UserAuthField struct { - Key string `json:"key" datastore:"key"` - Value string `json:"value" datastore:"value,noindex"` -} - -// Not environment, but execution environment -type Environment struct { - Name string `datastore:"name"` - Type string `datastore:"type"` - Registered bool `datastore:"registered"` - Default bool `datastore:"default" json:"default"` - Archived bool `datastore:"archived" json:"archived"` - Id string `datastore:"id" json:"id"` - OrgId string `datastore:"org_id" json:"org_id"` -} - -type User struct { - Username string `datastore:"Username" json:"username"` - Password string `datastore:"password,noindex" password:"password,omitempty"` - Session string `datastore:"session,noindex" json:"session"` - Verified bool `datastore:"verified,noindex" json:"verified"` - PrivateApps []WorkflowApp `datastore:"privateapps" json:"privateapps":` - Role string `datastore:"role" json:"role"` - Roles []string `datastore:"roles" json:"roles"` - VerificationToken string `datastore:"verification_token" json:"verification_token"` - ApiKey string `datastore:"apikey" json:"apikey"` - ResetReference string `datastore:"reset_reference" json:"reset_reference"` - Executions ExecutionInfo `datastore:"executions" json:"executions"` - Limits UserLimits `datastore:"limits" json:"limits"` - Authentication []UserAuth `datastore:"authentication,noindex" json:"authentication"` - ResetTimeout int64 `datastore:"reset_timeout,noindex" json:"reset_timeout"` - Id string `datastore:"id" json:"id"` - Orgs []string `datastore:"orgs" json:"orgs"` - CreationTime int64 `datastore:"creation_time" json:"creation_time"` - ActiveOrg Org `json:"active_org" datastore:"active_org"` - Active bool `datastore:"active" json:"active"` -} - -// timeout maybe? idk -type session struct { - Username string `datastore:"Username,noindex"` - Id string `datastore:"Id,noindex"` - Session string `datastore:"session,noindex"` -} - -type loginStruct struct { - Username string `json:"username"` - Password string `json:"password"` -} - -type Contact struct { - Firstname string `json:"firstname"` - Lastname string `json:"lastname"` - Title string `json:"title"` - Companyname string `json:"companyname"` - Phone string `json:"phone"` - Email string `json:"email"` - Message string `json:"message"` -} - -type Translator struct { - Src struct { - Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value,noindex"` - Description string `json:"description" datastore:"description,noindex"` - Required string `json:"required" datastore:"required"` - Type string `json:"type" datastore:"type"` - Schema struct { - Type string `json:"type" datastore:"type"` - } `json:"schema" datastore:"schema"` - } `json:"src" datastore:"src"` - Dst struct { - Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value,noindex"` - Type string `json:"type" datastore:"type"` - Description string `json:"description" datastore:"description,noindex"` - Required string `json:"required" datastore:"required"` - Schema struct { - Type string `json:"type" datastore:"type"` - } `json:"schema" datastore:"schema"` - } `json:"dst" datastore:"dst"` -} - -type Appconfig struct { - Key string `json:"key" datastore:"key"` - Value string `json:"value" datastore:"value,noindex"` -} - -type ScheduleApp struct { - Foldername string `json:"foldername" datastore:"foldername,noindex"` - Name string `json:"name" datastore:"name,noindex"` - Id string `json:"id" datastore:"id,noindex"` - Description string `json:"description" datastore:"description,noindex"` - Action string `json:"action" datastore:"action,noindex"` - Config []Appconfig `json:"config,omitempty" datastore:"config,noindex"` -} - -type AppInfo struct { - SourceApp ScheduleApp `json:"sourceapp,omitempty" datastore:"sourceapp,noindex"` - DestinationApp ScheduleApp `json:"destinationapp,omitempty" datastore:"destinationapp,noindex"` -} - -// May 2020: Reused for onprem schedules - Id, Seconds, WorkflowId and argument -type ScheduleOld struct { - Id string `json:"id" datastore:"id"` - StartNode string `json:"start_node" datastore:"start_node"` - Seconds int `json:"seconds" datastore:"seconds"` - WorkflowId string `json:"workflow_id" datastore:"workflow_id", ` - Argument string `json:"argument" datastore:"argument"` - WrappedArgument string `json:"wrapped_argument" datastore:"wrapped_argument"` - AppInfo AppInfo `json:"appinfo" datastore:"appinfo,noindex"` - Finished bool `json:"finished" finished:"id"` - BaseAppLocation string `json:"base_app_location" datastore:"baseapplocation,noindex"` - Translator []Translator `json:"translator,omitempty" datastore:"translator"` - Org string `json:"org" datastore:"org"` - CreatedBy string `json:"createdby" datastore:"createdby"` - Availability string `json:"availability" datastore:"availability"` - CreationTime int64 `json:"creationtime" datastore:"creationtime,noindex"` - LastModificationtime int64 `json:"lastmodificationtime" datastore:"lastmodificationtime,noindex"` - LastRuntime int64 `json:"lastruntime" datastore:"lastruntime,noindex"` - Frequency string `json:"frequency" datastore:"frequency,noindex"` - Environment string `json:"environment" datastore:"environment"` -} - -// Returned from /GET /schedules -type Schedules struct { - Schedules []ScheduleOld `json:"schedules"` - Success bool `json:"success"` -} - -type ScheduleApps struct { - Apps []ApiYaml `json:"apps"` - Success bool `json:"success"` -} - -// The yaml that is uploaded -type ApiYaml struct { - Name string `json:"name" yaml:"name" required:"true datastore:"name"` - Foldername string `json:"foldername" yaml:"foldername" required:"true datastore:"foldername"` - Id string `json:"id" yaml:"id",required:"true, datastore:"id"` - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - AppVersion string `json:"app_version" yaml:"app_version",datastore:"app_version"` - ContactInfo struct { - Name string `json:"name" datastore:"name" yaml:"name"` - Url string `json:"url" datastore:"url" yaml:"url"` - } `json:"contact_info" datastore:"contact_info" yaml:"contact_info"` - Types []string `json:"types" datastore:"types" yaml:"types"` - Input []struct { - Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - InputParameters []struct { - Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - Required string `json:"required" datastore:"required" yaml:"required"` - Schema struct { - Type string `json:"type" datastore:"type" yaml:"type"` - } `json:"schema" datastore:"schema" yaml:"schema"` - } `json:"inputparameters" datastore:"inputparameters" yaml:"inputparameters"` - OutputParameters []struct { - Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - Required string `json:"required" datastore:"required" yaml:"required"` - Schema struct { - Type string `json:"type" datastore:"type" yaml:"type"` - } `json:"schema" datastore:"schema" yaml:"schema"` - } `json:"outputparameters" datastore:"outputparameters" yaml:"outputparameters"` - Config []struct { - Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - Required string `json:"required" datastore:"required" yaml:"required"` - Schema struct { - Type string `json:"type" datastore:"type" yaml:"type"` - } `json:"schema" datastore:"schema" yaml:"schema"` - } `json:"config" datastore:"config" yaml:"config"` - } `json:"input" datastore:"input" yaml:"input"` - Output []struct { - Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - Config []struct { - Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - Required string `json:"required" datastore:"required" yaml:"required"` - Schema struct { - Type string `json:"type" datastore:"type" yaml:"type"` - } `json:"schema" datastore:"schema" yaml:"schema"` - } `json:"config" datastore:"config" yaml:"config"` - InputParameters []struct { - Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - Required string `json:"required" datastore:"required" yaml:"required"` - Schema struct { - Type string `json:"type" datastore:"type" yaml:"type"` - } `json:"schema" datastore:"schema" yaml:"schema"` - } `json:"inputparameters" datastore:"inputparameters" yaml:"inputparameters"` - OutputParameters []struct { - Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - Required string `json:"required" datastore:"required" yaml:"required"` - Schema struct { - Type string `json:"type" datastore:"type" yaml:"type"` - } `json:"schema" datastore:"schema" yaml:"schema"` - } `json:"outputparameters" datastore:"outputparameters" yaml:"outputparameters"` - } `json:"output" datastore:"output" yaml:"output"` -} - -type Hooks struct { - Hooks []Hook `json:"hooks"` - Success bool `json:"-"` -} - -type Info struct { - Url string `json:"url" datastore:"url"` - Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description,noindex"` -} - -// shuffle.Actions to be done by webhooks etc -// Field is the actual field to use from json -type HookAction struct { - Type string `json:"type" datastore:"type"` - Name string `json:"name" datastore:"name"` - Id string `json:"id" datastore:"id"` - Field string `json:"field" datastore:"field"` -} - -type Hook struct { - Id string `json:"id" datastore:"id"` - Start string `json:"start" datastore:"start"` - Info Info `json:"info" datastore:"info"` - Actions []HookAction `json:"actions" datastore:"actions,noindex"` - Type string `json:"type" datastore:"type"` - Owner string `json:"owner" datastore:"owner"` - Status string `json:"status" datastore:"status"` - Workflows []string `json:"workflows" datastore:"workflows"` - Running bool `json:"running" datastore:"running"` - OrgId string `json:"org_id" datastore:"org_id"` - Environment string `json:"environment" datastore:"environment"` -} - -type ExecutionRequest struct { - ExecutionId string `json:"execution_id,omitempty"` - ExecutionArgument string `json:"execution_argument,omitempty"` - ExecutionSource string `json:"execution_source,omitempty"` - WorkflowId string `json:"workflow_id,omitempty"` - Environments []string `json:"environments,omitempty"` - Authorization string `json:"authorization,omitempty"` - Status string `json:"status,omitempty"` - Start string `json:"start,omitempty"` - Type string `json:"type,omitempty"` -} - -type SyncFeatures struct { - Webhook SyncData `json:"webhook" datastore:"webhook"` - Schedules SyncData `json:"schedules" datastore:"schedules"` - UserInput SyncData `json:"user_input" datastore:"user_input"` - SendMail SyncData `json:"send_mail" datastore:"send_mail"` - SendSms SyncData `json:"send_sms" datastore:"send_sms"` - Updates SyncData `json:"updates" datastore:"updates"` - Notifications SyncData `json:"notifications" datastore:"notifications"` - EmailTrigger SyncData `json:"email_trigger" datastore:"email_trigger"` - AppExecutions SyncData `json:"app_executions" datastore:"app_executions"` - shuffle.WorkflowExecutions SyncData `json:"workflow_executions" datastore:"workflow_executions"` - Apps SyncData `json:"apps" datastore:"apps"` - Workflows SyncData `json:"workflows" datastore:"workflows"` - Autocomplete SyncData `json:"autocomplete" datastore:"autocomplete"` - Authentication SyncData `json:"authentication" datastore:"authentication"` - Schedule SyncData `json:"schedule" datastore:"schedule"` -} - -type SyncData struct { - Active bool `json:"active" datastore:"active"` - Type string `json:"type,omitempty" datastore:"type"` - Name string `json:"name,omitempty" datastore:"name"` - Description string `json:"description,omitempty" datastore:"description"` - Limit int64 `json:"limit,omitempty" datastore:"limit"` - StartDate int64 `json:"start_date,omitempty" datastore:"start_date"` - EndDate int64 `json:"end_date,omitempty" datastore:"end_date"` - DataCollection int64 `json:"data_collection,omitempty" datastore:"data_collection"` -} - -type SyncConfig struct { - Interval int64 `json:"interval" datastore:"interval"` - Apikey string `json:"api_key" datastore:"api_key"` -} - -// Role is just used for feedback for a user -type Org struct { - Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description"` - Image string `json:"image" datastore:"image,noindex"` - Id string `json:"id" datastore:"id"` - Org string `json:"org" datastore:"org"` - Users []User `json:"users" datastore:"users"` - Role string `json:"role" datastore:"role"` - Roles []string `json:"roles" datastore:"roles"` - CloudSync bool `json:"cloud_sync" datastore:"CloudSync"` - SyncConfig SyncConfig `json:"sync_config" datastore:"sync_config"` - SyncFeatures SyncFeatures `json:"sync_features" datastore:"sync_features"` - Created int64 `json:"created" datastore:"created"` - Edited int64 `json:"edited" datastore:"edited"` -} - -type AppAuthenticationStorage struct { - Active bool `json:"active" datastore:"active"` - Label string `json:"label" datastore:"label"` - Id string `json:"id" datastore:"id"` - App WorkflowApp `json:"app" datastore:"app,noindex"` - Fields []AuthenticationStore `json:"fields" datastore:"fields"` - Usage []AuthenticationUsage `json:"usage" datastore:"usage"` - WorkflowCount int64 `json:"workflow_count" datastore:"workflow_count"` - NodeCount int64 `json:"node_count" datastore:"node_count"` - OrgId string `json:"org_id" datastore:"org_id"` - Created int64 `json:"created" datastore:"created"` - Edited int64 `json:"edited" datastore:"edited"` -} - -type AuthenticationUsage struct { - WorkflowId string `json:"workflow_id" datastore:"workflow_id"` - Nodes []string `json:"nodes" datastore:"nodes"` -} - -// An app inside Shuffle -// Source string `json:"source" datastore:"soure" yaml:"source"` - downloadlocation -type WorkflowApp struct { - Name string `json:"name" yaml:"name" required:true datastore:"name"` - IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"` - ID string `json:"id" yaml:"id,omitempty" required:false datastore:"id"` - Link string `json:"link" yaml:"link" required:false datastore:"link,noindex"` - AppVersion string `json:"app_version" yaml:"app_version" required:true datastore:"app_version"` - SharingConfig string `json:"sharing_config" yaml:"sharing_config" datastore:"sharing_config"` - Generated bool `json:"generated" yaml:"generated" required:false datastore:"generated"` - Downloaded bool `json:"downloaded" yaml:"downloaded" required:false datastore:"downloaded"` - Sharing bool `json:"sharing" yaml:"sharing" required:false datastore:"sharing"` - Verified bool `json:"verified" yaml:"verified" required:false datastore:"verified"` - Activated bool `json:"activated" yaml:"activated" required:false datastore:"activated"` - Tested bool `json:"tested" yaml:"tested" required:false datastore:"tested"` - Owner string `json:"owner" datastore:"owner" yaml:"owner"` - Hash string `json:"hash" datastore:"hash" yaml:"hash"` // api.yaml+dockerfile+src/app.py for apps - PrivateID string `json:"private_id" yaml:"private_id" required:false datastore:"private_id"` - Description string `json:"description" datastore:"description,noindex" required:false yaml:"description"` - Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"` - SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"` - LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false` - ContactInfo struct { - Name string `json:"name" datastore:"name" yaml:"name"` - Url string `json:"url" datastore:"url" yaml:"url"` - } `json:"contact_info" datastore:"contact_info" yaml:"contact_info" required:false` - Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions,noindex"` - Authentication Authentication `json:"authentication" yaml:"authentication" required:false datastore:"authentication"` - Tags []string `json:"tags" yaml:"tags" required:false datastore:"activated"` - Categories []string `json:"categories" yaml:"categories" required:false datastore:"categories"` - Created int64 `json:"created" datastore:"created"` - Edited int64 `json:"edited" datastore:"edited"` - LastRuntime int64 `json:"last_runtime" datastore:"last_runtime"` -} - -type shuffle.WorkflowAppActionParameter struct { - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - ID string `json:"id" datastore:"id" yaml:"id,omitempty"` - Name string `json:"name" datastore:"name" yaml:"name"` - Example string `json:"example" datastore:"example,noindex" yaml:"example"` - Value string `json:"value" datastore:"value,noindex" yaml:"value,omitempty"` - Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` - Options []string `json:"options" datastore:"options" yaml:"options"` - ActionField string `json:"action_field" datastore:"action_field" yaml:"actionfield,omitempty"` - Variant string `json:"variant" datastore:"variant" yaml:"variant,omitempty"` - Required bool `json:"required" datastore:"required" yaml:"required"` - Configuration bool `json:"configuration" datastore:"configuration" yaml:"configuration"` - Tags []string `json:"tags" datastore:"tags" yaml:"tags"` - Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` - SkipMulticheck bool `json:"skip_multicheck" datastore:"skip_multicheck" yaml:"skip_multicheck"` - ValueReplace []Valuereplace `json:"value_replace" datastore:"value_replace,noindex" yaml:"value_replace,omitempty"` - UniqueToggled bool `json:"unique_toggled" datastore:"unique_toggled" yaml:"unique_toggled"` -} - -type Valuereplace struct { - Key string `json:"key" datastore:"key" yaml:"key"` - Value string `json:"value" datastore:"value" yaml:"value"` -} - -type SchemaDefinition struct { - Type string `json:"type" datastore:"type"` -} - -type WorkflowAppAction struct { - Description string `json:"description" datastore:"description,noindex"` - ID string `json:"id" datastore:"id" yaml:"id,omitempty"` - Name string `json:"name" datastore:"name"` - Label string `json:"label" datastore:"label"` - NodeType string `json:"node_type" datastore:"node_type"` - Environment string `json:"environment" datastore:"environment"` - Sharing bool `json:"sharing" datastore:"sharing"` - PrivateID string `json:"private_id" datastore:"private_id"` - AppID string `json:"app_id" datastore:"app_id"` - Tags []string `json:"tags" datastore:"tags" yaml:"tags"` - Authentication []AuthenticationStore `json:"authentication" datastore:"authentication,noindex" yaml:"authentication,omitempty"` - Tested bool `json:"tested" datastore:"tested" yaml:"tested"` - Parameters []shuffle.WorkflowAppActionParameter `json:"parameters" datastore: "parameters"` - ExecutionVariable struct { - Description string `json:"description" datastore:"description,noindex"` - ID string `json:"id" datastore:"id"` - Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value,noindex"` - } `json:"execution_variable" datastore:"execution_variables"` - Returns struct { - Description string `json:"description" datastore:"returns" yaml:"description,omitempty"` - Example string `json:"example" datastore:"example" yaml:"example"` - ID string `json:"id" datastore:"id" yaml:"id,omitempty"` - Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` - } `json:"returns" datastore:"returns"` - AuthenticationId string `json:"authentication_id" datastore:"authentication_id"` - Example string `json:"example" datastore:"example" yaml:"example"` - AuthNotRequired bool `json:"auth_not_required" datastore:"auth_not_required" yaml:"auth_not_required"` -} - -type shuffle.WorkflowExecution struct { - Type string `json:"type" datastore:"type"` - Status string `json:"status" datastore:"status"` - Start string `json:"start" datastore:"start"` - ExecutionArgument string `json:"execution_argument" datastore:"execution_argument,noindex"` - ExecutionId string `json:"execution_id" datastore:"execution_id"` - ExecutionSource string `json:"execution_source" datastore:"execution_source"` - ExecutionParent string `json:"execution_parent" datastore:"execution_parent"` - ExecutionOrg string `json:"execution_org" datastore:"execution_org"` - WorkflowId string `json:"workflow_id" datastore:"workflow_id"` - LastNode string `json:"last_node" datastore:"last_node"` - Authorization string `json:"authorization" datastore:"authorization"` - Result string `json:"result" datastore:"result,noindex"` - StartedAt int64 `json:"started_at" datastore:"started_at"` - CompletedAt int64 `json:"completed_at" datastore:"completed_at"` - ProjectId string `json:"project_id" datastore:"project_id"` - Locations []string `json:"locations" datastore:"locations"` - Workflow Workflow `json:"workflow" datastore:"workflow,noindex"` - Results []ActionResult `json:"results" datastore:"results,noindex"` - ExecutionVariables []struct { - Description string `json:"description" datastore:"description,noindex"` - ID string `json:"id" datastore:"id"` - Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value,noindex"` - } `json:"execution_variables,omitempty" datastore:"execution_variables,omitempty"` - OrgId string `json:"org_id" datastore:"org_id"` -} - -type shuffle.Action struct { - AppName string `json:"app_name,omitempty" datastore:"app_name"` - AppVersion string `json:"app_version,omitempty" datastore:"app_version"` - AppID string `json:"app_id,omitempty" datastore:"app_id"` - Errors []string `json:"errors,omitempty" datastore:"errors"` - ID string `json:"id,omitempty" datastore:"id"` - IsValid bool `json:"is_valid,omitempty" datastore:"is_valid"` - IsStartNode bool `json:"isStartNode,omitempty" datastore:"isStartNode"` - Sharing bool `json:"sharing,omitempty" datastore:"sharing"` - PrivateID string `json:"private_id,omitempty" datastore:"private_id"` - Label string `json:"label,omitempty" datastore:"label"` - SmallImage string `json:"small_image,omitempty" datastore:"small_image,noindex" required:false yaml:"small_image"` - LargeImage string `json:"large_image,omitempty" datastore:"large_image,noindex" yaml:"large_image" required:false` - Environment string `json:"environment,omitempty" datastore:"environment"` - Name string `json:"name,omitempty" datastore:"name"` - Parameters []shuffle.WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"` - ExecutionVariable struct { - Description string `json:"description,omitempty" datastore:"description,noindex"` - ID string `json:"id,omitempty" datastore:"id"` - Name string `json:"name,omitempty" datastore:"name"` - Value string `json:"value,omitempty" datastore:"value,noindex"` - } `json:"execution_variable,omitempty" datastore:"execution_variable,omitempty"` - Position struct { - X float64 `json:"x,omitempty" datastore:"x"` - Y float64 `json:"y,omitempty" datastore:"y"` - } `json:"position,omitempty"` - Priority int `json:"priority,omitempty" datastore:"priority"` - AuthenticationId string `json:"authentication_id,omitempty" datastore:"authentication_id"` - Example string `json:"example,omitempty" datastore:"example"` - AuthNotRequired bool `json:"auth_not_required,omitempty" datastore:"auth_not_required" yaml:"auth_not_required"` -} - -// Added environment for location to execute -type shuffle.Trigger struct { - AppName string `json:"app_name" datastore:"app_name"` - Description string `json:"description" datastore:"description,noindex"` - LongDescription string `json:"long_description" datastore:"long_description"` - Status string `json:"status" datastore:"status"` - AppVersion string `json:"app_version" datastore:"app_version"` - Errors []string `json:"errors" datastore:"errors"` - ID string `json:"id" datastore:"id"` - IsValid bool `json:"is_valid" datastore:"is_valid"` - IsStartNode bool `json:"isStartNode" datastore:"isStartNode"` - Label string `json:"label" datastore:"label"` - SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"` - LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false` - Environment string `json:"environment" datastore:"environment"` - TriggerType string `json:"trigger_type" datastore:"trigger_type"` - Name string `json:"name" datastore:"name"` - Tags []string `json:"tags" datastore:"tags" yaml:"tags"` - Parameters []shuffle.WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"` - Position struct { - X float64 `json:"x" datastore:"x"` - Y float64 `json:"y" datastore:"y"` - } `json:"position"` - Priority int `json:"priority" datastore:"priority"` -} - -type Branch struct { - DestinationID string `json:"destination_id" datastore:"destination_id"` - ID string `json:"id" datastore:"id"` - SourceID string `json:"source_id" datastore:"source_id"` - Label string `json:"label" datastore:"label"` - HasError bool `json:"has_errors" datastore: "has_errors"` - Conditions []Condition `json:"conditions" datastore: "conditions,noindex"` -} - -// Same format for a lot of stuff -type Condition struct { - Condition shuffle.WorkflowAppActionParameter `json:"condition" datastore:"condition"` - Source shuffle.WorkflowAppActionParameter `json:"source" datastore:"source"` - Destination shuffle.WorkflowAppActionParameter `json:"destination" datastore:"destination"` -} - -type Schedule struct { - Name string `json:"name" datastore:"name"` - Frequency string `json:"frequency" datastore:"frequency"` - ExecutionArgument string `json:"execution_argument" datastore:"execution_argument,noindex"` - Id string `json:"id" datastore:"id"` - OrgId string `json:"org_id" datastore:"org_id"` - Environment string `json:"environment" datastore:"environment"` -} - -type Workflow struct { - Actions []Action `json:"actions" datastore:"actions,noindex"` - Branches []Branch `json:"branches" datastore:"branches,noindex"` - Triggers []Trigger `json:"triggers" datastore:"triggers,noindex"` - Schedules []Schedule `json:"schedules" datastore:"schedules,noindex"` - Configuration struct { - ExitOnError bool `json:"exit_on_error" datastore:"exit_on_error"` - StartFromTop bool `json:"start_from_top" datastore:"start_from_top"` - } `json:"configuration,omitempty" datastore:"configuration"` - Created int64 `json:"created" datastore:"created"` - Edited int64 `json:"edited" datastore:"edited"` - LastRuntime int64 `json:"last_runtime" datastore:"last_runtime"` - Errors []string `json:"errors,omitempty" datastore:"errors"` - Tags []string `json:"tags,omitempty" datastore:"tags"` - ID string `json:"id" datastore:"id"` - IsValid bool `json:"is_valid" datastore:"is_valid"` - Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description,noindex"` - Start string `json:"start" datastore:"start"` - Owner string `json:"owner" datastore:"owner"` - Sharing string `json:"sharing" datastore:"sharing"` - Org []Org `json:"org,omitempty" datastore:"org"` - ExecutingOrg Org `json:"execution_org,omitempty" datastore:"execution_org"` - OrgId string `json:"org_id,omitempty" datastore:"org_id"` - WorkflowVariables []struct { - Description string `json:"description" datastore:"description,noindex"` - ID string `json:"id" datastore:"id"` - Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value,noindex"` - } `json:"workflow_variables" datastore:"workflow_variables"` - ExecutionVariables []struct { - Description string `json:"description" datastore:"description,noindex"` - ID string `json:"id" datastore:"id"` - Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value,noindex"` - } `json:"execution_variables,omitempty" datastore:"execution_variables"` - ExecutionEnvironment string `json:"execution_environment" datastore:"execution_environment"` -} - -type shuffle.ActionResult struct { - Action shuffle.Action `json:"action" datastore:"action,noindex"` - ExecutionId string `json:"execution_id" datastore:"execution_id"` - Authorization string `json:"authorization" datastore:"authorization"` - Result string `json:"result" datastore:"result,noindex"` - StartedAt int64 `json:"started_at" datastore:"started_at"` - CompletedAt int64 `json:"completed_at" datastore:"completed_at"` - Status string `json:"status" datastore:"status"` -} - -type Authentication struct { - Required bool `json:"required" datastore:"required" yaml:"required" ` - Parameters []AuthenticationParams `json:"parameters" datastore:"parameters" yaml:"parameters"` -} - -type AuthenticationParams struct { - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - ID string `json:"id" datastore:"id" yaml:"id"` - Name string `json:"name" datastore:"name" yaml:"name"` - Example string `json:"example" datastore:"example" yaml:"example"` - Value string `json:"value,omitempty" datastore:"value,noindex" yaml:"value"` - Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` - Required bool `json:"required" datastore:"required" yaml:"required"` - In string `json:"in" datastore:"in" yaml:"in"` - Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` - Scheme string `json:"scheme" datastore:"scheme" yaml:"scheme"` // Deprecated -} - -type AuthenticationStore struct { - Key string `json:"key" datastore:"key"` - Value string `json:"value" datastore:"value,noindex"` -} - -type ExecutionRequestWrapper struct { - Data []ExecutionRequest `json:"data"` -} - -type AppExecutionExample struct { - AppName string `json:"app_name" datastore:"app_name"` - AppVersion string `json:"app_version" datastore:"app_version"` - AppAction string `json:"app_action" datastore:"app_action"` - AppId string `json:"app_id" datastore:"app_id"` - ExampleId string `json:"example_id" datastore:"example_id"` - SuccessExamples []string `json:"success_examples" datastore:"success_examples,noindex"` - FailureExamples []string `json:"failure_examples" datastore:"failure_examples,noindex"` -} -*/ - // removes every container except itself (worker) func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason string, handleResultSend bool) { log.Printf("[INFO] Shutdown (%s) started with reason %s", workflowExecution.Status, reason) From 6574f122f18869b6f1617452f57cb86f1285e04e Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 21 Mar 2021 18:01:09 +0100 Subject: [PATCH 45/57] Upgrade pre shuffle-shared merge --- backend/app_sdk/app_base.py | 6 +-- backend/go-app/codegen.go | 14 +++---- backend/go-app/main.go | 28 ++++++++++++- backend/go-app/walkoff.go | 10 ++++- frontend/src/defaultCytoscapeStyle.js | 14 +++++-- frontend/src/views/AngularWorkflow.jsx | 55 +++++++++++++++++++------- 6 files changed, 94 insertions(+), 33 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index c18be280..5e2cdd3e 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -141,11 +141,11 @@ class AppBase: print(f"VALUE APPEND: {value}") param_value += value - if param["name"] not in param_names: param_names.append(param["name"]) + except (KeyError, NameError) as e: - print(f"Key/NameError in param handler: {e}") + print(f"""Key/NameError in param handler for {param["name"]}: {e}""") print(f"OUTER VALUE: {param_value}") if len(param_value) > 0: @@ -2292,7 +2292,7 @@ class AppBase: if action_result["result"] == "": action_result["result"] = result - self.logger.debug(f"Executed {action['label']}-{action['id']} with result: {result}") + self.logger.debug(f"Executed {action['label']}-{action['id']}")#with result: {result}") #self.logger.debug(f"Data: %s" % action_result) except TypeError as e: print("TypeError issue: %s" % e) diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index b0b9b14d..5943d4bd 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -429,15 +429,13 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet ) // Use lowercase when checking - /* - if strings.Contains(functionname, "login") { - //log.Printf("FUNCTION: %s", data) - log.Println(data) - log.Printf("Queries: %s", queryString) - } - */ - //log.Printf(data) + if strings.Contains(functionname, "attachment") { + //log.Printf("FUNCTION: %s", data) + //log.Println(data) + //log.Printf("Queries: %s", queryString) + } + return functionname, data } diff --git a/backend/go-app/main.go b/backend/go-app/main.go index abb13c8b..deeb5bc8 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3405,7 +3405,7 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { //resp.WriteHeader(200) //resp.Write([]byte(`{"success": true}`)) if hook.Status == "stopped" { - log.Printf("Not running %s because hook status is stopped", hook.Id) + log.Printf("[WARNING] Not running %s because hook status is stopped", hook.Id) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "The webhook isn't running. Click start to start it"}`))) return @@ -3436,10 +3436,34 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { return } + //log.Printf("BODY: %s", parsedBody) + + // This is a specific fix for MSteams and may fix other things as well + // Scared whether it may stop other things though, but that's a future problem + // (famous last words) + parsedBody := string(body) + if strings.Contains(parsedBody, "choice") { + if strings.Count(parsedBody, `\\n`) > 2 { + parsedBody = strings.Replace(parsedBody, `\\n`, "", -1) + } + if strings.Count(parsedBody, `\u0022`) > 2 { + parsedBody = strings.Replace(parsedBody, `\u0022`, `"`, -1) + } + if strings.Count(parsedBody, `\\"`) > 2 { + parsedBody = strings.Replace(parsedBody, `\\"`, `"`, -1) + } + + if strings.Contains(parsedBody, `"extra": "{`) { + parsedBody = strings.Replace(parsedBody, `"extra": "{`, `"extra": {`, 1) + parsedBody = strings.Replace(parsedBody, `}"}`, `}}`, 1) + } + } + + //log.Printf("\n\nPARSEDBODY: %s", parsedBody) newBody := ExecutionStruct{ Start: hook.Start, ExecutionSource: "webhook", - ExecutionArgument: string(body), + ExecutionArgument: parsedBody, } b, err := json.Marshal(newBody) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 456ecb4f..8af8055e 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -981,6 +981,10 @@ func validateNewWorkerExecution(body []byte) error { return errors.New(fmt.Sprintf("Bad length of trigger: %d (probably normal app)", len(execution.Workflow.Triggers))) } + if baseExecution.Status != "WAITING" && baseExecution.Status != "EXECUTING" { + return errors.New(fmt.Sprintf("Workflow is already finished or failed. Can't update")) + } + if execution.Status == "EXECUTING" { //log.Printf("[INFO] Inside executing.") extra := 0 @@ -994,6 +998,8 @@ func validateNewWorkerExecution(body []byte) error { if len(execution.Workflow.Actions)+extra == len(execution.Results) { execution.Status = "FINISHED" } + + log.Printf("BASEEXECUTION LENGTH: %d", len(baseExecution.Workflow.Actions)+extra) } // FIXME: Add extra here @@ -3699,14 +3705,14 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf Status: "SKIPPED", }) } else { - log.Printf("SHOULD KEEP TRIGGER %s", trigger.ID) + //log.Printf("SHOULD KEEP TRIGGER %s", trigger.ID) } } } //childNodes := findChildNodes(workflowExecution, workflowExecution.Start) if !startFound { - log.Printf("Startnode %s doesn't exist!", workflowExecution.Start) + log.Printf("[ERROR] Startnode %s doesn't exist!!", workflowExecution.Start) return WorkflowExecution{}, fmt.Sprintf("Workflow action %s doesn't exist in workflow", workflowExecution.Start), errors.New(fmt.Sprintf(`Workflow start node "%s" doesn't exist. Exiting!`, workflowExecution.Start)) } diff --git a/frontend/src/defaultCytoscapeStyle.js b/frontend/src/defaultCytoscapeStyle.js index 90c7aae8..ff9990a9 100644 --- a/frontend/src/defaultCytoscapeStyle.js +++ b/frontend/src/defaultCytoscapeStyle.js @@ -164,7 +164,7 @@ const data = [{ css: { 'background-color': '#ffef47', 'border-color': '#ffef47', - 'border-width': '5px', + 'border-width': '8px', 'transition-property': 'border-width', 'transition-duration': '0.25s', }, @@ -212,10 +212,13 @@ const data = [{ selector: 'edge.success-highlight', css: { 'width': '5px', - 'target-arrow-color': '#399645', - 'line-color': '#399645', + 'target-arrow-color': '#41dcab', + 'line-color': '#41dcab', 'transition-property': 'line-color, width', 'transition-duration': '0.5s', + "line-fill": "linear-gradient", + "line-gradient-stop-positions": ["0.0", "100"], + "line-gradient-stop-colors": ["#41dcab", "#41dcab"], }, }, { @@ -223,7 +226,10 @@ const data = [{ css: { 'target-arrow-color': '#991818', 'line-color': '#991818', - 'line-style': 'dashed' + 'line-style': 'dashed', + "line-fill": "linear-gradient", + "line-gradient-stop-positions": ["0.0", "100"], + "line-gradient-stop-colors": ["#991818", "#991818"], }, }, { diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 40f9b948..4cd4939a 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -447,7 +447,7 @@ const AngularWorkflow = (props) => { return response.json() }) .then((responseJson) => { - handleUpdateResults(responseJson) + handleUpdateResults(responseJson, executionRequest) }) .catch(error => { console.log("Error: ", error) @@ -484,7 +484,7 @@ const AngularWorkflow = (props) => { // Controls the colors and direction of execution results. // Style is in defaultCytoscapeStyle.js - const handleUpdateResults = (responseJson) => { + const handleUpdateResults = (responseJson, executionRequest) => { //console.log(responseJson) // Loop nodes and find results // Update on every interval? idk @@ -496,11 +496,16 @@ const AngularWorkflow = (props) => { setExecutionData(responseJson) } } + + //console.log("PRE LOOPING RESULTS: !", responseJson.execution_id, executionRequest.execution_id) + if (responseJson.execution_id !== executionRequest.execution_id) { cy.elements().removeClass('success-highlight failure-highlight executing-highlight') return } + //console.log("LOOPING RESULTS!") + if (responseJson.results !== null && responseJson.results !== []) { for (var key in responseJson.results) { var item = responseJson.results[key] @@ -562,6 +567,8 @@ const AngularWorkflow = (props) => { currentnode.removeClass('shuffle-hover-highlight') currentnode.removeClass('awaiting-data-highlight') currentnode.addClass('success-highlight') + incomingEdges.addClass('success-highlight') + outgoingEdges.addClass('success-highlight') if (visited !== undefined && visited !== null && !visited.includes(item.action.label)) { if (executionRunning) { @@ -603,7 +610,7 @@ const AngularWorkflow = (props) => { currentnode.addClass('failure-highlight') if (!visited.includes(item.action.label)) { - if (!item.action.result.includes("failed condition")) { + if (item.action.result !== undefined && item.action.result !== null && !item.action.result.includes("failed condition")) { alert.error("Error for "+item.action.label+" with result "+item.result) } visited.push(item.action.label) @@ -4014,7 +4021,7 @@ const AngularWorkflow = (props) => { } } }}> - + @@ -4029,10 +4036,16 @@ const AngularWorkflow = (props) => {
-
- +
+ {selectedAction.id === workflow.start ? null : + + + + }
@@ -6690,17 +6703,21 @@ const AngularWorkflow = (props) => { {}} onMouseOut={() => {}} onClick={() => { - if (data.result === undefined || data.result === null || data.result.length === 0) { - setExecutionRequest({ - "execution_id": data.execution_id, - "authorization": data.authorization, - }) + if ((data.result === undefined || data.result === null || data.result.length === 0) && data.status !== "FINISHED" && data.status !== "ABORTED") { + start() setExecutionRunning(true) setExecutionRequestStarted(false) } + + const cur_execution = { + "execution_id": data.execution_id, + "authorization": data.authorization, + } + setExecutionRequest(cur_execution) setExecutionModalView(1) setExecutionData(data) + handleUpdateResults(data, cur_execution) }}>
@@ -6880,7 +6897,17 @@ const AngularWorkflow = (props) => { } return ( -
+
{ + var currentnode = cy.getElementById(data.action.id) + if (currentnode.length !== 0) { + currentnode.addClass('shuffle-hover-highlight') + } + }} onMouseOut={() => { + var currentnode = cy.getElementById(data.action.id) + if (currentnode.length !== 0) { + currentnode.removeClass('shuffle-hover-highlight') + } + }}>
{ setSelectedResult(data) From d4d704f1700b996b31e7c1a971859ba870e7adf1 Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 21 Mar 2021 19:38:16 +0100 Subject: [PATCH 46/57] MAJOR migration fixes for hybrid usage --- backend/go-app/codegen.go | 2004 ------------------------------------- backend/go-app/docker.go | 4 +- backend/go-app/files.go | 882 ---------------- backend/go-app/go.mod | 32 +- backend/go-app/go.sum | 159 +++ backend/go-app/main.go | 645 ++++-------- backend/go-app/oauth2.go | 36 +- backend/go-app/walkoff.go | 1735 ++++++++++++++++---------------- 8 files changed, 1270 insertions(+), 4227 deletions(-) delete mode 100644 backend/go-app/codegen.go delete mode 100644 backend/go-app/files.go diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go deleted file mode 100644 index 5943d4bd..00000000 --- a/backend/go-app/codegen.go +++ /dev/null @@ -1,2004 +0,0 @@ -package main - -import ( - "archive/zip" - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "io/ioutil" - "log" - "os" - "strconv" - "strings" - - "cloud.google.com/go/storage" - "github.com/getkin/kin-openapi/openapi3" - //"github.com/satori/go.uuid" - "gopkg.in/yaml.v2" -) - -func copyFile(fromfile, tofile string) error { - from, err := os.Open(fromfile) - if err != nil { - return err - } - defer from.Close() - - to, err := os.OpenFile(tofile, os.O_RDWR|os.O_CREATE, 0666) - if err != nil { - return err - } - defer to.Close() - - _, err = io.Copy(to, from) - if err != nil { - return err - } - - return nil -} - -func formatAppfile(filedata string) (string, string) { - lines := strings.Split(filedata, "\n") - - newfile := []string{} - classname := "" - for _, line := range lines { - if strings.Contains(line, "walkoff_app_sdk") { - continue - } - - // Remap logging. CBA this right now - // This issue also persists in onprem apps because of await thingies.. :( - // FIXME - if strings.Contains(line, "console_logger") && strings.Contains(line, "await") { - continue - //line = strings.Replace(line, "console_logger", "logger", -1) - //log.Println(line) - } - - // Might not work with different import names - // Could be fucked up with spaces everywhere? Idk - if strings.Contains(line, "class") && strings.Contains(line, "(AppBase)") { - items := strings.Split(line, " ") - if len(items) > 0 && strings.Contains(items[1], "(AppBase)") { - classname = strings.Split(items[1], "(")[0] - } else { - // This could break something.. - classname = "TMP" - } - } - - if strings.Contains(line, "if __name__ ==") { - break - } - - // asyncio.run(HelloWorld.run(), debug=True) - - newfile = append(newfile, line) - } - - filedata = strings.Join(newfile, "\n") - return classname, filedata -} - -// Streams the data into a zip to be used for a cloud function -func streamZipdata(ctx context.Context, identifier, pythoncode, requirements string) (string, error) { - filename := fmt.Sprintf("generated_cloudfunctions/%s.zip", identifier) - - buf := new(bytes.Buffer) - zipWriter := zip.NewWriter(buf) - - zipFile, err := zipWriter.Create("main.py") - if err != nil { - log.Printf("Packing failed to create zip file from bucket: %v", err) - return filename, err - } - - // Have to use Fprintln otherwise it tries to parse all strings etc. - if _, err := fmt.Fprintln(zipFile, pythoncode); err != nil { - return filename, err - } - - zipFile, err = zipWriter.Create("requirements.txt") - if err != nil { - log.Printf("Packing failed to create zip file from bucket: %v", err) - return filename, err - } - if _, err := fmt.Fprintln(zipFile, requirements); err != nil { - return filename, err - } - - err = zipWriter.Close() - if err != nil { - log.Printf("Packing failed to close zip file writer from bucket: %v", err) - return filename, err - } - - return filename, nil -} - -func getAppbase() ([]byte, []byte, error) { - // 1. Have baseline in bucket/generated_apps/baseline - // 2. Copy the baseline to a new folder with identifier name - static := "../app_sdk/static_baseline.py" - appbase := "../app_sdk/app_base.py" - - staticData, err := ioutil.ReadFile(static) - if err != nil { - return []byte{}, []byte{}, err - } - - appbaseData, err := ioutil.ReadFile(appbase) - if err != nil { - return []byte{}, []byte{}, err - } - - return appbaseData, staticData, nil -} - -// Builds the structure for the new generated app in storage (copying baseline files) -func getAppbaseGCP(ctx context.Context, client *storage.Client) ([]byte, []byte, error) { - // 1. Have baseline in bucket/generated_apps/baseline - // 2. Copy the baseline to a new folder with identifier name - basePath := "generated_apps/baseline" - static, err := client.Bucket(bucketName).Object(fmt.Sprintf("%s/static_baseline.py", basePath)).NewReader(ctx) - if err != nil { - return []byte{}, []byte{}, err - } - appbase, err := client.Bucket(bucketName).Object(fmt.Sprintf("%s/app_base.py", basePath)).NewReader(ctx) - if err != nil { - return []byte{}, []byte{}, err - } - - defer static.Close() - defer appbase.Close() - - staticData, err := ioutil.ReadAll(static) - if err != nil { - return []byte{}, []byte{}, err - } - - appbaseData, err := ioutil.ReadAll(appbase) - if err != nil { - return []byte{}, []byte{}, err - } - - return appbaseData, staticData, nil -} - -func fixAppbase(appbase []byte) []string { - record := false - validLines := []string{} - for _, line := range strings.Split(string(appbase), "\n") { - if strings.Contains(line, "#STOPCOPY") { - //log.Println("Stopping copy") - break - } - - if record { - validLines = append(validLines, line) - } - - if strings.Contains(line, "#STARTCOPY") { - //log.Println("Starting copy") - record = true - } - } - - return validLines -} - -// Builds the structure for the new generated app in storage (copying baseline files) -func buildStructureGCP(ctx context.Context, client *storage.Client, swagger *openapi3.Swagger, curHash string) (string, error) { - // 1. Have baseline in bucket/generated_apps/baseline - // 2. Copy the baseline to a new folder with identifier name - - basePath := "generated_apps" - identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, curHash) - appPath := fmt.Sprintf("%s/%s", basePath, identifier) - fileNames := []string{"Dockerfile", "requirements.txt"} - for _, file := range fileNames { - src := client.Bucket(bucketName).Object(fmt.Sprintf("%s/baseline/%s", basePath, file)) - dst := client.Bucket(bucketName).Object(fmt.Sprintf("%s/%s", appPath, file)) - if _, err := dst.CopierFrom(src).Run(ctx); err != nil { - return "", err - } - } - - return appPath, nil -} - -// Builds the base structure for the app that we're making -// Returns error if anything goes wrong. This has to work if -// the python code is supposed to be generated -func buildStructure(swagger *openapi3.Swagger, curHash string) (string, error) { - //log.Printf("%#v", swagger) - - // adding md5 based on input data to not overwrite earlier data. - generatedPath := "generated" - subpath := "../app_gen/openapi/" - identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, curHash) - appPath := fmt.Sprintf("%s/%s", generatedPath, identifier) - - os.MkdirAll(appPath, os.ModePerm) - os.Mkdir(fmt.Sprintf("%s/src", appPath), os.ModePerm) - - err := copyFile(fmt.Sprintf("%sbaseline/Dockerfile", subpath), fmt.Sprintf("%s/%s", appPath, "Dockerfile")) - if err != nil { - log.Println("Failed to move Dockerfile") - return appPath, err - } - - err = copyFile(fmt.Sprintf("%sbaseline/requirements.txt", subpath), fmt.Sprintf("%s/%s", appPath, "requirements.txt")) - if err != nil { - log.Println("Failed to move requrements.txt") - return appPath, err - } - - return appPath, nil -} - -// This function generates the python code that's being used. -// This is really meta when you program it. Handling parameters is hard here. -func makePythoncode(swagger *openapi3.Swagger, name, url, method string, parameters, optionalQueries, headers []string, fileField string) (string, string) { - method = strings.ToLower(method) - queryString := "" - queryData := "" - - // FIXME - this might break - need to check if ? or & should be set as query - parameterData := "" - if len(optionalQueries) > 0 { - queryString += ", " - for index, query := range optionalQueries { - // Check if it's a part of the URL already - queryString += fmt.Sprintf("%s=\"\"", query) - if index != len(optionalQueries)-1 { - queryString += ", " - } - - /* - queryData += fmt.Sprintf(` - if %s: - url += f"&%s={%s}"`, query, query, query) - */ - queryData += fmt.Sprintf(` - if %s: - params["%s"] = %s`, query, query, query) - } - } else { - //log.Printf("No optional queries?") - } - - // api.Authentication.Parameters[0].Value = "BearerAuth" - authenticationParameter := "" - authenticationSetup := "" - authenticationAddin := "" - // Python configuration code that should work :) - if swagger.Components.SecuritySchemes != nil { - if swagger.Components.SecuritySchemes["BearerAuth"] != nil { - authenticationParameter = ", apikey" - authenticationSetup = "if apikey != \" \": headers[\"Authorization\"] = f\"Bearer {apikey}\"" - } else if swagger.Components.SecuritySchemes["BasicAuth"] != nil { - authenticationParameter = ", username_basic, password_basic" - authenticationAddin = ", auth=(username_basic, password_basic)" - } else if swagger.Components.SecuritySchemes["ApiKeyAuth"] != nil { - authenticationParameter = ", apikey" - if swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.In == "header" { - // This is a way to bypass apikeys by passing " " - authenticationSetup = fmt.Sprintf(`if apikey != " ": headers["%s"] = apikey`, swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.Name) - } else if swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.In == "query" { - // This might suck lol - key := "?" - if strings.Contains(url, "?") { - key = "&" - } - - authenticationSetup = fmt.Sprintf("if apikey != \" \": url+=f\"%s%s={apikey}\"", key, swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.Name) - } - } - } - - //baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) - // This is a quickfix for onpremises stuff. Does work, but should really be - // part of the authentication scheme from openapi3 - urlParameter := "" - urlInline := "" - //log.Printf("URL: %s", url) - if !strings.HasPrefix(strings.ToLower(url), "http") { - urlParameter = ", url" - urlInline = "{url}" - } - - // Specific check for SSL verification - // This is critical for onprem stuff. - //verifyParam := "" - //verifyWrapper := "" - //verifyAddin := "" - verifyParam := ", ssl_verify=False" - verifyWrapper := `if type(ssl_verify) == str: ssl_verify = False if ssl_verify.lower() == "false" or ssl_verify == "0" else True` - verifyAddin := ", verify=ssl_verify" - - if len(parameters) > 0 { - parameterData = fmt.Sprintf(", %s", strings.Join(parameters, ", ")) - } - - // FIXME - add checks for query data etc - - functionname := strings.ToLower(fmt.Sprintf("%s_%s", method, name)) - if strings.Contains(strings.ToLower(name), strings.ToLower(method)) { - functionname = strings.ToLower(name) - } - - bodyParameter := "" - bodyAddin := "" - bodyFormatter := "" - postParameters := []string{"post", "patch", "put"} - for _, item := range postParameters { - if method == item { - bodyParameter = ", body=\"\"" - bodyAddin = ", data=body" - - // FIXME: Does JSON data work? - bodyFormatter = `body = " ".join(body.strip().split()).encode("utf-8")` - } - } - - preparedHeaders := "headers={}" - if len(headers) > 0 { - preparedHeaders = "headers={" - for count, header := range headers { - headerSplit := strings.Split(header, "=") - added := false - if len(headerSplit) == 2 { - if strings.Contains(preparedHeaders, headerSplit[0]) { - continue - } - - preparedHeaders += fmt.Sprintf(`"%s": "%s"`, headerSplit[0], headerSplit[1]) - added = true - } - - if count != len(headers)-1 && added { - preparedHeaders += "," - } - } - - preparedHeaders += "}" - } - - fileBalance := "" - fileAdder := `` - fileGrabber := `` - fileParameter := `` - if method == "post" && len(fileField) > 0 { - fileParameter = ", file_id" - fileGrabber = `filedata = self.get_file(file_id)` - - // This indentation is confusing (but correct) ROFL - fileAdder = fmt.Sprintf(`if not filedata["success"]: - return file_id+" is not a valid File ID" - files = {"%s": (filedata["filename"], filedata["data"])}`, fileField) - fileBalance = ", files=files" - } - - // Extra param for url if it's changeable - // Extra param for authentication scheme(s) - // The last weird one is the body.. Tabs & spaces sucks. - data := fmt.Sprintf(` async def %s(self%s%s%s%s%s%s%s): - params={} - %s - url=f"%s%s" - %s - %s - %s - %s - %s - %s - ret = requests.%s(url, headers=headers, params=params%s%s%s%s) - try: - return ret.json() - except json.decoder.JSONDecodeError: - return ret.text - `, - functionname, - authenticationParameter, - urlParameter, - fileParameter, - parameterData, - queryString, - bodyParameter, - verifyParam, - preparedHeaders, - urlInline, - url, - verifyWrapper, - authenticationSetup, - queryData, - bodyFormatter, - fileGrabber, - fileAdder, - method, - authenticationAddin, - bodyAddin, - verifyAddin, - fileBalance, - ) - - // Use lowercase when checking - - if strings.Contains(functionname, "attachment") { - //log.Printf("FUNCTION: %s", data) - //log.Println(data) - //log.Printf("Queries: %s", queryString) - } - - return functionname, data -} - -func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, WorkflowApp, []string, error) { - api := WorkflowApp{} - //log.Printf("%#v", swagger.Info) - - if len(swagger.Info.Title) == 0 { - return swagger, WorkflowApp{}, []string{}, errors.New("Swagger.Info.Title can't be empty.") - } - - if len(swagger.Servers) == 0 { - //return swagger, WorkflowApp{}, []string{}, errors.New("Swagger.Servers can't be empty. Add 'servers':[{'url':'hostname.com'}'") - //return swagger, WorkflowApp{}, []string{}, errors.New("Swagger.Servers can't be empty. Add 'servers':[{'url':'hostname.com'}'") - swagger.Servers = openapi3.Servers{ - &openapi3.Server{ - URL: "https://hostname.com", - }, - } - } - - api.Name = swagger.Info.Title - api.Description = swagger.Info.Description - - // FIXME: Versioning issue? - api.ID = newmd5 - //uuid.NewV4().String() - - api.IsValid = true - api.Link = swagger.Servers[0].URL // host does not exist lol - if strings.HasSuffix(api.Link, "/") { - api.Link = api.Link[:len(api.Link)-1] - } - - api.AppVersion = "1.0.0" - api.Environment = "Shuffle" - api.SmallImage = "" - api.LargeImage = "" - api.Sharing = false - api.Verified = false - api.Tested = false - api.Invalid = false - api.PrivateID = newmd5 - api.Generated = true - api.Activated = true - // Setting up security schemes - extraParameters := []WorkflowAppActionParameter{} - - if val, ok := swagger.Info.ExtensionProps.Extensions["x-logo"]; ok { - j, err := json.Marshal(&val) - if err == nil { - if j[0] == 0x22 && j[len(j)-1] == 0x22 { - j = j[1 : len(j)-1] - } - - //log.Printf("%s", j) - api.SmallImage = string(j) - api.LargeImage = string(j) - } - } - - // Jesus what a clusterfuck. - // Handles parsing of categories from OpenApi3 custom field - if val, ok := swagger.Info.ExtensionProps.Extensions["x-categories"]; ok { - //log.Printf("Categories: %#v", val) - j, err := json.Marshal(&val) - if err == nil { - if j[0] == 0x22 && j[len(j)-1] == 0x22 { - j = j[1 : len(j)-1] - } - - parsedCategories := fmt.Sprintf(`{"categories": %s}`, string(j)) - type parsed struct { - Categories []string `json:"categories"` - } - - var parse parsed - err := json.Unmarshal([]byte(parsedCategories), &parse) - if err != nil { - log.Printf("Failed unmarshaling categories: %v", err) - } else { - api.Categories = parse.Categories - } - } - } - - if len(swagger.Tags) > 0 { - newTags := []string{} - for _, tag := range swagger.Tags { - newTags = append(newTags, tag.Name) - } - - api.Tags = newTags - } - - securitySchemes := swagger.Components.SecuritySchemes - if securitySchemes != nil { - //log.Printf("%#v", securitySchemes) - - api.Authentication = Authentication{ - Required: true, - Parameters: []AuthenticationParams{}, - } - - // Used for python code generation lol - // Not sure how this should work with oauth - if securitySchemes["BearerAuth"] != nil { - api.Authentication.Parameters = append(api.Authentication.Parameters, AuthenticationParams{ - Name: "apikey", - Value: "", - Example: "******", - Description: securitySchemes["BearerAuth"].Value.Description, - In: securitySchemes["BearerAuth"].Value.In, - Scheme: securitySchemes["BearerAuth"].Value.Scheme, - Schema: SchemaDefinition{ - Type: securitySchemes["BearerAuth"].Value.Scheme, - }, - }) - - //log.Printf("HANDLE BEARER AUTH") - extraParameters = append(extraParameters, WorkflowAppActionParameter{ - Name: "apikey", - Description: "The apikey to use", - Multiline: false, - Required: true, - Example: "The API key to use. Space = skip", - Configuration: true, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - } else if securitySchemes["ApiKeyAuth"] != nil { - api.Authentication.Parameters = append(api.Authentication.Parameters, AuthenticationParams{ - Name: "apikey", - Value: "", - Example: "******", - Description: securitySchemes["ApiKeyAuth"].Value.Description, - In: securitySchemes["ApiKeyAuth"].Value.In, - Scheme: securitySchemes["ApiKeyAuth"].Value.Scheme, - Schema: SchemaDefinition{ - Type: securitySchemes["ApiKeyAuth"].Value.Scheme, - }, - }) - - //log.Printf("HANDLE APIKEY AUTH") - extraParameters = append(extraParameters, WorkflowAppActionParameter{ - Name: "apikey", - Description: "The apikey to use", - Multiline: false, - Required: true, - Example: "**********", - Configuration: true, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - } else if securitySchemes["BasicAuth"] != nil { - api.Authentication.Parameters = append(api.Authentication.Parameters, AuthenticationParams{ - Name: "username_basic", - Value: "", - Example: "username", - Description: securitySchemes["BasicAuth"].Value.Description, - In: securitySchemes["BasicAuth"].Value.In, - Scheme: securitySchemes["BasicAuth"].Value.Scheme, - Schema: SchemaDefinition{ - Type: securitySchemes["BasicAuth"].Value.Scheme, - }, - }) - - api.Authentication.Parameters = append(api.Authentication.Parameters, AuthenticationParams{ - Name: "password_basic", - Value: "", - Example: "*****", - Description: securitySchemes["BasicAuth"].Value.Description, - In: securitySchemes["BasicAuth"].Value.In, - Scheme: securitySchemes["BasicAuth"].Value.Scheme, - Schema: SchemaDefinition{ - Type: securitySchemes["BasicAuth"].Value.Scheme, - }, - }) - - extraParameters = append(extraParameters, WorkflowAppActionParameter{ - Name: "username_basic", - Description: "The username to use", - Multiline: false, - Required: true, - Example: "The username to use", - Configuration: true, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - extraParameters = append(extraParameters, WorkflowAppActionParameter{ - Name: "password_basic", - Description: "The password to use", - Multiline: false, - Required: true, - Example: "***********", - Configuration: true, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - } - } - - // Adds a link parameter if it's not already defined - if len(api.Link) == 0 { - api.Authentication.Parameters = append(api.Authentication.Parameters, AuthenticationParams{ - Name: "url", - Description: "The URL of the app", - Multiline: false, - Required: true, - Example: "https://shuffler.io", - Schema: SchemaDefinition{ - Type: "string", - }, - }) - - extraParameters = append(extraParameters, WorkflowAppActionParameter{ - Name: "url", - Description: "The URL of the app", - Multiline: false, - Required: true, - Configuration: true, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - } - - // This is the python code to be generated - // Could just as well be go at this point lol - pythonFunctions := []string{} - //Verified bool `json:"verified" yaml:"verified" required:false datastore:"verified"` - for actualPath, path := range swagger.Paths { - actualPath = strings.Replace(actualPath, " ", "_", -1) - //actualPath = strings.Replace(actualPath, ".", "", -1) - actualPath = strings.Replace(actualPath, "\\", "", -1) - if !api.Invalid && strings.HasPrefix(actualPath, "tmp") { - log.Printf("[WARNING] Set api %s to invalid because of path %s", swagger.Info.Title, actualPath) - api.Invalid = true - } - - // FIXME: Handle everything behind questionmark (?) with dots as well. - // https://godoc.org/github.com/getkin/kin-openapi/openapi3#PathItem - if path.Get != nil { - action, curCode := handleGet(swagger, api, extraParameters, path, actualPath) - api.Actions = append(api.Actions, action) - pythonFunctions = append(pythonFunctions, curCode) - } - if path.Connect != nil { - action, curCode := handleConnect(swagger, api, extraParameters, path, actualPath) - api.Actions = append(api.Actions, action) - pythonFunctions = append(pythonFunctions, curCode) - } - if path.Head != nil { - action, curCode := handleHead(swagger, api, extraParameters, path, actualPath) - api.Actions = append(api.Actions, action) - pythonFunctions = append(pythonFunctions, curCode) - } - if path.Delete != nil { - action, curCode := handleDelete(swagger, api, extraParameters, path, actualPath) - api.Actions = append(api.Actions, action) - pythonFunctions = append(pythonFunctions, curCode) - } - if path.Post != nil { - action, curCode := handlePost(swagger, api, extraParameters, path, actualPath) - api.Actions = append(api.Actions, action) - pythonFunctions = append(pythonFunctions, curCode) - } - if path.Patch != nil { - action, curCode := handlePatch(swagger, api, extraParameters, path, actualPath) - api.Actions = append(api.Actions, action) - pythonFunctions = append(pythonFunctions, curCode) - } - if path.Put != nil { - action, curCode := handlePut(swagger, api, extraParameters, path, actualPath) - api.Actions = append(api.Actions, action) - pythonFunctions = append(pythonFunctions, curCode) - } - - // Has to be here because its used differently above. - // FIXING this is done during export instead? - //log.Printf("OLDPATH: %s", actualPath) - //if strings.Contains(actualPath, "?") { - // actualPath = strings.Split(actualPath, "?")[0] - //} - - //log.Printf("NEWPATH: %s", actualPath) - //newPaths[actualPath] = path - } - - return swagger, api, pythonFunctions, nil -} - -// FIXME - have this give a real version? -func verifyApi(api WorkflowApp) WorkflowApp { - if api.AppVersion == "" { - api.AppVersion = "1.0.0" - } - - return api -} - -func getBasePython() string { - baseString := `import requests -import asyncio -import json -import urllib3 - -from walkoff_app_sdk.app_base import AppBase - -class %s(AppBase): - """ - Autogenerated class by Shuffler - """ - - __version__ = "%s" - app_name = "%s" - - def __init__(self, redis, logger, console_logger=None): - self.verify = False - urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - super().__init__(redis, logger, console_logger) - -%s - -if __name__ == "__main__": - asyncio.run(%s.run(), debug=True) -` - return baseString -} - -func dumpPythonGCP(ctx context.Context, client *storage.Client, basePath, name, version string, pythonFunctions []string) (string, error) { - parsedCode := fmt.Sprintf(getBasePython(), name, version, name, strings.Join(pythonFunctions, "\n"), name) - - // Create bucket handle - bucket := client.Bucket(bucketName) - obj := bucket.Object(fmt.Sprintf("%s/src/app.py", basePath)) - w := obj.NewWriter(ctx) - if _, err := fmt.Fprintf(w, parsedCode); err != nil { - return "", err - } - // Close, just like writing a file. - if err := w.Close(); err != nil { - return "", err - } - - return parsedCode, nil -} - -func dumpPython(basePath, name, version string, pythonFunctions []string) (string, error) { - //log.Printf("%#v", api) - //log.Printf(strings.Join(pythonFunctions, "\n")) - - parsedCode := fmt.Sprintf(getBasePython(), name, version, name, strings.Join(pythonFunctions, "\n"), name) - - err := ioutil.WriteFile(fmt.Sprintf("%s/src/app.py", basePath), []byte(parsedCode), os.ModePerm) - if err != nil { - return "", err - } - //fmt.Println(parsedCode) - //log.Println(string(data)) - return parsedCode, nil -} - -func dumpApiGCP(ctx context.Context, client *storage.Client, swagger *openapi3.Swagger, basePath string, api WorkflowApp) error { - //log.Printf("%#v", api) - data, err := yaml.Marshal(api) - if err != nil { - log.Printf("Error with yaml marshal: %s", err) - return err - } - - // Create bucket handle - bucket := client.Bucket(bucketName) - obj := bucket.Object(fmt.Sprintf("%s/app.yaml", basePath)) - w := obj.NewWriter(ctx) - if _, err := fmt.Fprintln(w, string(data)); err != nil { - return err - } - // Close, just like writing a file. - if err := w.Close(); err != nil { - return err - } - - openapidata, err := yaml.Marshal(swagger) - if err != nil { - log.Printf("Error with yaml marshal: %s", err) - return err - } - obj = bucket.Object(fmt.Sprintf("%s/openapi.yaml", basePath)) - //log.Println(string(openapidata)) - w = obj.NewWriter(ctx) - if _, err := fmt.Fprintln(w, string(openapidata)); err != nil { - return err - } - // Close, just like writing a file. - if err := w.Close(); err != nil { - return err - } - - //log.Println(string(data)) - return nil -} - -func dumpApi(basePath string, api WorkflowApp) error { - //log.Printf("%#v", api) - data, err := yaml.Marshal(api) - if err != nil { - log.Printf("Error with yaml marshal: %s", err) - return err - } - - err = ioutil.WriteFile(fmt.Sprintf("%s/api.yaml", basePath), []byte(data), os.ModePerm) - if err != nil { - return err - } - - //log.Println(string(data)) - return nil -} - -func getRunner(classname string) string { - return fmt.Sprintf(` -# Run the actual thing after we've checked params -def run(request): - print("Started execution!") - action = request.get_json() - print(action) - print(type(action)) - authorization_key = action.get("authorization") - current_execution_id = action.get("execution_id") - - if action and "name" in action and "app_name" in action: - asyncio.run(%s.run(action), debug=True) - return f'Attempting to execute function {action["name"]} in app {action["app_name"]}' - else: - return f'Invalid action' - - `, classname) -} - -func deployAppToDatastore(ctx context.Context, workflowapp WorkflowApp) error { - err := setWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) - if err != nil { - log.Printf("[ERROR] Failed setting workflowapp: %s", err) - return err - } else { - log.Printf("[INFO] Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion) - } - - return nil -} - -// FIXME: -// https://docs.python.org/3.2/reference/lexical_analysis.html#identifiers -// This is used to build the python functions. -func fixFunctionName(functionName, actualPath string) string { - if len(functionName) == 0 { - functionName = actualPath - } - - // REGEX THIS SHIT - // ROFL - - //log.Printf("Fixing function name for %s", functionName) - functionName = strings.Replace(functionName, ".", "", -1) - functionName = strings.Replace(functionName, ",", "", -1) - functionName = strings.Replace(functionName, ".", "", -1) - functionName = strings.Replace(functionName, "&", "", -1) - functionName = strings.Replace(functionName, "/", "", -1) - functionName = strings.Replace(functionName, "\\", "", -1) - - functionName = strings.Replace(functionName, "!", "", -1) - functionName = strings.Replace(functionName, "?", "", -1) - functionName = strings.Replace(functionName, "@", "", -1) - functionName = strings.Replace(functionName, "#", "", -1) - functionName = strings.Replace(functionName, "$", "", -1) - functionName = strings.Replace(functionName, "&", "", -1) - functionName = strings.Replace(functionName, "*", "", -1) - functionName = strings.Replace(functionName, "(", "", -1) - functionName = strings.Replace(functionName, ")", "", -1) - functionName = strings.Replace(functionName, "[", "", -1) - functionName = strings.Replace(functionName, "]", "", -1) - functionName = strings.Replace(functionName, "{", "", -1) - functionName = strings.Replace(functionName, "}", "", -1) - functionName = strings.Replace(functionName, `"`, "", -1) - functionName = strings.Replace(functionName, `'`, "", -1) - functionName = strings.Replace(functionName, `|`, "", -1) - functionName = strings.Replace(functionName, `~`, "", -1) - - functionName = strings.Replace(functionName, " ", "_", -1) - functionName = strings.Replace(functionName, "-", "_", -1) - - functionName = strings.ToLower(functionName) - - return functionName -} - -// Returns a valid param name -func validateParameterName(name string) string { - invalid := []string{"False", - "await", - "else", - "import", - "pass", - "None", - "break", - "except", - "in", - "raise", - "True", - "class", - "finally", - "is", - "return", - "and", - "continue", - "for", - "lambda", - "try", - "as", - "def", - "from", - "nonlocal", - "while", - "assert", - "del", - "global", - "not", - "with", - "async", - "elif", - "if", - "or", - "yield", - } - - newname := name - for _, item := range invalid { - if item == name { - //log.Printf("%s is NOT a valid parameter name!", item) - newname = fmt.Sprintf("%s_shuffle", item) - break - } - } - - newname = strings.ReplaceAll(newname, " ", "_") - newname = strings.ReplaceAll(newname, ",", "_") - newname = strings.ReplaceAll(newname, ".", "_") - newname = strings.ReplaceAll(newname, "|", "_") - newname = strings.ReplaceAll(newname, "-", "_") - - return newname -} - -func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []WorkflowAppActionParameter, path *openapi3.PathItem, actualPath string) (WorkflowAppAction, string) { - // What to do with this, hmm - functionName := fixFunctionName(path.Connect.Summary, actualPath) - - action := WorkflowAppAction{ - Description: path.Connect.Description, - Name: fmt.Sprintf("%s %s", "Connect", path.Connect.Summary), - Label: fmt.Sprintf(path.Connect.Summary), - NodeType: "action", - Environment: api.Environment, - Parameters: extraParameters, - } - - action.Returns.Schema.Type = "string" - baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) - - //log.Println(path.Parameters) - - // Parameters: []WorkflowAppActionParameter{}, - // FIXME - add data for POST stuff - firstQuery := true - optionalQueries := []string{} - parameters := []string{} - optionalParameters := []WorkflowAppActionParameter{} - - headersFound := []string{} - if len(path.Connect.Parameters) > 0 { - for counter, param := range path.Connect.Parameters { - if param.Value.Schema == nil { - continue - } else if param.Value.In == "header" { - headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) - continue - } - - parsedName := param.Value.Name - parsedName = strings.ReplaceAll(parsedName, " ", "_") - parsedName = strings.ReplaceAll(parsedName, ",", "_") - parsedName = strings.ReplaceAll(parsedName, ".", "_") - parsedName = strings.ReplaceAll(parsedName, "|", "_") - parsedName = strings.ReplaceAll(parsedName, "-", "_") - parsedName = validateParameterName(parsedName) - param.Value.Name = parsedName - path.Connect.Parameters[counter].Value.Name = parsedName - - curParam := WorkflowAppActionParameter{ - Name: parsedName, - Description: param.Value.Description, - Multiline: false, - Required: param.Value.Required, - Schema: SchemaDefinition{ - Type: param.Value.Schema.Value.Type, - }, - } - - // FIXME: Example & Multiline - if param.Value.Example != nil { - curParam.Example = param.Value.Example.(string) - - if param.Value.Name == "body" { - curParam.Value = param.Value.Example.(string) - } - } - if val, ok := param.Value.ExtensionProps.Extensions["multiline"]; ok { - j, err := json.Marshal(&val) - if err == nil { - b, err := strconv.ParseBool(string(j)) - if err == nil { - curParam.Multiline = b - } - } - } - - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - - if param.Value.In == "path" { - parameters = append(parameters, curParam.Name) - //baseUrl = fmt.Sprintf("%s%s", baseUrl) - } else if param.Value.In == "query" { - //log.Printf("QUERY!: %s", param.Value.Name) - if !param.Value.Required { - optionalQueries = append(optionalQueries, param.Value.Name) - continue - } - - parameters = append(parameters, param.Value.Name) - - if strings.Contains(baseUrl, fmt.Sprintf("%s={%s}", param.Value.Name, param.Value.Name)) { - continue - } - - if strings.Contains(baseUrl, fmt.Sprintf("{%s}", param.Value.Name)) { - continue - } - - if firstQuery && !strings.Contains(baseUrl, "?") { - baseUrl = fmt.Sprintf("%s?%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } else { - baseUrl = fmt.Sprintf("%s&%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } - firstQuery = false - } - - } - } - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Options: []string{ - "True", - "False", - }, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - - // ensuring that they end up last in the specification - // (order is ish important for optional params) - they need to be last. - for _, optionalParam := range optionalParameters { - action.Parameters = append(action.Parameters, optionalParam) - } - - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "connect", parameters, optionalQueries, headersFound, "") - - if len(functionname) > 0 { - action.Name = functionname - } - - return action, curCode -} - -func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []WorkflowAppActionParameter, path *openapi3.PathItem, actualPath string) (WorkflowAppAction, string) { - // What to do with this, hmm - functionName := fixFunctionName(path.Get.Summary, actualPath) - - action := WorkflowAppAction{ - Description: path.Get.Description, - Name: fmt.Sprintf("%s %s", "Get", path.Get.Summary), - Label: fmt.Sprintf(path.Get.Summary), - NodeType: "action", - Environment: api.Environment, - Parameters: extraParameters, - } - - action.Returns.Schema.Type = "string" - baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) - - // Parameters: []WorkflowAppActionParameter{}, - // FIXME - add data for POST stuff - firstQuery := true - optionalQueries := []string{} - - // FIXME - remove this when authentication is properly introduced - parameters := []string{} - optionalParameters := []WorkflowAppActionParameter{} - - headersFound := []string{} - if len(path.Get.Parameters) > 0 { - for counter, param := range path.Get.Parameters { - if param.Value.Schema == nil { - continue - } else if param.Value.In == "header" { - headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) - continue - } - - parsedName := param.Value.Name - parsedName = strings.ReplaceAll(parsedName, " ", "_") - parsedName = strings.ReplaceAll(parsedName, ",", "_") - parsedName = strings.ReplaceAll(parsedName, ".", "_") - parsedName = strings.ReplaceAll(parsedName, "|", "_") - parsedName = validateParameterName(parsedName) - param.Value.Name = parsedName - path.Get.Parameters[counter].Value.Name = parsedName - - curParam := WorkflowAppActionParameter{ - Name: parsedName, - Description: param.Value.Description, - Multiline: false, - Required: param.Value.Required, - Schema: SchemaDefinition{ - Type: param.Value.Schema.Value.Type, - }, - } - - // FIXME: Example & Multiline - if param.Value.Example != nil { - curParam.Example = param.Value.Example.(string) - - if param.Value.Name == "body" { - curParam.Value = param.Value.Example.(string) - } - } - if val, ok := param.Value.ExtensionProps.Extensions["multiline"]; ok { - j, err := json.Marshal(&val) - if err == nil { - b, err := strconv.ParseBool(string(j)) - if err == nil { - curParam.Multiline = b - } - } - } - - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - - if param.Value.In == "path" { - parameters = append(parameters, curParam.Name) - //baseUrl = fmt.Sprintf("%s%s", baseUrl) - } else if param.Value.In == "query" { - //log.Printf("QUERY!: %s", param.Value.Name) - if !param.Value.Required { - optionalQueries = append(optionalQueries, param.Value.Name) - continue - } - - parameters = append(parameters, param.Value.Name) - - // Skipping simial - if strings.Contains(baseUrl, fmt.Sprintf("%s={%s}", param.Value.Name, param.Value.Name)) { - continue - } - - if strings.Contains(baseUrl, fmt.Sprintf("{%s}", param.Value.Name)) { - continue - } - - if firstQuery && !strings.Contains(baseUrl, "?") { - baseUrl = fmt.Sprintf("%s?%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } else { - baseUrl = fmt.Sprintf("%s&%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } - firstQuery = false - } - - } - } - - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Options: []string{ - "True", - "False", - }, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - - // ensuring that they end up last in the specification - // (order is ish important for optional params) - they need to be last. - for _, optionalParam := range optionalParameters { - action.Parameters = append(action.Parameters, optionalParam) - } - - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "get", parameters, optionalQueries, headersFound, "") - - if len(functionname) > 0 { - action.Name = functionname - } - - return action, curCode -} - -func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []WorkflowAppActionParameter, path *openapi3.PathItem, actualPath string) (WorkflowAppAction, string) { - // What to do with this, hmm - functionName := fixFunctionName(path.Head.Summary, actualPath) - - action := WorkflowAppAction{ - Description: path.Head.Description, - Name: fmt.Sprintf("%s %s", "Head", path.Head.Summary), - Label: fmt.Sprintf(path.Head.Summary), - NodeType: "action", - Environment: api.Environment, - Parameters: extraParameters, - } - - action.Returns.Schema.Type = "string" - baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) - - //log.Println(path.Parameters) - - // Parameters: []WorkflowAppActionParameter{}, - // FIXME - add data for POST stuff - firstQuery := true - optionalQueries := []string{} - parameters := []string{} - optionalParameters := []WorkflowAppActionParameter{} - - headersFound := []string{} - if len(path.Head.Parameters) > 0 { - for counter, param := range path.Head.Parameters { - if param.Value.Schema == nil { - continue - } else if param.Value.In == "header" { - headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) - continue - } - - parsedName := param.Value.Name - parsedName = strings.ReplaceAll(parsedName, " ", "_") - parsedName = strings.ReplaceAll(parsedName, ",", "_") - parsedName = strings.ReplaceAll(parsedName, ".", "_") - parsedName = strings.ReplaceAll(parsedName, "|", "_") - parsedName = validateParameterName(parsedName) - param.Value.Name = parsedName - path.Head.Parameters[counter].Value.Name = parsedName - - curParam := WorkflowAppActionParameter{ - Name: parsedName, - Description: param.Value.Description, - Multiline: false, - Required: param.Value.Required, - Schema: SchemaDefinition{ - Type: param.Value.Schema.Value.Type, - }, - } - - // FIXME: Example & Multiline - if param.Value.Example != nil { - curParam.Example = param.Value.Example.(string) - - if param.Value.Name == "body" { - curParam.Value = param.Value.Example.(string) - } - } - if val, ok := param.Value.ExtensionProps.Extensions["multiline"]; ok { - j, err := json.Marshal(&val) - if err == nil { - b, err := strconv.ParseBool(string(j)) - if err == nil { - curParam.Multiline = b - } - } - } - - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - - if param.Value.In == "path" { - parameters = append(parameters, curParam.Name) - //baseUrl = fmt.Sprintf("%s%s", baseUrl) - } else if param.Value.In == "query" { - //log.Printf("QUERY!: %s", param.Value.Name) - if !param.Value.Required { - optionalQueries = append(optionalQueries, param.Value.Name) - continue - } - - parameters = append(parameters, param.Value.Name) - - if strings.Contains(baseUrl, fmt.Sprintf("%s={%s}", param.Value.Name, param.Value.Name)) { - continue - } - - if strings.Contains(baseUrl, fmt.Sprintf("{%s}", param.Value.Name)) { - continue - } - - if firstQuery && !strings.Contains(baseUrl, "?") { - baseUrl = fmt.Sprintf("%s?%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } else { - baseUrl = fmt.Sprintf("%s&%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } - firstQuery = false - } - } - } - - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Options: []string{ - "True", - "False", - }, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - - // ensuring that they end up last in the specification - // (order is ish important for optional params) - they need to be last. - for _, optionalParam := range optionalParameters { - action.Parameters = append(action.Parameters, optionalParam) - } - - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "head", parameters, optionalQueries, headersFound, "") - - if len(functionname) > 0 { - action.Name = functionname - } - - return action, curCode -} - -func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []WorkflowAppActionParameter, path *openapi3.PathItem, actualPath string) (WorkflowAppAction, string) { - // What to do with this, hmm - functionName := fixFunctionName(path.Delete.Summary, actualPath) - - action := WorkflowAppAction{ - Description: path.Delete.Description, - Name: fmt.Sprintf("%s %s", "Delete", path.Delete.Summary), - Label: fmt.Sprintf(path.Delete.Summary), - NodeType: "action", - Environment: api.Environment, - Parameters: extraParameters, - } - - action.Returns.Schema.Type = "string" - baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) - - //log.Println(path.Parameters) - - // Parameters: []WorkflowAppActionParameter{}, - // FIXME - add data for POST stuff - firstQuery := true - optionalQueries := []string{} - parameters := []string{} - optionalParameters := []WorkflowAppActionParameter{} - - headersFound := []string{} - if len(path.Delete.Parameters) > 0 { - for counter, param := range path.Delete.Parameters { - if param.Value.Schema == nil { - continue - } else if param.Value.In == "header" { - headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) - continue - } - - parsedName := param.Value.Name - parsedName = strings.ReplaceAll(parsedName, " ", "_") - parsedName = strings.ReplaceAll(parsedName, ",", "_") - parsedName = strings.ReplaceAll(parsedName, ".", "_") - parsedName = strings.ReplaceAll(parsedName, "|", "_") - parsedName = validateParameterName(parsedName) - param.Value.Name = parsedName - path.Delete.Parameters[counter].Value.Name = parsedName - - curParam := WorkflowAppActionParameter{ - Name: parsedName, - Description: param.Value.Description, - Multiline: false, - Required: param.Value.Required, - Schema: SchemaDefinition{ - Type: param.Value.Schema.Value.Type, - }, - } - - // FIXME: Example & Multiline - if param.Value.Example != nil { - curParam.Example = param.Value.Example.(string) - - if param.Value.Name == "body" { - curParam.Value = param.Value.Example.(string) - } - } - if val, ok := param.Value.ExtensionProps.Extensions["multiline"]; ok { - j, err := json.Marshal(&val) - if err == nil { - b, err := strconv.ParseBool(string(j)) - if err == nil { - curParam.Multiline = b - } - } - } - - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - - if param.Value.In == "path" { - parameters = append(parameters, curParam.Name) - //baseUrl = fmt.Sprintf("%s%s", baseUrl) - } else if param.Value.In == "query" { - //log.Printf("QUERY!: %s", param.Value.Name) - if !param.Value.Required { - optionalQueries = append(optionalQueries, param.Value.Name) - continue - } - - parameters = append(parameters, param.Value.Name) - - if strings.Contains(baseUrl, fmt.Sprintf("%s={%s}", param.Value.Name, param.Value.Name)) { - continue - } - - if strings.Contains(baseUrl, fmt.Sprintf("{%s}", param.Value.Name)) { - continue - } - - if firstQuery && !strings.Contains(baseUrl, "?") { - baseUrl = fmt.Sprintf("%s?%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } else { - baseUrl = fmt.Sprintf("%s&%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } - firstQuery = false - } - - } - } - - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Options: []string{ - "True", - "False", - }, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - - // ensuring that they end up last in the specification - // (order is ish important for optional params) - they need to be last. - for _, optionalParam := range optionalParameters { - action.Parameters = append(action.Parameters, optionalParam) - } - - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "delete", parameters, optionalQueries, headersFound, "") - - if len(functionname) > 0 { - action.Name = functionname - } - - return action, curCode -} - -func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []WorkflowAppActionParameter, path *openapi3.PathItem, actualPath string) (WorkflowAppAction, string) { - // What to do with this, hmm - //log.Printf("PATH: %s", actualPath) - functionName := fixFunctionName(path.Post.Summary, actualPath) - - action := WorkflowAppAction{ - Description: path.Post.Description, - Name: fmt.Sprintf("%s %s", "Post", path.Post.Summary), - Label: fmt.Sprintf(path.Post.Summary), - NodeType: "action", - Environment: api.Environment, - Parameters: extraParameters, - } - - action.Returns.Schema.Type = "string" - baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) - - // Parameters: []WorkflowAppActionParameter{}, - // FIXME - add data for POST stuff - firstQuery := true - optionalQueries := []string{} - parameters := []string{} - optionalParameters := []WorkflowAppActionParameter{} - - fileField := "" - if path.Post.RequestBody != nil { - //log.Printf("DATA: %#v", - value := path.Post.RequestBody.Value - //log.Printf("VAL: %#v", value.Content) - if val, ok := value.Content["multipart/form-data"]; ok { - if val.Schema.Value != nil { - if innerval, ok := val.Schema.Value.Properties["fieldname"]; ok { - if extensionvalue, ok := innerval.Value.ExtensionProps.Extensions["value"]; ok { - fieldname := extensionvalue.(json.RawMessage) - newName := string(fmt.Sprintf("%s", string(fieldname))) - if newName[0] == 0x22 && newName[len(newName)-1] == 0x22 { - parsedName := newName[1 : len(newName)-1] - //log.Printf("[INFO] Parse name: %s", parsedName) - fileField = parsedName - - curParam := WorkflowAppActionParameter{ - Name: "file_id", - Description: "Files to be uploaded", - Multiline: false, - Required: true, - Schema: SchemaDefinition{ - Type: "string", - }, - } - - action.Parameters = append(action.Parameters, curParam) - } - } - } - } - } - } - - headersFound := []string{} - if len(path.Post.Parameters) > 0 { - for counter, param := range path.Post.Parameters { - if param.Value.Schema == nil { - continue - } else if param.Value.In == "header" { - headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) - continue - } - - parsedName := param.Value.Name - parsedName = strings.ReplaceAll(parsedName, " ", "_") - parsedName = strings.ReplaceAll(parsedName, ",", "_") - parsedName = strings.ReplaceAll(parsedName, ".", "_") - parsedName = strings.ReplaceAll(parsedName, "|", "_") - parsedName = validateParameterName(parsedName) - param.Value.Name = parsedName - path.Post.Parameters[counter].Value.Name = parsedName - - curParam := WorkflowAppActionParameter{ - Name: parsedName, - Description: param.Value.Description, - Multiline: false, - Required: param.Value.Required, - Schema: SchemaDefinition{ - Type: param.Value.Schema.Value.Type, - }, - } - - // FIXME: Example & Multiline - if param.Value.Example != nil { - curParam.Example = param.Value.Example.(string) - - if parsedName == "body" { - curParam.Value = param.Value.Example.(string) - } - } - if val, ok := param.Value.ExtensionProps.Extensions["multiline"]; ok { - j, err := json.Marshal(&val) - if err == nil { - b, err := strconv.ParseBool(string(j)) - if err == nil { - curParam.Multiline = b - } - } - } - - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - - if param.Value.In == "path" { - parameters = append(parameters, curParam.Name) - //baseUrl = fmt.Sprintf("%s%s", baseUrl) - } else if param.Value.In == "query" { - //log.Printf("QUERY!: %s", param.Value.Name) - if !param.Value.Required { - optionalQueries = append(optionalQueries, param.Value.Name) - continue - } - - parameters = append(parameters, param.Value.Name) - - if strings.Contains(baseUrl, fmt.Sprintf("%s={%s}", param.Value.Name, param.Value.Name)) { - continue - } - - if strings.Contains(baseUrl, fmt.Sprintf("{%s}", param.Value.Name)) { - continue - } - - if firstQuery && !strings.Contains(baseUrl, "?") { - baseUrl = fmt.Sprintf("%s?%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } else { - baseUrl = fmt.Sprintf("%s&%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } - firstQuery = false - } - } - } - - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Options: []string{ - "True", - "False", - }, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - - // ensuring that they end up last in the specification - // (order is ish important for optional params) - they need to be last. - for _, optionalParam := range optionalParameters { - action.Parameters = append(action.Parameters, optionalParam) - } - - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "post", parameters, optionalQueries, headersFound, fileField) - - if len(functionname) > 0 { - action.Name = functionname - } - - //log.Printf("PARAMS: %d", len(action.Parameters)) - //for _, param := range action.Parameters { - // log.Printf("%#v", param) - //} - - return action, curCode -} - -func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []WorkflowAppActionParameter, path *openapi3.PathItem, actualPath string) (WorkflowAppAction, string) { - // What to do with this, hmm - functionName := fixFunctionName(path.Patch.Summary, actualPath) - - action := WorkflowAppAction{ - Description: path.Patch.Description, - Name: fmt.Sprintf("%s %s", "Patch", path.Patch.Summary), - Label: fmt.Sprintf(path.Patch.Summary), - NodeType: "action", - Environment: api.Environment, - Parameters: extraParameters, - } - - action.Returns.Schema.Type = "string" - baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) - - //log.Println(path.Parameters) - - // Parameters: []WorkflowAppActionParameter{}, - // FIXME - add data for POST stuff - firstQuery := true - optionalQueries := []string{} - parameters := []string{} - optionalParameters := []WorkflowAppActionParameter{} - - headersFound := []string{} - if len(path.Patch.Parameters) > 0 { - for counter, param := range path.Patch.Parameters { - if param.Value.Schema == nil { - continue - } else if param.Value.In == "header" { - headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) - continue - } - - parsedName := param.Value.Name - parsedName = strings.ReplaceAll(parsedName, " ", "_") - parsedName = strings.ReplaceAll(parsedName, ",", "_") - parsedName = strings.ReplaceAll(parsedName, ".", "_") - parsedName = strings.ReplaceAll(parsedName, "|", "_") - parsedName = validateParameterName(parsedName) - param.Value.Name = parsedName - path.Patch.Parameters[counter].Value.Name = parsedName - - curParam := WorkflowAppActionParameter{ - Name: parsedName, - Description: param.Value.Description, - Multiline: false, - Required: param.Value.Required, - Schema: SchemaDefinition{ - Type: param.Value.Schema.Value.Type, - }, - } - - // FIXME: Example & Multiline - if param.Value.Example != nil { - curParam.Example = param.Value.Example.(string) - - if param.Value.Name == "body" { - curParam.Value = param.Value.Example.(string) - } - } - if val, ok := param.Value.ExtensionProps.Extensions["multiline"]; ok { - j, err := json.Marshal(&val) - if err == nil { - b, err := strconv.ParseBool(string(j)) - if err == nil { - curParam.Multiline = b - } - } - } - - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - - if param.Value.In == "path" { - parameters = append(parameters, curParam.Name) - //baseUrl = fmt.Sprintf("%s%s", baseUrl) - } else if param.Value.In == "query" { - //log.Printf("QUERY!: %s", param.Value.Name) - if !param.Value.Required { - optionalQueries = append(optionalQueries, param.Value.Name) - continue - } - - parameters = append(parameters, param.Value.Name) - - if strings.Contains(baseUrl, fmt.Sprintf("%s={%s}", param.Value.Name, param.Value.Name)) { - continue - } - - if strings.Contains(baseUrl, fmt.Sprintf("{%s}", param.Value.Name)) { - continue - } - - if firstQuery && !strings.Contains(baseUrl, "?") { - baseUrl = fmt.Sprintf("%s?%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } else { - baseUrl = fmt.Sprintf("%s&%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } - firstQuery = false - } - } - } - - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Options: []string{ - "True", - "False", - }, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - - // ensuring that they end up last in the specification - // (order is ish important for optional params) - they need to be last. - for _, optionalParam := range optionalParameters { - action.Parameters = append(action.Parameters, optionalParam) - } - - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "patch", parameters, optionalQueries, headersFound, "") - - if len(functionname) > 0 { - action.Name = functionname - } - - return action, curCode -} - -func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []WorkflowAppActionParameter, path *openapi3.PathItem, actualPath string) (WorkflowAppAction, string) { - // What to do with this, hmm - functionName := fixFunctionName(path.Put.Summary, actualPath) - - action := WorkflowAppAction{ - Description: path.Put.Description, - Name: fmt.Sprintf("%s %s", "Put", path.Put.Summary), - Label: fmt.Sprintf(path.Put.Summary), - NodeType: "action", - Environment: api.Environment, - Parameters: extraParameters, - } - - action.Returns.Schema.Type = "string" - baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) - - //log.Println(path.Parameters) - - // Parameters: []WorkflowAppActionParameter{}, - // FIXME - add data for POST stuff - firstQuery := true - optionalQueries := []string{} - parameters := []string{} - optionalParameters := []WorkflowAppActionParameter{} - - headersFound := []string{} - if len(path.Put.Parameters) > 0 { - for counter, param := range path.Put.Parameters { - if param.Value.Schema == nil { - continue - } else if param.Value.In == "header" { - headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) - continue - } - - parsedName := param.Value.Name - parsedName = strings.ReplaceAll(parsedName, " ", "_") - parsedName = strings.ReplaceAll(parsedName, ",", "_") - parsedName = strings.ReplaceAll(parsedName, ".", "_") - parsedName = strings.ReplaceAll(parsedName, "|", "_") - parsedName = validateParameterName(parsedName) - param.Value.Name = parsedName - path.Put.Parameters[counter].Value.Name = parsedName - - curParam := WorkflowAppActionParameter{ - Name: parsedName, - Description: param.Value.Description, - Multiline: false, - Required: param.Value.Required, - Schema: SchemaDefinition{ - Type: param.Value.Schema.Value.Type, - }, - } - - // FIXME: Example & Multiline - if param.Value.Example != nil { - curParam.Example = param.Value.Example.(string) - - if param.Value.Name == "body" { - curParam.Value = param.Value.Example.(string) - } - } - if val, ok := param.Value.ExtensionProps.Extensions["multiline"]; ok { - j, err := json.Marshal(&val) - if err == nil { - b, err := strconv.ParseBool(string(j)) - if err == nil { - curParam.Multiline = b - } - } - } - - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - - if param.Value.In == "path" { - parameters = append(parameters, param.Value.Name) - //baseUrl = fmt.Sprintf("%s%s", baseUrl) - } else if param.Value.In == "query" { - //log.Printf("QUERY!: %s", param.Value.Name) - if !param.Value.Required { - optionalQueries = append(optionalQueries, param.Value.Name) - continue - } - - parameters = append(parameters, param.Value.Name) - - if strings.Contains(baseUrl, fmt.Sprintf("%s={%s}", param.Value.Name, param.Value.Name)) { - continue - } - - if strings.Contains(baseUrl, fmt.Sprintf("{%s}", param.Value.Name)) { - continue - } - - if firstQuery && !strings.Contains(baseUrl, "?") { - baseUrl = fmt.Sprintf("%s?%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } else { - baseUrl = fmt.Sprintf("%s&%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } - firstQuery = false - } - - } - } - - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Options: []string{ - "True", - "False", - }, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - - // ensuring that they end up last in the specification - // (order is ish important for optional params) - they need to be last. - for _, optionalParam := range optionalParameters { - action.Parameters = append(action.Parameters, optionalParam) - } - - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "put", parameters, optionalQueries, headersFound, "") - - if len(functionname) > 0 { - action.Name = functionname - } - - return action, curCode -} diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index 27c8e436..e11934f8 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -2,6 +2,8 @@ package main // Docker import ( + "github.com/frikky/shuffle-shared" + "archive/tar" "path/filepath" @@ -797,7 +799,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { } // Just here to verify that the user is logged in - _, err := handleApiAuthentication(resp, request) + _, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in validate swagger: %s", err) resp.WriteHeader(401) diff --git a/backend/go-app/files.go b/backend/go-app/files.go deleted file mode 100644 index 419d834d..00000000 --- a/backend/go-app/files.go +++ /dev/null @@ -1,882 +0,0 @@ -package main - -/* - Handles files within Workflows.of Shuffle -*/ - -import ( - "bytes" - "context" - "crypto/sha256" - "encoding/json" - "errors" - "fmt" - "io" - "io/ioutil" - "log" - "net/http" - "os" - "strconv" - "strings" - "time" - - "cloud.google.com/go/datastore" - "github.com/satori/go.uuid" -) - -type File struct { - Id string `json:"id" datastore:"id"` - Type string `json:"type" datastore:"type"` - CreatedAt int64 `json:"created_at" datastore:"created_at"` - UpdatedAt int64 `json:"updated_at" datastore:"updated_at"` - MetaAccessAt int64 `json:"meta_access_at" datastore:"meta_access_at"` - DownloadAt int64 `json:"last_downloaded" datastore:"last_downloaded"` - Description string `json:"description" datastore:"description"` - ExpiresAt string `json:"expires_at" datastore:"expires_at"` - Status string `json:"status" datastore:"status"` - Filename string `json:"filename" datastore:"filename"` - URL string `json:"url" datastore:"org"` - OrgId string `json:"org_id" datastore:"org_id"` - WorkflowId string `json:"workflow_id" datastore:"workflow_id"` - Workflows []string `json:"workflows" datastore:"workflows"` - DownloadPath string `json:"download_path" datastore:"download_path"` - Md5sum string `json:"md5_sum" datastore:"md5_sum"` - Sha256sum string `json:"sha256_sum" datastore:"sha256_sum"` - FileSize int64 `json:"filesize" datastore:"filesize"` - Duplicate bool `json:"duplicate" datastore:"duplicate"` - Subflows []string `json:"subflows" datastore:"subflows"` -} - -var basepath = os.Getenv("SHUFFLE_FILE_LOCATION") - -func fileAuthentication(request *http.Request) (string, error) { - executionId, ok := request.URL.Query()["execution_id"] - if ok && len(executionId) > 0 { - ctx := context.Background() - workflowExecution, err := getWorkflowExecution(ctx, executionId[0]) - if err != nil { - log.Printf("[ERROR] Couldn't find execution ID %s", executionId[0]) - return "", err - } - - apikey := request.Header.Get("Authorization") - if !strings.HasPrefix(apikey, "Bearer ") { - log.Printf("[ERROR} Apikey doesn't start with bearer (2)") - return "", errors.New("No auth key found") - } - - apikeyCheck := strings.Split(apikey, " ") - if len(apikeyCheck) != 2 { - log.Printf("[ERROR] Invalid format for apikey (2)") - return "", errors.New("No space in authkey") - } - - // This is annoying af and is done because of maxlength lol - newApikey := apikeyCheck[1] - if newApikey != workflowExecution.Authorization { - //log.Printf("[ERROR] Bad apikey for execution %s. %s vs %s", executionId[0], apikey, workflowExecution.Authorization) - log.Printf("[ERROR] Bad apikey for execution %s.", executionId[0]) - //%s vs %s", executionId[0], apikey, workflowExecution.Authorization) - return "", errors.New("Bad authorization key") - } - - log.Printf("[INFO] Authorization is correct for execution %s!", executionId[0]) - //%s vs %s. Setting Org", executionId, apikey, workflowExecution.Authorization) - if len(workflowExecution.ExecutionOrg) > 0 { - return workflowExecution.ExecutionOrg, nil - } else if len(workflowExecution.Workflow.ExecutingOrg.Id) > 0 { - return workflowExecution.ExecutionOrg, nil - } else { - log.Printf("[ERROR] Couldn't find org for workflow execution, but auth was correct.") - } - } - - return "", errors.New("No execution id specified") -} - -// https://golangcode.com/check-if-a-file-exists/ -func fileExists(filename string) bool { - info, err := os.Stat(filename) - if os.IsNotExist(err) { - return false - } - return !info.IsDir() -} - -func handleGetFiles(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - // 1. Check user directly - // 2. Check workflow execution authorization - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("[INFO] INITIAL Api authentication failed in file LIST: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if user.Role != "admin" { - log.Printf("[AUTH] User isn't admin") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Need to be admin"}`))) - return - } - - ctx := context.Background() - files, err := getAllFiles(ctx, user.ActiveOrg.Id) - if err != nil { - log.Printf("[ERROR] Failed to get files: %s", err) - resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error getting files."}`))) - return - } - - log.Printf("[INFO] Got %d files for org %s", len(files), user.ActiveOrg.Id) - newBody, err := json.Marshal(files) - if err != nil { - log.Printf("[ERROR] Failed marshaling files: %s", err) - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false, "reason": "Failed to marshal files"}`)) - return - } - - resp.WriteHeader(200) - resp.Write([]byte(newBody)) -} - -func handleGetFileMeta(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - var fileId string - location := strings.Split(request.URL.String(), "/") - if location[1] == "api" { - if len(location) <= 4 { - log.Printf("[INFO] Path too short: %d", len(location)) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - if strings.Contains(fileId, "?") { - fileId = strings.Split(fileId, "?")[0] - } - - if len(fileId) != 36 { - log.Printf("Bad format for fileId %s", fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`)) - return - } - - log.Printf("\n\n[INFO] User is trying to GET File Meta for %s\n\n", fileId) - - // 1. Check user directly - // 2. Check workflow execution authorization - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("[INFO] INITIAL Api authentication failed in file deletion: %s", err) - - orgId, err := fileAuthentication(request) - if err != nil { - log.Printf("[ERROR] Bad file authentication in get: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - user.ActiveOrg.Id = orgId - user.Username = "Execution File API" - } - - // 1. Verify if the user has access to the file: org_id and workflow - log.Printf("[INFO] Should GET FILE META for %s if user has access", fileId) - ctx := context.Background() - file, err := getFile(ctx, fileId) - if err != nil { - log.Printf("[INFO] File %s not found: %s", fileId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - found := false - if file.OrgId == user.ActiveOrg.Id { - found = true - } else { - for _, item := range user.Orgs { - if item == file.OrgId { - found = true - break - } - } - } - - if !found { - log.Printf("[INFO] User %s doesn't have access to %s", user.Username, fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - newBody, err := json.Marshal(file) - if err != nil { - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false, "reason": "Failed to marshal filedata"}`)) - return - } - - log.Printf("[INFO] Successfully got file meta for %s", fileId) - resp.WriteHeader(200) - resp.Write([]byte(newBody)) -} - -func handleDeleteFile(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - var fileId string - location := strings.Split(request.URL.String(), "/") - if location[1] == "api" { - if len(location) <= 4 { - log.Printf("[INFO] Path too short: %d", len(location)) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - if strings.Contains(fileId, "?") { - fileId = strings.Split(fileId, "?")[0] - } - - if len(fileId) != 36 { - log.Printf("Bad format for fileId %s", fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`)) - return - } - - log.Printf("\n\n[INFO] User is trying to delete file %s\n\n", fileId) - - // 1. Check user directly - // 2. Check workflow execution authorization - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("[INFO] INITIAL Api authentication failed in file deletion: %s", err) - - orgId, err := fileAuthentication(request) - if err != nil { - log.Printf("[ERROR] Bad file authentication in get: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - user.ActiveOrg.Id = orgId - user.Username = "Execution File API" - } - - // 1. Verify if the user has access to the file: org_id and workflow - log.Printf("[INFO] Should DELETE file %s if user has access", fileId) - ctx := context.Background() - file, err := getFile(ctx, fileId) - if err != nil { - log.Printf("[INFO] File %s not found: %s", fileId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - found := false - if file.OrgId == user.ActiveOrg.Id { - found = true - } else { - for _, item := range user.Orgs { - if item == file.OrgId { - found = true - break - } - } - } - - if !found { - log.Printf("[INFO] User %s doesn't have access to %s", user.Username, fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if file.Status == "deleted" { - log.Printf("[INFO] File with ID %s is already deleted.", fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if fileExists(file.DownloadPath) { - err = os.Remove(file.DownloadPath) - if err != nil { - log.Printf("[ERROR] Failed deleting file locally: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed deleting filein path %s"}`, file.DownloadPath))) - return - } - - log.Printf("[INFO] Deleted file %s locally. Next is database.", file.DownloadPath) - } else { - log.Printf("[ERROR] File doesn't exist. Can't delete. Should maybe delete file anyway?") - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "File in location %s doesn't exist"}`, file.DownloadPath))) - return - } - - file.Status = "deleted" - err = setFile(ctx, *file) - if err != nil { - log.Printf("[ERROR] Failed setting file to deleted") - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false, "reason": "Failed setting file to deleted"}`)) - return - } - - /* - //Actually delete it? - err = DeleteKey(ctx, "files", fileId) - if err != nil { - log.Printf("Failed deleting file with ID %s: %s", fileId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - */ - - log.Printf("[INFO] Successfully deleted file %s", fileId) - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true}`)) -} - -func handleGetFileContent(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - var fileId string - location := strings.Split(request.URL.String(), "/") - if location[1] == "api" { - if len(location) <= 4 { - log.Printf("Path too short: %d", len(location)) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - if len(fileId) != 36 { - log.Printf("Bad format for fileId %s", fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`)) - return - } - - log.Printf("\n\n[INFO] User is trying to download file %s\n\n", fileId) - - // 1. Check user directly - // 2. Check workflow execution authorization - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("INITIAL Api authentication failed in file download: %s", err) - - orgId, err := fileAuthentication(request) - if err != nil { - log.Printf("Bad file authentication in get: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - user.ActiveOrg.Id = orgId - user.Username = "Execution File API" - /* - } else { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - */ - } - - // 1. Verify if the user has access to the file: org_id and workflow - log.Printf("[INFO] Should get file %s", fileId) - ctx := context.Background() - file, err := getFile(ctx, fileId) - if err != nil { - log.Printf("[ERROR] File %s not found: %s", fileId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - found := false - if file.OrgId == user.ActiveOrg.Id { - found = true - } else { - for _, item := range user.Orgs { - if item == file.OrgId { - found = true - break - } - } - } - - if !found { - log.Printf("User %s doesn't have access to %s", user.Username, fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if file.Status != "active" { - log.Printf("[ERROR] File status isn't active, but %s. Can't continue.", file.Status) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "The file isn't ready to be downloaded yet. Status required: active"}`)) - return - } - - // Fixme: More auth: org and workflow! - downloadPath := file.DownloadPath - log.Printf("[INFO] Downloadpath: %s", downloadPath) - Openfile, err := os.Open(downloadPath) - defer Openfile.Close() //Close after function return - if err != nil { - file.Status = "deleted" - err = setFile(ctx, *file) - if err != nil { - log.Printf("Failed setting file to uploading") - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false, "reason": "Failed setting file to uploading"}`)) - return - } - - //File not found, send 404 - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "File doesn't exist locally"}`)) - return - } - - //File is found, create and send the correct headers - //Get the Content-Type of the file - //Create a buffer to store the header of the file in - FileHeader := make([]byte, 512) - //Copy the headers into the FileHeader buffer - Openfile.Read(FileHeader) - //Get content type of file - FileContentType := http.DetectContentType(FileHeader) - - //Get the file size - FileStat, _ := Openfile.Stat() //Get info from file - FileSize := strconv.FormatInt(FileStat.Size(), 10) //Get file size as a string - - //Send the headers - resp.Header().Set("Content-Disposition", "attachment; filename="+fileId) - resp.Header().Set("Content-Type", FileContentType) - resp.Header().Set("Content-Length", FileSize) - - //Send the file - //We read 512 bytes from the file already, so we reset the offset back to 0 - Openfile.Seek(0, 0) - io.Copy(resp, Openfile) //'Copy' the file to the client - return - - //log.Printf("Should download file %s", downloadPath) -} -func handleUploadFile(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - var fileId string - location := strings.Split(request.URL.String(), "/") - if location[1] == "api" { - if len(location) <= 4 { - log.Printf("Path too short: %d", len(location)) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - if len(fileId) != 36 { - log.Printf("Bad format for fileId %s", fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`)) - return - } - - // 1. Check user directly - // 2. Check workflow execution authorization - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("INITIAL Api authentication failed in file upload: %s", err) - - orgId, err := fileAuthentication(request) - if err != nil { - log.Printf("Bad file authentication in create file: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - user.ActiveOrg.Id = orgId - user.Username = "Execution File API" - } - - log.Printf("[INFO] Should UPLOAD file %s if user has access", fileId) - ctx := context.Background() - file, err := getFile(ctx, fileId) - if err != nil { - log.Printf("File %s not found: %s", fileId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - found := false - if file.OrgId == user.ActiveOrg.Id { - found = true - } else { - for _, item := range user.Orgs { - if item == file.OrgId { - found = true - break - } - } - } - - if !found { - log.Printf("User %s doesn't have access to %s", user.Username, fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - log.Printf("[INFO] STATUS: %s", file.Status) - if file.Status != "created" { - log.Printf("File status isn't created. Can't upload.") - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "This file already has data."}`)) - return - } - - request.ParseMultipartForm(32 << 20) - parsedFile, _, err := request.FormFile("shuffle_file") - if err != nil { - log.Printf("[ERROR] Couldn't upload file: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Failed uploading file"}`)) - return - } - defer parsedFile.Close() - - file.Status = "uploading" - err = setFile(ctx, *file) - if err != nil { - log.Printf("Failed setting file to uploading") - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false, "reason": "Failed setting file to uploading"}`)) - return - } - - // Can be used for validation files for change - var buf bytes.Buffer - io.Copy(&buf, parsedFile) - contents := buf.Bytes() - file.FileSize = int64(len(contents)) - md5 := md5sum(contents) - buf.Reset() - - sha256Sum := sha256.Sum256(contents) - //parsedFile.Reset() - - f, err := os.OpenFile(file.DownloadPath, os.O_WRONLY|os.O_CREATE, os.ModePerm) - if err != nil { - // Rolling back file - file.Status = "created" - setFile(ctx, *file) - - log.Printf("[ERROR] Failed uploading and creating file: %s", err) - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false}`)) - return - } - - defer f.Close() - parsedFile.Seek(0, io.SeekStart) - io.Copy(f, parsedFile) - - // FIXME: Set this one to 200 anyway? Can't download file then tho.. - file.Status = "active" - file.Md5sum = md5 - file.Sha256sum = fmt.Sprintf("%x", sha256Sum) - log.Printf("[INFO] MD5 for file %s (%s) is %s and SHA256 is %s", file.Filename, file.Id, file.Md5sum, file.Sha256sum) - - err = setFile(ctx, *file) - if err != nil { - log.Printf("[ERROR] Failed setting file back to active") - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false, "reason": "Failed setting file to active"}`)) - return - } - - log.Printf("[INFO] Successfully uploaded file ID %s", file.Id) - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) -} - -func handleCreateFile(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - // 1. Check user directly - // 2. Check workflow execution authorization - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("[INFO] INITIAL Api authentication failed in file creation: %s", err) - - orgId, err := fileAuthentication(request) - if err != nil { - log.Printf("[ERROR] Bad file authentication in create file: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - user.ActiveOrg.Id = orgId - user.Username = "Execution File API" - } - - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Println("Failed reading body") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to read data"}`))) - return - } - - type FileStructure struct { - Filename string `json:"filename"` - OrgId string `json:"org_id"` - WorkflowId string `json:"workflow_id"` - } - - var curfile FileStructure - err = json.Unmarshal(body, &curfile) - if err != nil { - log.Printf("[ERROR] Failed unmarshaling: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to unmarshal data"}`))) - return - } - - // Loads of validation below - if len(curfile.Filename) == 0 || len(curfile.OrgId) == 0 || len(curfile.WorkflowId) == 0 { - log.Printf("[ERROR] Missing field during fileupload. Required: filename, org_id, workflow_id") - log.Printf("INPUT: %s", string(body)) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Missing field. Required: filename, org_id, workflow_id"}`))) - return - } - - ctx := context.Background() - if user.ActiveOrg.Id != curfile.OrgId { - log.Printf("[ERROR] User can't access org %s", curfile.OrgId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Error with organization"}`)) - return - } - - var workflow *Workflow - if curfile.WorkflowId == "global" { - // PS: Not a security issue. - // Files are global anyway, but the workflow_id is used to identify origin - log.Printf("[INFO] Uploading filename %s for org %s as global file.", curfile.Filename, curfile.OrgId) - } else { - // Try to get the org and workflow in case they don't exist - workflow, err = getWorkflow(ctx, curfile.WorkflowId) - if err != nil { - log.Printf("[ERROR] Workflow %s doesn't exist.", curfile.WorkflowId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`)) - return - } - - _, err = getOrg(ctx, curfile.OrgId) - if err != nil { - log.Printf("[ERROR] Org %s doesn't exist.", curfile.OrgId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`)) - return - } - - if workflow.ExecutingOrg.Id != curfile.OrgId { - found := false - for _, curorg := range workflow.Org { - if curorg.Id == curfile.OrgId { - found = true - break - } - } - - if !found { - log.Printf("[ERROR] Org %s doesn't have access to %s.", curfile.OrgId, curfile.WorkflowId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`)) - return - } - } - } - - if strings.Contains(curfile.Filename, "/") || strings.Contains(curfile.Filename, `"`) || strings.Contains(curfile.Filename, "..") || strings.Contains(curfile.Filename, "~") { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Invalid characters in filename"}`)) - return - } - - // 1. Create the file object. - if len(basepath) == 0 { - basepath = "shuffle-files" - } - folderPath := fmt.Sprintf("%s/%s/%s", basepath, curfile.OrgId, curfile.WorkflowId) - - // Try to make the full file location - err = os.MkdirAll(folderPath, os.ModePerm) - if err != nil { - log.Printf("[ERROR] Writing issue for file location creation: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Failed creating upload location"}`)) - return - } - - filename := curfile.Filename - fileId := uuid.NewV4().String() - downloadPath := fmt.Sprintf("%s/%s", folderPath, fileId) - - duplicateWorkflows := []string{} - if curfile.WorkflowId != "global" { - for _, trigger := range workflow.Triggers { - if trigger.AppName == "Shuffle Workflow" && trigger.TriggerType == "SUBFLOW" { - for _, parameter := range trigger.Parameters { - if parameter.Name == "workflow" && len(parameter.Value) > 0 { - - found := false - for _, workflow := range duplicateWorkflows { - if workflow == parameter.Value { - found = true - break - } - } - - if !found { - duplicateWorkflows = append(duplicateWorkflows, parameter.Value) - } - - break - } - } - } - } - } - - timeNow := time.Now().Unix() - newFile := File{ - Id: fileId, - CreatedAt: timeNow, - UpdatedAt: timeNow, - Description: "", - Status: "created", - Filename: filename, - OrgId: curfile.OrgId, - WorkflowId: curfile.WorkflowId, - DownloadPath: downloadPath, - Subflows: duplicateWorkflows, - } - - err = setFile(ctx, newFile) - if err != nil { - log.Printf("[ERROR] Failed setting file: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Failed setting file reference"}`)) - return - } else { - log.Printf("[INFO] Created file %s", newFile.DownloadPath) - } - - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s"}`, fileId))) - -} - -func getFile(ctx context.Context, id string) (*File, error) { - key := datastore.NameKey("Files", id, nil) - curFile := &File{} - if err := dbclient.Get(ctx, key, curFile); err != nil { - return &File{}, err - } - - return curFile, nil -} - -func setFile(ctx context.Context, file File) error { - // clear session_token and API_token for user - timeNow := time.Now().Unix() - file.UpdatedAt = timeNow - - k := datastore.NameKey("Files", file.Id, nil) - if _, err := dbclient.Put(ctx, k, &file); err != nil { - log.Println(err) - return err - } - - return nil -} - -func getAllFiles(ctx context.Context, orgId string) ([]File, error) { - var files []File - q := datastore.NewQuery("Files").Filter("org_id =", orgId).Order("-updated_at").Limit(100) - - _, err := dbclient.GetAll(ctx, q, &files) - if err != nil { - if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") { - q = q.Limit(50) - _, err := dbclient.GetAll(ctx, q, &files) - if err != nil { - return []File{}, err - } - } else { - return []File{}, err - } - } - - return files, nil -} diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 0bf0ccce..e403c8bd 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -2,11 +2,15 @@ module shuffle go 1.13 +replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared + +replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi + require ( - cloud.google.com/go v0.57.0 - cloud.google.com/go/datastore v1.1.0 + cloud.google.com/go v0.75.0 + cloud.google.com/go/datastore v1.4.0 cloud.google.com/go/pubsub v1.3.1 - cloud.google.com/go/storage v1.7.0 + cloud.google.com/go/storage v1.12.0 github.com/Microsoft/go-winio v0.4.14 // indirect github.com/basgys/goxml2json v1.1.0 github.com/carlescere/scheduler v0.0.0-20170109141437-ee74d2f83d82 @@ -14,24 +18,26 @@ require ( github.com/docker/docker v1.13.1 github.com/docker/go-connections v0.4.0 github.com/docker/go-units v0.4.0 // indirect - github.com/getkin/kin-openapi v0.8.0 + github.com/frikky/shuffle-shared v0.0.12 // indirect + github.com/getkin/kin-openapi v0.52.0 // indirect + //github.com/getkin/kin-openapi v0.8.0 github.com/ghodss/yaml v1.0.0 github.com/go-git/go-billy/v5 v5.0.0 github.com/go-git/go-git/v5 v5.0.0 github.com/google/go-github/v28 v28.1.1 github.com/gorilla/handlers v1.4.2 // indirect - github.com/gorilla/mux v1.7.4 + github.com/gorilla/mux v1.8.0 github.com/h2non/filetype v1.0.12 github.com/opencontainers/go-digest v1.0.0-rc1 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible github.com/satori/go.uuid v1.2.0 - golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79 - golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d - google.golang.org/api v0.23.0 - google.golang.org/appengine v1.6.6 - google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31 - google.golang.org/grpc v1.29.1 + golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 + golang.org/x/oauth2 v0.0.0-20210113160501-8b1d76fa0423 + google.golang.org/api v0.36.0 + google.golang.org/appengine v1.6.7 + google.golang.org/genproto v0.0.0-20210113195801-ae06605f4595 + google.golang.org/grpc v1.34.1 gopkg.in/src-d/go-git.v4 v4.13.1 - gopkg.in/yaml.v2 v2.2.8 - gopkg.in/yaml.v3 v3.0.0-20200506231410-2ff61e1afc86 + gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 2d63be91..de114e19 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -12,14 +12,24 @@ cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bP cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= cloud.google.com/go v0.57.0 h1:EpMNVUorLiZIELdMZbCYX/ByTFCdoYopYAGxaGVz9ms= cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.66.0/go.mod h1:dgqGAjKCDxyhGTtC9dAREQGUJpkceNm1yt590Qno0Ko= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.75.0 h1:XgtDnVJRCPEUG21gjFiRPz4zI1Mjg16R+NYQjfmU4XY= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.6.0/go.mod h1:hyFDG0qSGdHNz8Q6nDN8rYIkld0q/+5uBZaelxiDLfE= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/datastore v1.4.0 h1:CFDJm15RpYXeEblQ0TMDUrYtqmBmbAWTy536nA8JIc8= +cloud.google.com/go/datastore v1.4.0/go.mod h1:d18825/a9bICdAIJy2EkHs9joU4RlIZ1t6l8WDdbdY0= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -30,6 +40,10 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.7.0 h1:DzdLPI8Em+DEk7IzA2a10ivq3mxIEASC9GeNJ6FFt5Q= cloud.google.com/go/storage v1.7.0/go.mod h1:jGMIBwF+L/tL6WN/W5InNgYYu4HP0DvGB6rQ1mufWfs= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.12.0 h1:4y3gHptW1EHVtcPAVE0eBBlFuGqEejTTG3KdIE0lUX4= +cloud.google.com/go/storage v1.12.0/go.mod h1:fFLk2dp2oAhDz8QFKwqrjdJvxSp/W2g7nillojlL5Ho= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -49,6 +63,7 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -66,10 +81,17 @@ github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/frikky/kin-openapi v0.38.0 h1:V7ttwIJS8Vks4KL+mZVj1ZSqhIcQtgaG8akeqXEQgsE= +github.com/frikky/kin-openapi v0.38.0/go.mod h1:Fr28TtCHL4K0kIqtqui8HWxN1LG5uAh3z/tDfFyiA1s= +github.com/frikky/shuffle-shared v0.0.12 h1:+0EIfThmK47Po+LogPYZR4XjbS4Ds19WNMFu2YUSjhw= +github.com/frikky/shuffle-shared v0.0.12/go.mod h1:SEY432/xs4oBkOUnGwxiYKCZWZLRaheGInhAoW7N8ww= github.com/getkin/kin-openapi v0.8.0 h1:a6TQjTqwkyscC4/hShJX7WhCVE+4bi9lzw61XHQW5hE= github.com/getkin/kin-openapi v0.8.0/go.mod h1:zZQMFkVgRHCdhgb6ihCTIo9dyDZFvX0k/xAKqw1FhPw= +github.com/getkin/kin-openapi v0.52.0 h1:6WqsF5d6PfJ8AscdD+9Rtb2RP2iBWyC7V6GcjssWg7M= +github.com/getkin/kin-openapi v0.52.0/go.mod h1:fRpo2Nw4Czgy0QnrIesRrEXs5+15N1F9mGZLP/aIomE= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= @@ -85,6 +107,10 @@ github.com/go-git/go-git/v5 v5.0.0/go.mod h1:oYD8y9kWsGINPFJoLdaScGCN6dlKg23blmC github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -96,6 +122,7 @@ github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFU github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -108,6 +135,10 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -115,19 +146,32 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= github.com/google/go-github/v28 v28.1.1 h1:kORf5ekX5qwXO2mGzXXOjMe/g6ap8ahVe0sBEulhSxo= github.com/google/go-github/v28 v28.1.1/go.mod h1:bsqJWQX05omyWVmc00nEUql9mhQyv38lDZ8kPZcQVoM= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200905233945-acf8798be1f7/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= @@ -135,11 +179,14 @@ github.com/gorilla/handlers v1.4.2 h1:0QniY0USkHQ1RGCLfKxeNHK9bkDHGRYGNDFBCS+YAR github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/h2non/filetype v1.0.12 h1:yHCsIe0y2cvbDARtJhGBTD2ecvqMSTvlIcph9En/Zao= github.com/h2non/filetype v1.0.12/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -155,6 +202,9 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= @@ -183,15 +233,21 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -201,6 +257,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79 h1:IaQbIIB2X/Mp/DKctl6ROxz1KyMlKp4uyvL6+kQ7C88= golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -224,6 +282,7 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -232,6 +291,9 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -242,6 +304,7 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -252,12 +315,28 @@ golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5 h1:WQ8q63x+f/zpC8Ac1s9wLElVoHhm32p6tudrU72n1QA= golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b h1:iFwSg7t5GZmB/Q5TjiEAsdoLDrdJRC1RiF2WhuV29Qw= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210113160501-8b1d76fa0423 h1:/hEknzWkMPCjTo7StMHRrBRa8YBbXuBWfck8680k3RE= +golang.org/x/oauth2 v0.0.0-20210113160501-8b1d76fa0423/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -266,6 +345,9 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -291,11 +373,25 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200409092240-59c9f1ba88fa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e h1:hq86ru83GdWTlfQFZGO4nZJTU4Bs2wfHl8oFHRaXsfc= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3 h1:kzM6+9dur93BcC2kVlYl34cHU+TYZLanmpSJHVMmL64= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -336,10 +432,25 @@ golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWc golang.org/x/tools v0.0.0-20200409170454-77362c5149f0/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d h1:lzLdP95xJmMpwQ6LUHwrc5V7js93hTiY7gkznu0BgmY= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20200915173823-2db8f0ff891c/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/tools v0.0.0-20200918232735-d647fc253266/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -355,6 +466,15 @@ google.golang.org/api v0.21.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/ google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.23.0 h1:YlvGEOq2NA2my8cZ/9V8BcEO9okD48FlJcdqN0xJL3s= google.golang.org/api v0.23.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.31.0/go.mod h1:CL+9IBCa2WWU6gRuBWaKqGWLFFwbEUXkfeMkHLQWYWo= +google.golang.org/api v0.32.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0 h1:l2Nfbl2GPXdWorv+dT2XfinX2jOOw4zv1VhLstx+6rE= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -362,6 +482,8 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -387,6 +509,22 @@ google.golang.org/genproto v0.0.0-20200409111301-baae70f3302d/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31 h1:Bz1qTn2YRWV+9OKJtxHJiQKCiXIdf+kwuKXdt9cBxyU= google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200831141814-d751682dd103/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200914193844-75d14daec038/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200921151605-7abf4a1a14d5/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210113195801-ae06605f4595 h1:x7nk+/4+SvuTDI4wnzQUlhvi+DTpyfncXBo3QWTFs7U= +google.golang.org/genproto v0.0.0-20210113195801-ae06605f4595/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -399,12 +537,26 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.28.1/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.34.1 h1:ugq+9++ZQPFzM2pKUMCIK8gj9M0pFyuUWO9Q8kwEDQw= +google.golang.org/grpc v1.34.1/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -421,8 +573,14 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200506231410-2ff61e1afc86 h1:OfFoIUYv/me30yv7XlMy4F9RJw8DEm8WQ6QG1Ph4bH0= gopkg.in/yaml.v3 v3.0.0-20200506231410-2ff61e1afc86/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -430,6 +588,7 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index deeb5bc8..11df5536 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -1,6 +1,8 @@ package main import ( + "github.com/frikky/shuffle-shared" + "bufio" "bytes" @@ -29,12 +31,11 @@ import ( "cloud.google.com/go/datastore" "cloud.google.com/go/pubsub" "cloud.google.com/go/storage" - "google.golang.org/api/option" "google.golang.org/appengine/mail" - "github.com/getkin/kin-openapi/openapi2" - "github.com/getkin/kin-openapi/openapi2conv" - "github.com/getkin/kin-openapi/openapi3" + "github.com/frikky/kin-openapi/openapi2" + "github.com/frikky/kin-openapi/openapi2conv" + "github.com/frikky/kin-openapi/openapi3" /* "github.com/frikky/kin-openapi/openapi2" "github.com/frikky/kin-openapi/openapi2conv" @@ -63,6 +64,7 @@ import ( // Web "github.com/gorilla/mux" "github.com/patrickmn/go-cache" + "google.golang.org/api/option" "google.golang.org/grpc" http2 "gopkg.in/src-d/go-git.v4/plumbing/transport/http" ) @@ -150,11 +152,11 @@ type UserLimits struct { } type retStruct struct { - Success bool `json:"success"` - SyncFeatures SyncFeatures `json:"sync_features"` - SessionKey string `json:"session_key"` - IntervalSeconds int64 `json:"interval_seconds"` - Reason string `json:"reason"` + Success bool `json:"success"` + SyncFeatures shuffle.SyncFeatures `json:"sync_features"` + SessionKey string `json:"session_key"` + IntervalSeconds int64 `json:"interval_seconds"` + Reason string `json:"reason"` } // Saves some data, not sure what to have here lol @@ -172,37 +174,15 @@ type UserAuthField struct { } // Not environment, but execution environment -type Environment struct { - Name string `datastore:"name"` - Type string `datastore:"type"` - Registered bool `datastore:"registered"` - Default bool `datastore:"default" json:"default"` - Archived bool `datastore:"archived" json:"archived"` - Id string `datastore:"id" json:"id"` - OrgId string `datastore:"org_id" json:"org_id"` -} - -type User struct { - Username string `datastore:"Username" json:"username"` - Password string `datastore:"password,noindex" password:"password,omitempty"` - Session string `datastore:"session,noindex" json:"session"` - Verified bool `datastore:"verified,noindex" json:"verified"` - PrivateApps []WorkflowApp `datastore:"privateapps" json:"privateapps":` - Role string `datastore:"role" json:"role"` - Roles []string `datastore:"roles" json:"roles"` - VerificationToken string `datastore:"verification_token" json:"verification_token"` - ApiKey string `datastore:"apikey" json:"apikey"` - ResetReference string `datastore:"reset_reference" json:"reset_reference"` - Executions ExecutionInfo `datastore:"executions" json:"executions"` - Limits UserLimits `datastore:"limits" json:"limits"` - Authentication []UserAuth `datastore:"authentication,noindex" json:"authentication"` - ResetTimeout int64 `datastore:"reset_timeout,noindex" json:"reset_timeout"` - Id string `datastore:"id" json:"id"` - Orgs []string `datastore:"orgs" json:"orgs"` - CreationTime int64 `datastore:"creation_time" json:"creation_time"` - ActiveOrg Org `json:"active_org" datastore:"active_org"` - Active bool `datastore:"active" json:"active"` -} +//type Environment struct { +// Name string `datastore:"name"` +// Type string `datastore:"type"` +// Registered bool `datastore:"registered"` +// Default bool `datastore:"default" json:"default"` +// Archived bool `datastore:"archived" json:"archived"` +// Id string `datastore:"id" json:"id"` +// OrgId string `datastore:"org_id" json:"org_id"` +//} // timeout maybe? idk type session struct { @@ -611,83 +591,6 @@ func checkFileExistsLocal(basepath string, filepath string) bool { return true } -func handleApiAuthentication(resp http.ResponseWriter, request *http.Request) (User, error) { - apikey := request.Header.Get("Authorization") - if len(apikey) > 0 { - if !strings.HasPrefix(apikey, "Bearer ") { - log.Printf("[WARNING] Apikey doesn't start with bearer") - return User{}, errors.New("No bearer token for authorization header") - } - - apikeyCheck := strings.Split(apikey, " ") - if len(apikeyCheck) != 2 { - log.Printf("[WARNING] Invalid format for apikey.") - return User{}, errors.New("Invalid format for apikey") - } - - // This is annoying af and is done because of maxlength lol - newApikey := apikeyCheck[1] - if len(newApikey) > 249 { - newApikey = newApikey[0:248] - } - - ctx := context.Background() - - // Make specific check for just service user? - // Get the user based on APIkey here - Userdata, err := getApikey(ctx, apikeyCheck[1]) - if err != nil { - log.Printf("Apikey %s doesn't exist: %s", apikey, err) - return User{}, err - } - - if len(Userdata.Username) > 0 { - return Userdata, nil - } else { - return Userdata, errors.New(fmt.Sprintf("[WARNING] User is invalid - no username found")) - } - } - - // One time API keys - authorizationArr, ok := request.URL.Query()["authorization"] - ctx := context.Background() - if ok { - authorization := "" - if len(authorizationArr) > 0 { - authorization = authorizationArr[0] - } - _ = authorization - } - - c, err := request.Cookie("session_token") - if err == nil { - sessionToken := c.Value - session, err := getSession(ctx, sessionToken) - if err != nil { - log.Printf("[WARNING] Session %s doesn't exist (session auth): %s", sessionToken, err) - return User{}, err - } - - // Get session first - // Should basically never happen - Userdata, err := getUser(ctx, session.Id) - if err != nil { - log.Printf("[INFO] Username %s doesn't exist (authcheck): %s", session.Username, err) - return User{}, err - } - - if Userdata.Session != sessionToken { - return User{}, errors.New("Wrong session token") - } - - // Means session exists, but - return *Userdata, nil - } - - // Key = apikey - return User{}, errors.New("Missing authentication") -} - func handleGetallSchedules(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -800,7 +703,7 @@ func deleteUser(resp http.ResponseWriter, request *http.Request) { return } - userInfo, userErr := handleApiAuthentication(resp, request) + userInfo, userErr := shuffle.HandleApiAuthentication(resp, request) if userErr != nil { log.Printf("Api authentication failed in edit workflow: %s", userErr) resp.WriteHeader(401) @@ -828,7 +731,7 @@ func deleteUser(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - foundUser, err := getUser(ctx, userId) + foundUser, err := shuffle.GetUser(ctx, userId) if err != nil { log.Printf("Can't find user %s (delete user): %s", userId, err) resp.WriteHeader(401) @@ -864,7 +767,7 @@ func deleteUser(resp http.ResponseWriter, request *http.Request) { foundUser.Active = true } - err = setUser(ctx, foundUser) + err = shuffle.SetUser(ctx, foundUser) if err != nil { log.Printf("Failed swapping active for user %s (%s)", foundUser.Username, foundUser.Id) resp.WriteHeader(401) @@ -920,7 +823,7 @@ func handleRegisterVerification(resp http.ResponseWriter, request *http.Request) // With user, do a search for workflows with user or user's org attached // Only giving 200 to not give any suspicion whether they're onto an actual user or not q := datastore.NewQuery("Users").Filter("verification_token =", reference) - var users []User + var users []shuffle.User _, err := dbclient.GetAll(ctx, q, &users) if err != nil { log.Printf("Failed getting users for verification token: %s", err) @@ -941,7 +844,7 @@ func handleRegisterVerification(resp http.ResponseWriter, request *http.Request) // FIXME: Not for cloud! Userdata.Verified = true - err = setUser(ctx, &Userdata) + err = shuffle.SetUser(ctx, &Userdata) if err != nil { log.Printf("Failed adding verification for user %s: %s", Userdata.Username, err) resp.WriteHeader(401) @@ -962,7 +865,7 @@ func handleSetEnvironments(resp http.ResponseWriter, request *http.Request) { // FIXME: Overhaul the top part. // Only admin can change environments, but if there are no users, anyone can make (first) - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Can't handle set env auth"}`)) @@ -976,7 +879,7 @@ func handleSetEnvironments(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - var environments []Environment + var environments []shuffle.Environment q := datastore.NewQuery("Environments").Filter("org_id =", user.ActiveOrg.Id) _, err = dbclient.GetAll(ctx, q, &environments) if err != nil { @@ -993,7 +896,7 @@ func handleSetEnvironments(resp http.ResponseWriter, request *http.Request) { return } - var newEnvironments []Environment + var newEnvironments []shuffle.Environment err = json.Unmarshal(body, &newEnvironments) if err != nil { log.Printf("Failed unmarshaling: %s", err) @@ -1053,7 +956,7 @@ func handleSetEnvironments(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(`{"success": true}`)) } -func createNewUser(username, password, role, apikey string, org Org) error { +func createNewUser(username, password, role, apikey string, org shuffle.Org) error { // Returns false if there is an issue // Use this for register err := checkPasswordStrength(password) @@ -1070,7 +973,7 @@ func createNewUser(username, password, role, apikey string, org Org) error { ctx := context.Background() q := datastore.NewQuery("Users").Filter("Username =", username) - var users []User + var users []shuffle.User _, err = dbclient.GetAll(ctx, q, &users) if err != nil { log.Printf("Failed getting user for registration: %s", err) @@ -1087,7 +990,7 @@ func createNewUser(username, password, role, apikey string, org Org) error { return err } - newUser := new(User) + newUser := new(shuffle.User) newUser.Username = username newUser.Password = string(hashedPassword) newUser.Verified = false @@ -1136,13 +1039,13 @@ func createNewUser(username, password, role, apikey string, org Org) error { newUser.Id = ID.String() newUser.VerificationToken = verifyToken.String() - err = setUser(ctx, newUser) + err = shuffle.SetUser(ctx, newUser) if err != nil { log.Printf("Error adding User %s: %s", username, err) return err } - neworg, err := getOrg(ctx, org.Id) + neworg, err := shuffle.GetOrg(ctx, org.Id) if err == nil { //neworg.Users = append(neworg.Users, *newUser) err = setOrg(ctx, *neworg, neworg.Id) @@ -1170,7 +1073,7 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) { // FIXME: Overhaul the top part. // Only admin can CREATE users, but if there are no users, anyone can make (first) count, countErr := getUserCount() - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { if (countErr == nil && count > 0) || countErr != nil { resp.WriteHeader(401) @@ -1206,7 +1109,7 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) { if user.ActiveOrg.Id == "" { log.Printf("There's no active org for the user. Checking if there's a single one to assing it to.") - var orgs []Org + var orgs []shuffle.Org q := datastore.NewQuery("Organizations") _, err = dbclient.GetAll(ctx, q, &orgs) if err == nil && len(orgs) == 1 { @@ -1255,7 +1158,7 @@ func handleLogout(resp http.ResponseWriter, request *http.Request) { Expires: time.Unix(0, 0), }) - userInfo, err := handleApiAuthentication(resp, request) + userInfo, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in handleLogout: %s", err) resp.WriteHeader(200) @@ -1264,7 +1167,7 @@ func handleLogout(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - session, err := getSession(ctx, userInfo.Session) + session, err := shuffle.GetSession(ctx, userInfo.Session) if err != nil { log.Printf("Session %#v doesn't exist: %s", session, err) resp.WriteHeader(401) @@ -1295,7 +1198,7 @@ func handleLogout(resp http.ResponseWriter, request *http.Request) { // Get session first // Should basically never happen - //_, err = getUser(ctx, session.Id) + //_, err = shuffle.GetUser(ctx, session.Id) //if err != nil { // log.Printf("Username %s doesn't exist (logout): %s", session.Username, err) // resp.WriteHeader(401) @@ -1309,7 +1212,7 @@ func handleLogout(resp http.ResponseWriter, request *http.Request) { // FIXME // Session might delete someone elses here? // No need to think about before possible scale..? - err = SetSession(ctx, userInfo, "") + err = shuffle.SetSession(ctx, userInfo, "") if err != nil { log.Printf("Error removing session for: %s", err) resp.WriteHeader(401) @@ -1326,7 +1229,7 @@ func handleLogout(resp http.ResponseWriter, request *http.Request) { } userInfo.Session = "" - err = setUser(ctx, &userInfo) + err = shuffle.SetUser(ctx, &userInfo) if err != nil { log.Printf("Failed updating user: %s", err) resp.WriteHeader(401) @@ -1340,35 +1243,13 @@ func handleLogout(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(`{"success": false, "reason": "Successfully logged out"}`)) } -func generateApikey(ctx context.Context, userInfo User) (User, error) { - // Generate UUID - // Set uuid to apikey in backend (update) - apikey := uuid.NewV4() - userInfo.ApiKey = apikey.String() - - err := SetApikey(ctx, userInfo) - if err != nil { - log.Printf("Failed updating apikey: %s", err) - return userInfo, err - } - - // Updating user - err = setUser(ctx, &userInfo) - if err != nil { - log.Printf("Failed updating user: %s", err) - return userInfo, err - } - - return userInfo, nil -} - func handleUpdateUser(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } - userInfo, err := handleApiAuthentication(resp, request) + userInfo, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in apigen: %s", err) resp.WriteHeader(401) @@ -1409,7 +1290,7 @@ func handleUpdateUser(resp http.ResponseWriter, request *http.Request) { return } - foundUser, err := getUser(ctx, t.UserId) + foundUser, err := shuffle.GetUser(ctx, t.UserId) if err != nil { log.Printf("Can't find user %s (update user): %s", t.UserId, err) resp.WriteHeader(401) @@ -1452,7 +1333,7 @@ func handleUpdateUser(resp http.ResponseWriter, request *http.Request) { if len(t.Username) > 0 { q := datastore.NewQuery("Users").Filter("username =", t.Username) - var users []User + var users []shuffle.User _, err = dbclient.GetAll(ctx, q, &users) if err != nil { resp.WriteHeader(401) @@ -1477,7 +1358,7 @@ func handleUpdateUser(resp http.ResponseWriter, request *http.Request) { foundUser.Username = t.Username } - err = setUser(ctx, foundUser) + err = shuffle.SetUser(ctx, foundUser) if err != nil { log.Printf("Error patching user %s: %s", foundUser.Username, err) resp.WriteHeader(401) @@ -1495,7 +1376,7 @@ func handleApiGeneration(resp http.ResponseWriter, request *http.Request) { return } - userInfo, err := handleApiAuthentication(resp, request) + userInfo, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in apigen: %s", err) resp.WriteHeader(401) @@ -1505,7 +1386,7 @@ func handleApiGeneration(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() if request.Method == "GET" { - newUserInfo, err := generateApikey(ctx, userInfo) + newUserInfo, err := shuffle.GenerateApikey(ctx, userInfo) if err != nil { log.Printf("Failed to generate apikey for user %s: %s", userInfo.Username, err) resp.WriteHeader(401) @@ -1544,7 +1425,7 @@ func handleApiGeneration(resp http.ResponseWriter, request *http.Request) { return } - foundUser, err := getUser(ctx, t.UserId) + foundUser, err := shuffle.GetUser(ctx, t.UserId) if err != nil { log.Printf("Can't find user %s (apikey gen): %s", t.UserId, err) resp.WriteHeader(401) @@ -1552,7 +1433,7 @@ func handleApiGeneration(resp http.ResponseWriter, request *http.Request) { return } - newUserInfo, err := generateApikey(ctx, *foundUser) + newUserInfo, err := shuffle.GenerateApikey(ctx, *foundUser) if err != nil { log.Printf("Failed to generate apikey for user %s: %s", foundUser.Username, err) resp.WriteHeader(401) @@ -1576,7 +1457,7 @@ func handleSettings(resp http.ResponseWriter, request *http.Request) { return } - userInfo, err := handleApiAuthentication(resp, request) + userInfo, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in apigen: %s", err) resp.WriteHeader(401) @@ -1594,7 +1475,7 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { return } - userInfo, err := handleApiAuthentication(resp, request) + userInfo, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in handleInfo: %s", err) resp.WriteHeader(401) @@ -1623,7 +1504,7 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() q := datastore.NewQuery("Users") - var users []User + var users []shuffle.User _, err = dbclient.GetAll(ctx, q, &users) if err != nil { resp.WriteHeader(401) @@ -1688,14 +1569,14 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { // Updating user info if there's something wrong if (len(userInfo.ActiveOrg.Name) == 0 || len(userInfo.ActiveOrg.Id) == 0) && len(userInfo.Orgs) > 0 { - _, err := getOrg(ctx, userInfo.Orgs[0]) + _, err := shuffle.GetOrg(ctx, userInfo.Orgs[0]) if err != nil { - var orgs []Org + var orgs []shuffle.Org q := datastore.NewQuery("Organizations") _, err = dbclient.GetAll(ctx, q, &orgs) if err == nil { newStringOrgs := []string{} - newOrgs := []Org{} + newOrgs := []shuffle.Org{} for _, org := range orgs { if strings.ToLower(org.Name) == strings.ToLower(userInfo.Orgs[0]) { newOrgs = append(newOrgs, org) @@ -1707,7 +1588,7 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { userInfo.ActiveOrg = newOrgs[0] userInfo.Orgs = newStringOrgs - err = setUser(ctx, &userInfo) + err = shuffle.SetUser(ctx, &userInfo) if err != nil { log.Printf("Error patching User for activeOrg: %s", err) } else { @@ -1721,10 +1602,10 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { } else { // 1. Check if the org exists by ID // 2. if it does, overwrite user - userInfo.ActiveOrg = Org{ + userInfo.ActiveOrg = shuffle.Org{ Id: userInfo.Orgs[0], } - err = setUser(ctx, &userInfo) + err = shuffle.SetUser(ctx, &userInfo) if err != nil { log.Printf("Error patching User for activeOrg: %s", err) } @@ -1732,10 +1613,10 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { } // FIXME: Remove this dependency by updating users' orgs when org itself is updated - org, err := getOrg(ctx, userInfo.ActiveOrg.Id) + org, err := shuffle.GetOrg(ctx, userInfo.ActiveOrg.Id) if err == nil { userInfo.ActiveOrg = *org - userInfo.ActiveOrg.Users = []User{} + userInfo.ActiveOrg.Users = []shuffle.User{} } currentOrg, err := json.Marshal(userInfo.ActiveOrg) @@ -1816,7 +1697,7 @@ func handlePasswordReset(resp http.ResponseWriter, request *http.Request) { // With user, do a search for workflows with user or user's org attached // Only giving 200 to not give any suspicion whether they're onto an actual user or not q := datastore.NewQuery("Users").Filter("reset_reference =", t.Reference) - var users []User + var users []shuffle.User _, err = dbclient.GetAll(ctx, q, &users) if err != nil { log.Printf("Failed getting users: %s", err) @@ -1845,7 +1726,7 @@ func handlePasswordReset(resp http.ResponseWriter, request *http.Request) { Userdata.Password = string(hashedPassword) Userdata.ResetTimeout = 0 Userdata.ResetReference = "" - err = setUser(ctx, &Userdata) + err = shuffle.SetUser(ctx, &Userdata) if err != nil { log.Printf("Error adding User %s: %s", Userdata.Username, err) resp.WriteHeader(200) @@ -1884,7 +1765,7 @@ func handlePasswordChange(resp http.ResponseWriter, request *http.Request) { return } - userInfo, err := handleApiAuthentication(resp, request) + userInfo, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -1929,11 +1810,11 @@ func handlePasswordChange(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - foundUser := User{} + foundUser := shuffle.User{} if !curUserFound { log.Printf("Have to find a different user") q := datastore.NewQuery("Users").Filter("Username =", strings.ToLower(t.Username)) - var users []User + var users []shuffle.User _, err = dbclient.GetAll(ctx, q, &users) if err != nil { log.Printf("Failed getting user %s", t.Username) @@ -1998,7 +1879,7 @@ func handlePasswordChange(resp http.ResponseWriter, request *http.Request) { } userInfo.Password = string(hashedPassword) - err = setUser(ctx, &foundUser) + err = shuffle.SetUser(ctx, &foundUser) if err != nil { log.Printf("Error fixing password for user %s: %s", userInfo.Username, err) resp.WriteHeader(401) @@ -2087,7 +1968,7 @@ func handleGetSchedules(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -2130,7 +2011,7 @@ func handleGetEnvironments(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -2139,7 +2020,7 @@ func handleGetEnvironments(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - var environments []Environment + var environments []shuffle.Environment q := datastore.NewQuery("Environments").Filter("org_id =", user.ActiveOrg.Id) _, err = dbclient.GetAll(ctx, q, &environments) if err != nil { @@ -2181,7 +2062,7 @@ func handleGetOrg(resp http.ResponseWriter, request *http.Request) { fileId = location[4] } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -2190,7 +2071,7 @@ func handleGetOrg(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - org, err := getOrg(ctx, fileId) + org, err := shuffle.GetOrg(ctx, fileId) if err != nil { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Failed getting org users"}`)) @@ -2212,7 +2093,7 @@ func handleGetOrg(resp http.ResponseWriter, request *http.Request) { return } - org.Users = []User{} + org.Users = []shuffle.User{} org.SyncConfig.Apikey = "" newjson, err := json.Marshal(org) if err != nil { @@ -2232,7 +2113,7 @@ func handleGetOrgs(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -2247,7 +2128,7 @@ func handleGetOrgs(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - var orgs []Org + var orgs []shuffle.Org q := datastore.NewQuery("Organizations") _, err = dbclient.GetAll(ctx, q, &orgs) if err != nil { @@ -2287,7 +2168,7 @@ func handleGetUsers(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -2303,14 +2184,14 @@ func handleGetUsers(resp http.ResponseWriter, request *http.Request) { // FIXME: Check by org. ctx := context.Background() - org, err := getOrg(ctx, user.ActiveOrg.Id) + org, err := shuffle.GetOrg(ctx, user.ActiveOrg.Id) if err != nil { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Failed getting org users"}`)) return } - newUsers := []User{} + newUsers := []shuffle.User{} for _, item := range org.Users { if len(item.Username) == 0 { continue @@ -2390,7 +2271,7 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() log.Printf("[INFO] Login Username: %s", data.Username) q := datastore.NewQuery("Users").Filter("Username =", data.Username) - var users []User + var users []shuffle.User _, err = dbclient.GetAll(ctx, q, &users) if err != nil { log.Printf("Failed getting user %s", data.Username) @@ -2438,7 +2319,7 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) { loginData = fmt.Sprintf(`{"success": true, "cookies": [{"key": "session_token", "value": "%s", "expiration": %d}]}`, Userdata.Session, expiration.Unix()) //log.Printf("SESSION LENGTH MORE THAN 0 IN LOGIN: %s", Userdata.Session) - err = SetSession(ctx, Userdata, Userdata.Session) + err = shuffle.SetSession(ctx, Userdata, Userdata.Session) if err != nil { log.Printf("Error adding session to database: %s", err) } @@ -2458,13 +2339,13 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) { }) // ADD TO DATABASE - err = SetSession(ctx, Userdata, sessionToken) + err = shuffle.SetSession(ctx, Userdata, sessionToken) if err != nil { log.Printf("Error adding session to database: %s", err) } Userdata.Session = sessionToken - err = setUser(ctx, &Userdata) + err = shuffle.SetUser(ctx, &Userdata) if err != nil { log.Printf("Failed updating user when setting session: %s", err) resp.WriteHeader(500) @@ -2481,46 +2362,7 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(loginData)) } -func getApikey(ctx context.Context, apikey string) (User, error) { - // Query for the specifci workflowId - q := datastore.NewQuery("Users").Filter("apikey =", apikey) - var users []User - _, err := dbclient.GetAll(ctx, q, &users) - if err != nil { - log.Printf("[ERROR] Error getting users apikey (getapikey): %s", err) - return User{}, err - } - - if len(users) == 0 { - log.Printf("[WARNING] No users found for apikey %s", apikey) - return User{}, err - } - - return users[0], nil -} - -func getSession(ctx context.Context, thissession string) (*session, error) { - key := datastore.NameKey("sessions", thissession, nil) - curUser := &session{} - if err := dbclient.Get(ctx, key, curUser); err != nil { - return &session{}, err - } - - return curUser, nil -} - -// ListBooks returns a list of books, ordered by title. -func getOrg(ctx context.Context, id string) (*Org, error) { - key := datastore.NameKey("Organizations", id, nil) - curOrg := &Org{} - if err := dbclient.Get(ctx, key, curOrg); err != nil { - return &Org{}, err - } - - return curOrg, nil -} - -func setOrg(ctx context.Context, org Org, id string) error { +func setOrg(ctx context.Context, org shuffle.Org, id string) error { // clear session_token and API_token for user timeNow := int64(time.Now().Unix()) if org.Created == 0 { @@ -2543,15 +2385,15 @@ func setOrg(ctx context.Context, org Org, id string) error { } // ListBooks returns a list of books, ordered by title. -func getUser(ctx context.Context, id string) (*User, error) { - key := datastore.NameKey("Users", id, nil) - curUser := &User{} - if err := dbclient.Get(ctx, key, curUser); err != nil { - return &User{}, err - } - - return curUser, nil -} +//func getUser(ctx context.Context, id string) (*User, error) { +// key := datastore.NameKey("Users", id, nil) +// curUser := &User{} +// if err := dbclient.Get(ctx, key, curUser); err != nil { +// return &User{}, err +// } +// +// return curUser, nil +//} // Index = Username func DeleteKeys(ctx context.Context, entity string, value []string) error { @@ -2584,52 +2426,6 @@ func DeleteKey(ctx context.Context, entity string, value string) error { return nil } -// Index = Username -func SetApikey(ctx context.Context, Userdata User) error { - // Non indexed User data - newapiUser := new(Userapi) - newapiUser.ApiKey = Userdata.ApiKey - newapiUser.Username = Userdata.Username - key1 := datastore.NameKey("apikey", newapiUser.ApiKey, nil) - - // New struct, to not add body, author etc - if _, err := dbclient.Put(ctx, key1, newapiUser); err != nil { - log.Printf("Error adding apikey: %s", err) - return err - } - - return nil -} - -// Index = Username -func SetSession(ctx context.Context, Userdata User, value string) error { - // Non indexed User data - Userdata.Session = value - key1 := datastore.NameKey("Users", Userdata.Id, nil) - - // New struct, to not add body, author etc - if _, err := dbclient.Put(ctx, key1, &Userdata); err != nil { - log.Printf("rror adding Usersession: %s", err) - return err - } - - if len(Userdata.Session) > 0 { - // Indexed session data - sessiondata := new(session) - sessiondata.Username = Userdata.Username - sessiondata.Session = Userdata.Session - sessiondata.Id = Userdata.Id - key2 := datastore.NameKey("sessions", sessiondata.Session, nil) - - if _, err := dbclient.Put(ctx, key2, sessiondata); err != nil { - log.Printf("Error adding session: %s", err) - return err - } - } - - return nil -} - func setOpenApiDatastore(ctx context.Context, id string, data ParsedOpenApi) error { k := datastore.NameKey("openapi3", id, nil) if _, err := dbclient.Put(ctx, k, &data); err != nil { @@ -2649,7 +2445,7 @@ func getOpenApiDatastore(ctx context.Context, id string) (ParsedOpenApi, error) return *api, nil } -func setEnvironment(ctx context.Context, data *Environment) error { +func setEnvironment(ctx context.Context, data *shuffle.Environment) error { // clear session_token and API_token for user k := datastore.NameKey("Environments", strings.ToLower(data.Name), nil) @@ -2663,7 +2459,7 @@ func setEnvironment(ctx context.Context, data *Environment) error { return nil } -func fixOrgUser(ctx context.Context, org *Org) *Org { +func fixOrgUser(ctx context.Context, org *shuffle.Org) *shuffle.Org { //found := false //for _, id := range user.Orgs { // if user.ActiveOrg.Id == id { @@ -2682,7 +2478,7 @@ func fixOrgUser(ctx context.Context, org *Org) *Org { // continue // } - // org, err := getOrg(ctx, orgId) + // org, err := shuffle.GetOrg(ctx, orgId) // if err != nil { // log.Printf("Error getting org %s", orgId) // continue @@ -2718,21 +2514,7 @@ func fixOrgUser(ctx context.Context, org *Org) *Org { return org } -// ListBooks returns a list of books, ordered by title. -func setUser(ctx context.Context, data *User) error { - data = fixUserOrg(ctx, data) - - // clear session_token and API_token for user - k := datastore.NameKey("Users", data.Id, nil) - if _, err := dbclient.Put(ctx, k, data); err != nil { - log.Println(err) - return err - } - - return nil -} - -func fixUserOrg(ctx context.Context, user *User) *User { +func fixUserOrg(ctx context.Context, user *shuffle.User) *shuffle.User { found := false for _, id := range user.Orgs { if user.ActiveOrg.Id == id { @@ -2751,7 +2533,7 @@ func fixUserOrg(ctx context.Context, user *User) *User { continue } - org, err := getOrg(ctx, orgId) + org, err := shuffle.GetOrg(ctx, orgId) if err != nil { log.Printf("Error getting org %s", orgId) continue @@ -2768,10 +2550,10 @@ func fixUserOrg(ctx context.Context, user *User) *User { } if userFound { - user.PrivateApps = []WorkflowApp{} - user.Executions = ExecutionInfo{} - user.Limits = UserLimits{} - user.Authentication = []UserAuth{} + user.PrivateApps = []shuffle.WorkflowApp{} + user.Executions = shuffle.ExecutionInfo{} + user.Limits = shuffle.UserLimits{} + user.Authentication = []shuffle.UserAuth{} org.Users[orgIndex] = *user } else { @@ -2955,7 +2737,7 @@ func handleSetHook(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -3197,15 +2979,15 @@ func setSpecificSchedule(resp http.ResponseWriter, request *http.Request) { return } -func getSchedule(ctx context.Context, schedulename string) (*ScheduleOld, error) { - key := datastore.NameKey("schedules", strings.ToLower(schedulename), nil) - curUser := &ScheduleOld{} - if err := dbclient.Get(ctx, key, curUser); err != nil { - return &ScheduleOld{}, err - } - - return curUser, nil -} +//func GetSchedule(ctx context.Context, schedulename string) (*ScheduleOld, error) { +// key := datastore.NameKey("schedules", strings.ToLower(schedulename), nil) +// curUser := &ScheduleOld{} +// if err := dbclient.Get(ctx, key, curUser); err != nil { +// return &ScheduleOld{}, err +// } +// +// return curUser, nil +//} func getSpecificWebhook(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) @@ -3234,7 +3016,7 @@ func getSpecificWebhook(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() // FIXME: Schedule = trigger? - schedule, err := getSchedule(ctx, workflowId) + schedule, err := shuffle.GetSchedule(ctx, workflowId) if err != nil { log.Printf("Failed setting schedule: %s", err) resp.WriteHeader(401) @@ -3265,7 +3047,7 @@ func handleDeleteSchedule(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -3476,7 +3258,7 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { for _, item := range hook.Workflows { //log.Printf("Running webhook for workflow %s with startnode %s", item, hook.Start) - workflow := Workflow{ + workflow := shuffle.Workflow{ ID: "", } @@ -3573,7 +3355,7 @@ func handleNewHook(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -3653,7 +3435,7 @@ func handleNewHook(resp http.ResponseWriter, request *http.Request) { if requestdata.Environment == "cloud" { // https://shuffler.io/v1/hooks/webhook_80184973-3e82-4852-842e-0290f7f34d7c log.Printf("[INFO] Should START a cloud webhook for url %s for startnode %s", currentUrl, startNode) - org, err := getOrg(ctx, user.ActiveOrg.Id) + org, err := shuffle.GetOrg(ctx, user.ActiveOrg.Id) if err != nil { log.Printf("Failed finding org %s: %s", org.Id, err) return @@ -3730,7 +3512,7 @@ func sendHookResult(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -3798,7 +3580,7 @@ func handleGetHook(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -3886,7 +3668,7 @@ func getSpecificSchedule(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - schedule, err := getSchedule(ctx, workflowId) + schedule, err := shuffle.GetSchedule(ctx, workflowId) if err != nil { log.Printf("Failed getting schedule: %s", err) resp.WriteHeader(401) @@ -3953,7 +3735,7 @@ func executeSchedule(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() log.Printf("[INFO] EXECUTING %s!", workflowId) - idConfig, err := getSchedule(ctx, workflowId) + idConfig, err := shuffle.GetSchedule(ctx, workflowId) if err != nil { log.Printf("Error getting schedule: %s", err) resp.WriteHeader(401) @@ -4064,7 +3846,7 @@ func uploadWorkflowResult(resp http.ResponseWriter, request *http.Request) { // FIXME - validate ID as well ctx := context.Background() - schedule, err := getSchedule(ctx, workflowId) + schedule, err := shuffle.GetSchedule(ctx, workflowId) if err != nil { log.Printf("Failed setting schedule %s: %s", workflowId, err) resp.WriteHeader(401) @@ -4496,7 +4278,7 @@ func handleGetallHooks(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -4566,7 +4348,7 @@ func findAvailablePorts(startRange int64, endRange int64) string { } func handleSendalert(resp http.ResponseWriter, request *http.Request) { - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in sendalert: %s", err) resp.WriteHeader(401) @@ -4903,7 +4685,7 @@ func handleGetSpecificStats(resp http.ResponseWriter, request *http.Request) { return } - _, err := handleApiAuthentication(resp, request) + _, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in getting specific workflow: %s", err) resp.WriteHeader(401) @@ -4953,7 +4735,7 @@ func getOpenapi(resp http.ResponseWriter, request *http.Request) { } // Just here to verify that the user is logged in - _, err := handleApiAuthentication(resp, request) + _, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in validate swagger: %s", err) resp.WriteHeader(401) @@ -4981,7 +4763,7 @@ func getOpenapi(resp http.ResponseWriter, request *http.Request) { // FIXME - FIX AUTH WITH APP ctx := context.Background() - //_, err = getApp(ctx, id) + //_, err = shuffle.GetApp(ctx, id) //if err == nil { // log.Println("You're supposed to be able to continue now.") //} @@ -5014,7 +4796,7 @@ func echoOpenapiData(resp http.ResponseWriter, request *http.Request) { } // Just here to verify that the user is logged in - _, err := handleApiAuthentication(resp, request) + _, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in validate swagger: %s", err) resp.WriteHeader(401) @@ -5201,7 +4983,7 @@ func validateSwagger(resp http.ResponseWriter, request *http.Request) { } // Just here to verify that the user is logged in - _, err := handleApiAuthentication(resp, request) + _, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in validate swagger: %s", err) resp.WriteHeader(401) @@ -5394,7 +5176,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { } log.Printf("[INFO] SETTING APP TO LIVE!!!") - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in verify swagger: %s", err) resp.WriteHeader(401) @@ -5431,7 +5213,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { if test.Editing { // Quick verification test ctx := context.Background() - app, err := getApp(ctx, test.Id) + app, err := shuffle.GetApp(ctx, test.Id) if err != nil { log.Printf("Error getting app when editing: %s", app.Name) resp.WriteHeader(401) @@ -5476,7 +5258,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { swagger.Info.Title = strings.Replace(swagger.Info.Title, " ", "_", -1) } - basePath, err := buildStructure(swagger, newmd5) + basePath, err := shuffle.BuildStructure(swagger, newmd5) if err != nil { log.Printf("Failed to build base structure: %s", err) resp.WriteHeader(500) @@ -5485,7 +5267,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { } //log.Printf("Should generate yaml") - swagger, api, pythonfunctions, err := generateYaml(swagger, newmd5) + swagger, api, pythonfunctions, err := shuffle.GenerateYaml(swagger, newmd5) if err != nil { log.Printf("Failed building and generating yaml: %s", err) resp.WriteHeader(500) @@ -5495,7 +5277,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { // FIXME: CHECK IF SAME NAME AS NORMAL APP // Can't overwrite existing normal app - workflowApps, err := getAllWorkflowApps(ctx, 500) + workflowApps, err := shuffle.GetAllWorkflowApps(ctx, 500) if err != nil { log.Printf("Failed getting all workflow apps from database to verify: %s", err) resp.WriteHeader(401) @@ -5515,7 +5297,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { api.Owner = user.Id - err = dumpApi(basePath, api) + err = shuffle.DumpApi(basePath, api) if err != nil { log.Printf("Failed dumping yaml: %s", err) resp.WriteHeader(500) @@ -5526,7 +5308,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, newmd5) classname := strings.Replace(identifier, " ", "", -1) classname = strings.Replace(classname, "-", "", -1) - parsedCode, err := dumpPython(basePath, classname, swagger.Info.Version, pythonfunctions) + parsedCode, err := shuffle.DumpPython(basePath, classname, swagger.Info.Version, pythonfunctions) if err != nil { log.Printf("Failed dumping python: %s", err) resp.WriteHeader(500) @@ -5546,7 +5328,8 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { // 5. Upload as cloud function // 1. Upload the API to datastore - err = deployAppToDatastore(ctx, api) + err = shuffle.DeployAppToDatastore(ctx, api) + //func DeployAppToDatastore(ctx context.Context, workflowapp WorkflowApp, bucketName string) error { if err != nil { log.Printf("Failed adding app to db: %s", err) resp.WriteHeader(500) @@ -5555,7 +5338,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { } // 2. Get all the required code - appbase, staticBaseline, err := getAppbase() + appbase, staticBaseline, err := shuffle.GetAppbase() if err != nil { log.Printf("Failed getting appbase: %s", err) resp.WriteHeader(500) @@ -5564,17 +5347,17 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { } // Have to do some quick checks of the python code (: - _, parsedCode = formatAppfile(parsedCode) + _, parsedCode = shuffle.FormatAppfile(parsedCode) - fixedAppbase := fixAppbase(appbase) - runner := getRunner(classname) + fixedAppbase := shuffle.FixAppbase(appbase) + runner := shuffle.GetRunnerOnprem(classname) // 2. Put it together stitched := string(staticBaseline) + strings.Join(fixedAppbase, "\n") + parsedCode + string(runner) //log.Println(stitched) // 3. Zip and stream it directly in the directory - _, err = streamZipdata(ctx, identifier, stitched, "requests\nurllib3") + _, err = shuffle.StreamZipdata(ctx, identifier, stitched, "requests\nurllib3", "") if err != nil { log.Printf("[ERROR] Zipfile error: %s", err) resp.WriteHeader(500) @@ -5643,7 +5426,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { user.PrivateApps[foundNumber] = api } - err = setUser(ctx, &user) + err = shuffle.SetUser(ctx, &user) if err != nil { log.Printf("[ERROR] Failed adding verification for user %s: %s", user.Username, err) resp.WriteHeader(500) @@ -5804,7 +5587,7 @@ func handleCloudExecutionOnprem(workflowId, startNode, executionSource, executio ctx := context.Background() // 1. Get the workflow // 2. Execute it with the data - workflow, err := getWorkflow(ctx, workflowId) + workflow, err := shuffle.GetWorkflow(ctx, workflowId) if err != nil { return err } @@ -5813,12 +5596,12 @@ func handleCloudExecutionOnprem(workflowId, startNode, executionSource, executio _ = workflow parsedArgument := executionArgument - newExec := ExecutionRequest{ + newExec := shuffle.ExecutionRequest{ ExecutionSource: executionSource, ExecutionArgument: parsedArgument, } - var execution ExecutionRequest + var execution shuffle.ExecutionRequest err = json.Unmarshal([]byte(parsedArgument), &execution) if err == nil { //log.Printf("[INFO] FOUND EXEC %#v", execution) @@ -5853,7 +5636,7 @@ func handleCloudExecutionOnprem(workflowId, startNode, executionSource, executio Body: ioutil.NopCloser(bytes.NewReader(b)), } - _, _, err = handleExecution(workflowId, Workflow{}, newRequest) + _, _, err = handleExecution(workflowId, shuffle.Workflow{}, newRequest) return err } @@ -5974,7 +5757,7 @@ func handleCloudJob(job CloudSyncJob) error { return err } - _, _, err = handleExecution(job.PrimaryItemId, Workflow{}, newRequest) + _, _, err = handleExecution(job.PrimaryItemId, shuffle.Workflow{}, newRequest) if err != nil { log.Printf("Failed continuing workflow from cloud user_input: %s", err) return err @@ -5999,7 +5782,7 @@ func handleCloudJob(job CloudSyncJob) error { } */ - newResults := []ActionResult{} + newResults := []shuffle.ActionResult{} for _, result := range workflowExecution.Results { if result.Action.AppName == "User Input" && result.Result == "Waiting for user feedback based on configuration" { result.Status = "ABORTED" @@ -6026,7 +5809,7 @@ func handleCloudJob(job CloudSyncJob) error { } // Handles jobs from remote (cloud) -func remoteOrgJobController(org Org, body []byte) error { +func remoteOrgJobController(org shuffle.Org, body []byte) error { type retStruct struct { Success bool `json:"success"` Reason string `json:"reason"` @@ -6051,7 +5834,7 @@ func remoteOrgJobController(org Org, body []byte) error { log.Printf("[WARNING] STOPPING ORG SCHEDULE for: %s", org.Id) value.Lock() - org, err := getOrg(ctx, org.Id) + org, err := shuffle.GetOrg(ctx, org.Id) if err != nil { log.Printf("[WARNING] Failed finding org %s: %s", org.Id, err) return err @@ -6091,7 +5874,7 @@ func remoteOrgJobController(org Org, body []byte) error { return nil } -func remoteOrgJobHandler(org Org, interval int) error { +func remoteOrgJobHandler(org shuffle.Org, interval int) error { client := &http.Client{} syncUrl := fmt.Sprintf("%s/api/v1/cloud/sync", syncUrl) req, err := http.NewRequest( @@ -6179,7 +5962,7 @@ func runInit(ctx context.Context) { setUsers := false orgQuery := datastore.NewQuery("Organizations") - var activeOrgs []Org + var activeOrgs []shuffle.Org _, err = dbclient.GetAll(ctx, orgQuery, &activeOrgs) if err != nil { log.Printf("Error getting organizations!") @@ -6194,11 +5977,11 @@ func runInit(ctx context.Context) { log.Printf(`No orgs. Setting org "default"`) orgSetupName := "default" orgId := uuid.NewV4().String() - newOrg := Org{ + newOrg := shuffle.Org{ Name: orgSetupName, Id: orgId, Org: orgSetupName, - Users: []User{}, + Users: []shuffle.User{}, Roles: []string{"admin", "user"}, CloudSync: false, } @@ -6220,15 +6003,15 @@ func runInit(ctx context.Context) { activeOrg := activeOrgs[0] q := datastore.NewQuery("Users") - var users []User + var users []shuffle.User _, err = dbclient.GetAll(ctx, q, &users) if err == nil { setOrgBool := false for _, user := range users { - newUser := User{ + newUser := shuffle.User{ Username: user.Username, Id: user.Id, - ActiveOrg: Org{ + ActiveOrg: shuffle.Org{ Id: activeOrg.Id, }, Orgs: []string{activeOrg.Id}, @@ -6272,13 +6055,13 @@ func runInit(ctx context.Context) { // Fix active users etc q := datastore.NewQuery("Users").Filter("active =", true) - var activeusers []User + var activeusers []shuffle.User _, err = dbclient.GetAll(ctx, q, &activeusers) if err != nil { log.Printf("Error getting users during init: %s", err) } else { q := datastore.NewQuery("Users") - var users []User + var users []shuffle.User _, err := dbclient.GetAll(ctx, q, &users) if len(activeusers) == 0 && len(users) > 0 { @@ -6298,13 +6081,13 @@ func runInit(ctx context.Context) { if len(user.Orgs) == 0 { defaultName := "default" user.Orgs = []string{defaultName} - user.ActiveOrg = Org{ + user.ActiveOrg = shuffle.Org{ Name: defaultName, Role: "user", } } - err = setUser(ctx, &user) + err = shuffle.SetUser(ctx, &user) if err != nil { log.Printf("Failed to reset user") } else { @@ -6325,7 +6108,7 @@ func runInit(ctx context.Context) { } else { apikey := os.Getenv("SHUFFLE_DEFAULT_APIKEY") - tmpOrg := Org{ + tmpOrg := shuffle.Org{ Name: "default", } err = createNewUser(username, password, "admin", apikey, tmpOrg) @@ -6348,7 +6131,7 @@ func runInit(ctx context.Context) { for _, user := range users { if user.ActiveOrg.Id == "" && len(user.Username) > 0 { user.ActiveOrg = activeOrgs[0] - err = setUser(ctx, &user) + err = shuffle.SetUser(ctx, &user) if err != nil { log.Printf("Failed updating user %s with org", user.Username) } else { @@ -6365,7 +6148,7 @@ func runInit(ctx context.Context) { count, err := getEnvironmentCount() if count == 0 && err == nil && len(activeOrgs) == 1 { log.Printf("Setting up environment with org %s", activeOrgs[0].Id) - item := Environment{ + item := shuffle.Environment{ Name: "Shuffle", Type: "onprem", OrgId: activeOrgs[0].Id, @@ -6378,7 +6161,7 @@ func runInit(ctx context.Context) { } } else if len(activeOrgs) == 1 { log.Printf("Setting up all environments with org %s", activeOrgs[0].Id) - var environments []Environment + var environments []shuffle.Environment q := datastore.NewQuery("Environments") _, err = dbclient.GetAll(ctx, q, &environments) if err == nil { @@ -6399,7 +6182,7 @@ func runInit(ctx context.Context) { // Fixing workflows to have real activeorg IDs if len(activeOrgs) == 1 { q := datastore.NewQuery("workflow").Limit(35) - var workflows []Workflow + var workflows []shuffle.Workflow _, err = dbclient.GetAll(ctx, q, &workflows) if err != nil { log.Printf("Error getting workflows in runinit: %s", err) @@ -6467,7 +6250,7 @@ func runInit(ctx context.Context) { } */ - var allworkflowapps []AppAuthenticationStorage + var allworkflowapps []shuffle.AppAuthenticationStorage q = datastore.NewQuery("workflowappauth") _, err = dbclient.GetAll(ctx, q, &allworkflowapps) if err == nil { @@ -6479,7 +6262,7 @@ func runInit(ctx context.Context) { //log.Printf("Should update auth for %#v!", item) item.OrgId = activeOrgs[0].Id - err = setWorkflowAppAuthDatastore(ctx, item, item.Id) + err = shuffle.SetWorkflowAppAuthDatastore(ctx, item, item.Id) if err != nil { log.Printf("Failed adding AUTH to org %s", activeOrgs[0].Id) } @@ -6567,7 +6350,7 @@ func runInit(ctx context.Context) { Body: ioutil.NopCloser(strings.NewReader(schedule.WrappedArgument)), } - _, _, err := handleExecution(schedule.WorkflowId, Workflow{}, request) + _, _, err := handleExecution(schedule.WorkflowId, shuffle.Workflow{}, request) if err != nil { log.Printf("Failed to execute %s: %s", schedule.WorkflowId, err) } @@ -6593,18 +6376,18 @@ func runInit(ctx context.Context) { // Getting apps to see if we should initialize a test log.Printf("Getting remote workflow apps") - workflowapps, err := getAllWorkflowApps(ctx, 500) + workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500) if err != nil { log.Printf("Failed getting apps (runInit): %s", err) } else if err == nil && len(workflowapps) > 0 { - //getAllWorkflowApps(ctx context.Context) ([]WorkflowApp, error) { - var allworkflowapps []WorkflowApp + //shuffle.GetAllWorkflowApps(ctx context.Context) ([]WorkflowApp, error) { + var allworkflowapps []shuffle.WorkflowApp q := datastore.NewQuery("workflowapp") _, err := dbclient.GetAll(ctx, q, &allworkflowapps) if err == nil { for _, workflowapp := range allworkflowapps { if workflowapp.Edited == 0 { - err = setWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) + err = shuffle.SetWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) if err == nil { log.Printf("Updating time for workflowapp %s:%s", workflowapp.Name, workflowapp.AppVersion) } @@ -6693,7 +6476,7 @@ func runInit(ctx context.Context) { if len(workflowLocation) > 0 { log.Printf("Downloading WORKFLOWS from %s if no workflows - EXTRA workflows", workflowLocation) q := datastore.NewQuery("workflow").Limit(35) - var workflows []Workflow + var workflows []shuffle.Workflow _, err = dbclient.GetAll(ctx, q, &workflows) if err != nil { log.Printf("Error getting workflows: %s", err) @@ -6722,11 +6505,11 @@ func runInit(ctx context.Context) { log.Printf("[INFO] Finished INIT") } -func handleVerifyCloudsync(orgId string) (SyncFeatures, error) { +func handleVerifyCloudsync(orgId string) (shuffle.SyncFeatures, error) { ctx := context.Background() - org, err := getOrg(ctx, orgId) + org, err := shuffle.GetOrg(ctx, orgId) if err != nil { - return SyncFeatures{}, err + return shuffle.SyncFeatures{}, err } //r.HandleFunc("/api/v1/getorgs", handleGetOrgs).Methods("GET", "OPTIONS") @@ -6742,26 +6525,26 @@ func handleVerifyCloudsync(orgId string) (SyncFeatures, error) { req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, org.SyncConfig.Apikey)) newresp, err := client.Do(req) if err != nil { - return SyncFeatures{}, err + return shuffle.SyncFeatures{}, err } respBody, err := ioutil.ReadAll(newresp.Body) if err != nil { - return SyncFeatures{}, err + return shuffle.SyncFeatures{}, err } responseData := retStruct{} err = json.Unmarshal(respBody, &responseData) if err != nil { - return SyncFeatures{}, err + return shuffle.SyncFeatures{}, err } if newresp.StatusCode != 200 { - return SyncFeatures{}, errors.New(fmt.Sprintf("Got status code %d when getting org remotely. Expected 200. Contact support.", newresp.StatusCode)) + return shuffle.SyncFeatures{}, errors.New(fmt.Sprintf("Got status code %d when getting org remotely. Expected 200. Contact support.", newresp.StatusCode)) } if !responseData.Success { - return SyncFeatures{}, errors.New(responseData.Reason) + return shuffle.SyncFeatures{}, errors.New(responseData.Reason) } return responseData.SyncFeatures, nil @@ -6769,7 +6552,7 @@ func handleVerifyCloudsync(orgId string) (SyncFeatures, error) { // Actually stops syncing with cloud for an org. // Disables potential schedules, removes environments, breaks workflows etc. -func handleStopCloudSync(syncUrl string, org Org) error { +func handleStopCloudSync(syncUrl string, org shuffle.Org) error { if len(org.SyncConfig.Apikey) == 0 { return errors.New(fmt.Sprintf("Couldn't find any sync key to disable org %s", org.Id)) } @@ -6814,8 +6597,8 @@ func handleStopCloudSync(syncUrl string, org Org) error { ctx := context.Background() org.CloudSync = false - org.SyncFeatures = SyncFeatures{} - org.SyncConfig = SyncConfig{} + org.SyncFeatures = shuffle.SyncFeatures{} + org.SyncConfig = shuffle.SyncConfig{} err = setOrg(ctx, org, org.Id) if err != nil { @@ -6824,7 +6607,7 @@ func handleStopCloudSync(syncUrl string, org Org) error { return errors.New(newerror) } - var environments []Environment + var environments []shuffle.Environment q := datastore.NewQuery("Environments").Filter("org_id =", org.Id) _, err = dbclient.GetAll(ctx, q, &environments) if err != nil { @@ -6922,7 +6705,7 @@ func handleKeyValueCheck(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() - org, err := getOrg(ctx, tmpData.OrgId) + org, err := shuffle.GetOrg(ctx, tmpData.OrgId) if err != nil { log.Printf("[INFO] Organization doesn't exist: %s", err) resp.WriteHeader(401) @@ -7103,7 +6886,7 @@ func handleEditOrg(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in cloud setup: %s", err) resp.WriteHeader(401) @@ -7126,11 +6909,11 @@ func handleEditOrg(resp http.ResponseWriter, request *http.Request) { } type ReturnData struct { - Image string `json:"image" datastore:"image"` - Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description"` - OrgId string `json:"org_id" datastore:"org_id"` - Defaults Defaults `json:"defaults" datastore:"defaults"` + Image string `json:"image" datastore:"image"` + Name string `json:"name" datastore:"name"` + Description string `json:"description" datastore:"description"` + OrgId string `json:"org_id" datastore:"org_id"` + Defaults shuffle.Defaults `json:"defaults" datastore:"defaults"` } var tmpData ReturnData @@ -7163,7 +6946,7 @@ func handleEditOrg(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - org, err := getOrg(ctx, tmpData.OrgId) + org, err := shuffle.GetOrg(ctx, tmpData.OrgId) if err != nil { log.Printf("Organization doesn't exist: %s", err) resp.WriteHeader(401) @@ -7228,7 +7011,7 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in cloud setup: %s", err) resp.WriteHeader(401) @@ -7251,9 +7034,9 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { } type ReturnData struct { - Apikey string `datastore:"apikey"` - Organization Org `datastore:"organization"` - Disable bool `datastore:"disable"` + Apikey string `datastore:"apikey"` + Organization shuffle.Org `datastore:"organization"` + Disable bool `datastore:"disable"` } var tmpData ReturnData @@ -7266,7 +7049,7 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - org, err := getOrg(ctx, tmpData.Organization.Id) + org, err := shuffle.GetOrg(ctx, tmpData.Organization.Id) if err != nil { log.Printf("Organization doesn't exist: %s", err) resp.WriteHeader(401) @@ -7411,7 +7194,7 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { org.CloudSync = true org.SyncFeatures = responseData.SyncFeatures - org.SyncConfig = SyncConfig{ + org.SyncConfig = shuffle.SyncConfig{ Apikey: responseData.SessionKey, Interval: responseData.IntervalSeconds, } @@ -7440,7 +7223,7 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { // 1. Find environment // 2. If cloud env found, enable it (un-archive) // 3. If it doesn't create it - var environments []Environment + var environments []shuffle.Environment q := datastore.NewQuery("Environments").Filter("org_id =", org.Id) _, err = dbclient.GetAll(ctx, q, &environments) if err == nil { @@ -7465,7 +7248,7 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { if !found { log.Printf("Env for cloud not found. Should add it!") - newEnv := Environment{ + newEnv := shuffle.Environment{ Name: "Cloud", Type: "cloud", Archived: false, @@ -7509,7 +7292,7 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { // return // } // -// user, err := handleApiAuthentication(resp, request) +// user, err := shuffle.HandleApiAuthentication(resp, request) // if err != nil { // log.Printf("Api authentication failed in cloud setup: %s", err) // resp.WriteHeader(401) @@ -7584,7 +7367,7 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { // return // } // -// org, err := getOrg(ctx, tmpData.OrgId) +// org, err := shuffle.GetOrg(ctx, tmpData.OrgId) // if err != nil { // log.Printf("Organization doesn't exist: %s", err) // resp.WriteHeader(401) @@ -7684,12 +7467,18 @@ func initHandlers() { ctx := context.Background() log.Printf("Starting Shuffle backend - initializing database connection") - // option.WithoutAuthentication - + requestCache = cache.New(5*time.Minute, 10*time.Minute) dbclient, err = datastore.NewClient(ctx, gceProject, option.WithGRPCDialOption(grpc.WithNoProxy())) if err != nil { panic(fmt.Sprintf("DBclient error during init: %s", err)) } + + //dbclient, err := shuffle.GetDatastoreClient(ctx, gceProject) + //if err != nil { + // panic(fmt.Sprintf("Error setting datastore connector: %s", err)) + //} + + _ = shuffle.RunInit(*dbclient, storage.Client{}, gceProject, "onprem", true) log.Printf("Finished Shuffle database init") go runInit(ctx) @@ -7815,12 +7604,12 @@ func initHandlers() { // PS: For cloud, this has to use cloud storage. // https://developer.box.com/reference/get-files-id-content/ // 1. Creating the "get file" option. Make it possible to run this in the frontend. - r.HandleFunc("/api/v1/files/{fileId}/content", handleGetFileContent).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/files/create", handleCreateFile).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/files/{fileId}/upload", handleUploadFile).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/files/{fileId}", handleGetFileMeta).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/files/{fileId}", handleDeleteFile).Methods("DELETE", "OPTIONS") - r.HandleFunc("/api/v1/files", handleGetFiles).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/files/{fileId}/content", shuffle.HandleGetFileContent).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/files/create", shuffle.HandleCreateFile).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/files/{fileId}/upload", shuffle.HandleUploadFile).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/files/{fileId}", shuffle.HandleGetFileMeta).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/files/{fileId}", shuffle.HandleDeleteFile).Methods("DELETE", "OPTIONS") + r.HandleFunc("/api/v1/files", shuffle.HandleGetFiles).Methods("GET", "OPTIONS") // Trigger hmm r.HandleFunc("/api/v1/triggers/outlook/register", handleNewOutlookRegister).Methods("GET", "OPTIONS") diff --git a/backend/go-app/oauth2.go b/backend/go-app/oauth2.go index b00f6c7a..3c03ea13 100644 --- a/backend/go-app/oauth2.go +++ b/backend/go-app/oauth2.go @@ -1,6 +1,8 @@ package main import ( + "github.com/frikky/shuffle-shared" + "bytes" "context" "encoding/json" @@ -423,32 +425,32 @@ func handleNewOutlookRegister(resp http.ResponseWriter, request *http.Request) { } // Should also update the user - Userdata, err := getUser(ctx, senderUser) + Userdata, err := shuffle.GetUser(ctx, senderUser) if err != nil { log.Printf("[INFO] Username %s doesn't exist (oauth2): %s", trigger.Username, err) resp.WriteHeader(401) return } - Userdata.Authentication = append(Userdata.Authentication, UserAuth{ + Userdata.Authentication = append(Userdata.Authentication, shuffle.UserAuth{ Name: "Outlook", Description: "oauth2", Workflows: []string{trigger.WorkflowId}, Username: trigger.Username, - Fields: []UserAuthField{ - UserAuthField{ + Fields: []shuffle.UserAuthField{ + shuffle.UserAuthField{ Key: "trigger_id", Value: trigger.Id, }, - UserAuthField{ + shuffle.UserAuthField{ Key: "username", Value: trigger.Username, }, - UserAuthField{ + shuffle.UserAuthField{ Key: "code", Value: code, }, - UserAuthField{ + shuffle.UserAuthField{ Key: "type", Value: trigger.Type, }, @@ -456,7 +458,7 @@ func handleNewOutlookRegister(resp http.ResponseWriter, request *http.Request) { }) // Set apikey for the user if they don't have one - err = setUser(ctx, Userdata) + err = shuffle.SetUser(ctx, Userdata) if err != nil { log.Printf("Failed setting user data for %s: %s", Userdata.Username, err) resp.WriteHeader(401) @@ -636,7 +638,7 @@ func handleGetSpecificTrigger(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in getting specific workflow: %s", err) resp.WriteHeader(401) @@ -714,7 +716,7 @@ func createOutlookSub(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - workflow, err := getWorkflow(ctx, workflowId) + workflow, err := shuffle.GetWorkflow(ctx, workflowId) if err != nil { log.Printf("Failed getting the workflow locally (outlook sub): %s", err) resp.WriteHeader(401) @@ -722,7 +724,7 @@ func createOutlookSub(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in outlook deploy: %s", err) resp.WriteHeader(401) @@ -808,7 +810,7 @@ func createOutlookSub(resp http.ResponseWriter, request *http.Request) { // 10 * 5 = 50 seconds. That's waaay too much :( if runningEnvironment != "cloud" { - org, err := getOrg(ctx, user.ActiveOrg.Id) + org, err := shuffle.GetOrg(ctx, user.ActiveOrg.Id) if err != nil { log.Printf("Failed finding org %s: %s", org.Id, err) return @@ -1147,7 +1149,7 @@ func handleOutlookCallback(resp http.ResponseWriter, request *http.Request) { Body: ioutil.NopCloser(bytes.NewReader(b)), } - workflow := Workflow{ + workflow := shuffle.Workflow{ ID: "", } @@ -1196,7 +1198,7 @@ func removeOutlookSubscription(outlookClient *http.Client, subscriptionId string // Remove AUTH // Remove function // Remove subscription -func handleOutlookSubRemoval(ctx context.Context, user User, workflowId, triggerId string) error { +func handleOutlookSubRemoval(ctx context.Context, user shuffle.User, workflowId, triggerId string) error { // 1. Get the auth for trigger // 2. Stop the subscription // 3. Remove the function @@ -1209,7 +1211,7 @@ func handleOutlookSubRemoval(ctx context.Context, user User, workflowId, trigger if runningEnvironment != "cloud" { log.Printf("[INFO] SHOULD STOP OUTLOOK SUB ONPREM SYNC WITH CLOUD for workflow ID %s", workflowId) - org, err := getOrg(ctx, user.ActiveOrg.Id) + org, err := shuffle.GetOrg(ctx, user.ActiveOrg.Id) if err != nil { log.Printf("[INFO] Failed finding org %s during outlook removal: %s", org.Id, err) return err @@ -1289,7 +1291,7 @@ func handleDeleteOutlookSub(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - workflow, err := getWorkflow(ctx, workflowId) + workflow, err := shuffle.GetWorkflow(ctx, workflowId) if err != nil { log.Printf("Failed getting the workflow locally (delete outlook): %s", err) resp.WriteHeader(401) @@ -1297,7 +1299,7 @@ func handleDeleteOutlookSub(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in outlook deploy: %s", err) resp.WriteHeader(401) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 8af8055e..10b96f19 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -1,6 +1,8 @@ package main import ( + "github.com/frikky/shuffle-shared" + "bytes" "context" "encoding/json" @@ -29,7 +31,7 @@ import ( schedulerpb "google.golang.org/genproto/googleapis/cloud/scheduler/v1" newscheduler "github.com/carlescere/scheduler" - "github.com/getkin/kin-openapi/openapi3" + "github.com/frikky/kin-openapi/openapi3" "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-git/v5" @@ -63,435 +65,416 @@ var scheduledOrgs = map[string]*newscheduler.Job{} // }, //} -type ExecutionRequest struct { - ExecutionId string `json:"execution_id,omitempty"` - ExecutionArgument string `json:"execution_argument,omitempty"` - ExecutionSource string `json:"execution_source,omitempty"` - WorkflowId string `json:"workflow_id,omitempty"` - Environments []string `json:"environments,omitempty"` - Authorization string `json:"authorization,omitempty"` - Status string `json:"status,omitempty"` - Start string `json:"start,omitempty"` - Type string `json:"type,omitempty"` -} - -type SyncFeatures struct { - Webhook SyncData `json:"webhook" datastore:"webhook"` - Schedules SyncData `json:"schedules" datastore:"schedules"` - UserInput SyncData `json:"user_input" datastore:"user_input"` - SendMail SyncData `json:"send_mail" datastore:"send_mail"` - SendSms SyncData `json:"send_sms" datastore:"send_sms"` - Updates SyncData `json:"updates" datastore:"updates"` - Notifications SyncData `json:"notifications" datastore:"notifications"` - EmailTrigger SyncData `json:"email_trigger" datastore:"email_trigger"` - AppExecutions SyncData `json:"app_executions" datastore:"app_executions"` - WorkflowExecutions SyncData `json:"workflow_executions" datastore:"workflow_executions"` - Apps SyncData `json:"apps" datastore:"apps"` - Workflows SyncData `json:"workflows" datastore:"workflows"` - Autocomplete SyncData `json:"autocomplete" datastore:"autocomplete"` - Authentication SyncData `json:"authentication" datastore:"authentication"` - Schedule SyncData `json:"schedule" datastore:"schedule"` -} - -type SyncData struct { - Active bool `json:"active" datastore:"active"` - Type string `json:"type,omitempty" datastore:"type"` - Name string `json:"name,omitempty" datastore:"name"` - Description string `json:"description,omitempty" datastore:"description"` - Limit int64 `json:"limit,omitempty" datastore:"limit"` - StartDate int64 `json:"start_date,omitempty" datastore:"start_date"` - EndDate int64 `json:"end_date,omitempty" datastore:"end_date"` - DataCollection int64 `json:"data_collection,omitempty" datastore:"data_collection"` -} - -type SyncConfig struct { - Interval int64 `json:"interval" datastore:"interval"` - Apikey string `json:"api_key" datastore:"api_key"` -} - -// Role is just used for feedback for a user -type Org struct { - Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description"` - Image string `json:"image" datastore:"image,noindex"` - Id string `json:"id" datastore:"id"` - Org string `json:"org" datastore:"org"` - Users []User `json:"users" datastore:"users"` - Role string `json:"role" datastore:"role"` - Roles []string `json:"roles" datastore:"roles"` - CloudSync bool `json:"cloud_sync" datastore:"CloudSync"` - SyncConfig SyncConfig `json:"sync_config" datastore:"sync_config"` - SyncFeatures SyncFeatures `json:"sync_features" datastore:"sync_features"` - Subscriptions []PaymentSubscription `json:"subscriptions" datastore:"subscriptions"` - Created int64 `json:"created" datastore:"created"` - Edited int64 `json:"edited" datastore:"edited"` - Defaults Defaults `json:"defaults" datastore:"defaults"` -} - -type PaymentSubscription struct { - Active bool `json:"active" datastore:"active"` - Startdate int64 `json:"startdate" datastore:"startdate"` - CancellationDate int64 `json:"cancellationdate" datastore:"cancellationdate"` - Enddate int64 `json:"enddate" datastore:"enddate"` - Name string `json:"name" datastore:"name"` - Recurrence string `json:"recurrence" datastore:"recurrence"` - Reference string `json:"reference" datastore:"reference"` - Level string `json:"level" datastore:"level"` - Amount string `json:"amount" datastore:"amount"` - Currency string `json:"currency" datastore:"currency"` -} - -type Defaults struct { - AppDownloadRepo string `json:"app_download_repo" datastore:"app_download_repo"` - AppDownloadBranch string `json:"app_download_branch" datastore:"app_download_branch"` - WorkflowDownloadRepo string `json:"workflow_download_repo" datastore:"workflow_download_repo"` - WorkflowDownloadBranch string `json:"workflow_download_branch" datastore:"workflow_download_branch"` -} - -type AppAuthenticationStorage struct { - Active bool `json:"active" datastore:"active"` - Label string `json:"label" datastore:"label"` - Id string `json:"id" datastore:"id"` - App WorkflowApp `json:"app" datastore:"app,noindex"` - Fields []AuthenticationStore `json:"fields" datastore:"fields"` - Usage []AuthenticationUsage `json:"usage" datastore:"usage"` - WorkflowCount int64 `json:"workflow_count" datastore:"workflow_count"` - NodeCount int64 `json:"node_count" datastore:"node_count"` - OrgId string `json:"org_id" datastore:"org_id"` - Created int64 `json:"created" datastore:"created"` - Edited int64 `json:"edited" datastore:"edited"` - Defined bool `json:"defined" datastore:"defined"` -} - -type AuthenticationUsage struct { - WorkflowId string `json:"workflow_id" datastore:"workflow_id"` - Nodes []string `json:"nodes" datastore:"nodes"` -} - -// An app inside Shuffle -// Source string `json:"source" datastore:"soure" yaml:"source"` - downloadlocation -type WorkflowApp struct { - Name string `json:"name" yaml:"name" required:true datastore:"name"` - IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"` - ID string `json:"id" yaml:"id,omitempty" required:false datastore:"id"` - Link string `json:"link" yaml:"link" required:false datastore:"link,noindex"` - AppVersion string `json:"app_version" yaml:"app_version" required:true datastore:"app_version"` - SharingConfig string `json:"sharing_config" yaml:"sharing_config" datastore:"sharing_config"` - Generated bool `json:"generated" yaml:"generated" required:false datastore:"generated"` - Downloaded bool `json:"downloaded" yaml:"downloaded" required:false datastore:"downloaded"` - Sharing bool `json:"sharing" yaml:"sharing" required:false datastore:"sharing"` - Verified bool `json:"verified" yaml:"verified" required:false datastore:"verified"` - Invalid bool `json:"invalid" yaml:"invalid" required:false datastore:"invalid"` - Activated bool `json:"activated" yaml:"activated" required:false datastore:"activated"` - Tested bool `json:"tested" yaml:"tested" required:false datastore:"tested"` - Owner string `json:"owner" datastore:"owner" yaml:"owner"` - Hash string `json:"hash" datastore:"hash" yaml:"hash"` // api.yaml+dockerfile+src/app.py for apps - PrivateID string `json:"private_id" yaml:"private_id" required:false datastore:"private_id"` - Description string `json:"description" datastore:"description,noindex" required:false yaml:"description"` - Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"` - SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"` - LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false` - ContactInfo struct { - Name string `json:"name" datastore:"name" yaml:"name"` - Url string `json:"url" datastore:"url" yaml:"url"` - } `json:"contact_info" datastore:"contact_info" yaml:"contact_info" required:false` - ReferenceInfo struct { - DocumentationUrl string `json:"documentation_url" datastore:"documentation_url"` - GithubUrl string `json:"github_url" datastore:"github_url"` - } - FolderMount struct { - FolderMount bool `json:"folder_mount" datastore:"folder_mount"` - SourceFolder string `json:"source_folder" datastore:"source_folder"` - DestinationFolder string `json:"destination_folder" datastore:"destination_folder"` - } - Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions,noindex"` - Authentication Authentication `json:"authentication" yaml:"authentication" required:false datastore:"authentication"` - Tags []string `json:"tags" yaml:"tags" required:false datastore:"activated"` - Categories []string `json:"categories" yaml:"categories" required:false datastore:"categories"` - Created int64 `json:"created" datastore:"created"` - Edited int64 `json:"edited" datastore:"edited"` - LastRuntime int64 `json:"last_runtime" datastore:"last_runtime"` -} - -type WorkflowAppActionParameter struct { - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - ID string `json:"id" datastore:"id" yaml:"id,omitempty"` - Name string `json:"name" datastore:"name" yaml:"name"` - Example string `json:"example" datastore:"example,noindex" yaml:"example"` - Value string `json:"value" datastore:"value,noindex" yaml:"value,omitempty"` - Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` - Options []string `json:"options" datastore:"options" yaml:"options"` - ActionField string `json:"action_field" datastore:"action_field" yaml:"actionfield,omitempty"` - Variant string `json:"variant" datastore:"variant" yaml:"variant,omitempty"` - Required bool `json:"required" datastore:"required" yaml:"required"` - Configuration bool `json:"configuration" datastore:"configuration" yaml:"configuration"` - Tags []string `json:"tags" datastore:"tags" yaml:"tags"` - Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` - SkipMulticheck bool `json:"skip_multicheck" datastore:"skip_multicheck" yaml:"skip_multicheck"` - ValueReplace []Valuereplace `json:"value_replace" datastore:"value_replace,noindex" yaml:"value_replace,omitempty"` - UniqueToggled bool `json:"unique_toggled" datastore:"unique_toggled" yaml:"unique_toggled"` -} - -type Valuereplace struct { - Key string `json:"key" datastore:"key" yaml:"key"` - Value string `json:"value" datastore:"value" yaml:"value"` -} - -type SchemaDefinition struct { - Type string `json:"type" datastore:"type"` -} - -type WorkflowAppAction struct { - Description string `json:"description" datastore:"description,noindex"` - ID string `json:"id" datastore:"id" yaml:"id,omitempty"` - Name string `json:"name" datastore:"name"` - Label string `json:"label" datastore:"label"` - NodeType string `json:"node_type" datastore:"node_type"` - Environment string `json:"environment" datastore:"environment"` - Sharing bool `json:"sharing" datastore:"sharing"` - PrivateID string `json:"private_id" datastore:"private_id"` - AppID string `json:"app_id" datastore:"app_id"` - Tags []string `json:"tags" datastore:"tags" yaml:"tags"` - Authentication []AuthenticationStore `json:"authentication" datastore:"authentication,noindex" yaml:"authentication,omitempty"` - Tested bool `json:"tested" datastore:"tested" yaml:"tested"` - Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters"` - ExecutionVariable struct { - Description string `json:"description" datastore:"description,noindex"` - ID string `json:"id" datastore:"id"` - Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value,noindex"` - } `json:"execution_variable" datastore:"execution_variables"` - Returns struct { - Description string `json:"description" datastore:"returns" yaml:"description,omitempty"` - Example string `json:"example" datastore:"example,noindex" yaml:"example"` - ID string `json:"id" datastore:"id" yaml:"id,omitempty"` - Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` - } `json:"returns" datastore:"returns"` - AuthenticationId string `json:"authentication_id" datastore:"authentication_id"` - Example string `json:"example,noindex" datastore:"example" yaml:"example"` - AuthNotRequired bool `json:"auth_not_required" datastore:"auth_not_required" yaml:"auth_not_required"` -} +//type ExecutionRequest struct { +// ExecutionId string `json:"execution_id,omitempty"` +// ExecutionArgument string `json:"execution_argument,omitempty"` +// ExecutionSource string `json:"execution_source,omitempty"` +// WorkflowId string `json:"workflow_id,omitempty"` +// Environments []string `json:"environments,omitempty"` +// Authorization string `json:"authorization,omitempty"` +// Status string `json:"status,omitempty"` +// Start string `json:"start,omitempty"` +// Type string `json:"type,omitempty"` +//} +// +//type SyncFeatures struct { +// Webhook SyncData `json:"webhook" datastore:"webhook"` +// Schedules SyncData `json:"schedules" datastore:"schedules"` +// UserInput SyncData `json:"user_input" datastore:"user_input"` +// SendMail SyncData `json:"send_mail" datastore:"send_mail"` +// SendSms SyncData `json:"send_sms" datastore:"send_sms"` +// Updates SyncData `json:"updates" datastore:"updates"` +// Notifications SyncData `json:"notifications" datastore:"notifications"` +// EmailTrigger SyncData `json:"email_trigger" datastore:"email_trigger"` +// AppExecutions SyncData `json:"app_executions" datastore:"app_executions"` +// WorkflowExecutions SyncData `json:"workflow_executions" datastore:"workflow_executions"` +// Apps SyncData `json:"apps" datastore:"apps"` +// Workflows SyncData `json:"workflows" datastore:"workflows"` +// Autocomplete SyncData `json:"autocomplete" datastore:"autocomplete"` +// Authentication SyncData `json:"authentication" datastore:"authentication"` +// Schedule SyncData `json:"schedule" datastore:"schedule"` +//} +// +//type SyncData struct { +// Active bool `json:"active" datastore:"active"` +// Type string `json:"type,omitempty" datastore:"type"` +// Name string `json:"name,omitempty" datastore:"name"` +// Description string `json:"description,omitempty" datastore:"description"` +// Limit int64 `json:"limit,omitempty" datastore:"limit"` +// StartDate int64 `json:"start_date,omitempty" datastore:"start_date"` +// EndDate int64 `json:"end_date,omitempty" datastore:"end_date"` +// DataCollection int64 `json:"data_collection,omitempty" datastore:"data_collection"` +//} +// +//type SyncConfig struct { +// Interval int64 `json:"interval" datastore:"interval"` +// Apikey string `json:"api_key" datastore:"api_key"` +//} +// +//type PaymentSubscription struct { +// Active bool `json:"active" datastore:"active"` +// Startdate int64 `json:"startdate" datastore:"startdate"` +// CancellationDate int64 `json:"cancellationdate" datastore:"cancellationdate"` +// Enddate int64 `json:"enddate" datastore:"enddate"` +// Name string `json:"name" datastore:"name"` +// Recurrence string `json:"recurrence" datastore:"recurrence"` +// Reference string `json:"reference" datastore:"reference"` +// Level string `json:"level" datastore:"level"` +// Amount string `json:"amount" datastore:"amount"` +// Currency string `json:"currency" datastore:"currency"` +//} +// +//type Defaults struct { +// AppDownloadRepo string `json:"app_download_repo" datastore:"app_download_repo"` +// AppDownloadBranch string `json:"app_download_branch" datastore:"app_download_branch"` +// WorkflowDownloadRepo string `json:"workflow_download_repo" datastore:"workflow_download_repo"` +// WorkflowDownloadBranch string `json:"workflow_download_branch" datastore:"workflow_download_branch"` +//} +// +//type AppAuthenticationStorage struct { +// Active bool `json:"active" datastore:"active"` +// Label string `json:"label" datastore:"label"` +// Id string `json:"id" datastore:"id"` +// App WorkflowApp `json:"app" datastore:"app,noindex"` +// Fields []AuthenticationStore `json:"fields" datastore:"fields"` +// Usage []AuthenticationUsage `json:"usage" datastore:"usage"` +// WorkflowCount int64 `json:"workflow_count" datastore:"workflow_count"` +// NodeCount int64 `json:"node_count" datastore:"node_count"` +// OrgId string `json:"org_id" datastore:"org_id"` +// Created int64 `json:"created" datastore:"created"` +// Edited int64 `json:"edited" datastore:"edited"` +// Defined bool `json:"defined" datastore:"defined"` +//} +// +//type AuthenticationUsage struct { +// WorkflowId string `json:"workflow_id" datastore:"workflow_id"` +// Nodes []string `json:"nodes" datastore:"nodes"` +//} +// +//// An app inside Shuffle +//// Source string `json:"source" datastore:"soure" yaml:"source"` - downloadlocation +//type WorkflowApp struct { +// Name string `json:"name" yaml:"name" required:true datastore:"name"` +// IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"` +// ID string `json:"id" yaml:"id,omitempty" required:false datastore:"id"` +// Link string `json:"link" yaml:"link" required:false datastore:"link,noindex"` +// AppVersion string `json:"app_version" yaml:"app_version" required:true datastore:"app_version"` +// SharingConfig string `json:"sharing_config" yaml:"sharing_config" datastore:"sharing_config"` +// Generated bool `json:"generated" yaml:"generated" required:false datastore:"generated"` +// Downloaded bool `json:"downloaded" yaml:"downloaded" required:false datastore:"downloaded"` +// Sharing bool `json:"sharing" yaml:"sharing" required:false datastore:"sharing"` +// Verified bool `json:"verified" yaml:"verified" required:false datastore:"verified"` +// Invalid bool `json:"invalid" yaml:"invalid" required:false datastore:"invalid"` +// Activated bool `json:"activated" yaml:"activated" required:false datastore:"activated"` +// Tested bool `json:"tested" yaml:"tested" required:false datastore:"tested"` +// Owner string `json:"owner" datastore:"owner" yaml:"owner"` +// Hash string `json:"hash" datastore:"hash" yaml:"hash"` // api.yaml+dockerfile+src/app.py for apps +// PrivateID string `json:"private_id" yaml:"private_id" required:false datastore:"private_id"` +// Description string `json:"description" datastore:"description,noindex" required:false yaml:"description"` +// Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"` +// SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"` +// LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false` +// ContactInfo struct { +// Name string `json:"name" datastore:"name" yaml:"name"` +// Url string `json:"url" datastore:"url" yaml:"url"` +// } `json:"contact_info" datastore:"contact_info" yaml:"contact_info" required:false` +// ReferenceInfo struct { +// DocumentationUrl string `json:"documentation_url" datastore:"documentation_url"` +// GithubUrl string `json:"github_url" datastore:"github_url"` +// } +// FolderMount struct { +// FolderMount bool `json:"folder_mount" datastore:"folder_mount"` +// SourceFolder string `json:"source_folder" datastore:"source_folder"` +// DestinationFolder string `json:"destination_folder" datastore:"destination_folder"` +// } +// Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions,noindex"` +// Authentication Authentication `json:"authentication" yaml:"authentication" required:false datastore:"authentication"` +// Tags []string `json:"tags" yaml:"tags" required:false datastore:"activated"` +// Categories []string `json:"categories" yaml:"categories" required:false datastore:"categories"` +// Created int64 `json:"created" datastore:"created"` +// Edited int64 `json:"edited" datastore:"edited"` +// LastRuntime int64 `json:"last_runtime" datastore:"last_runtime"` +//} +// +//type WorkflowAppActionParameter struct { +// Description string `json:"description" datastore:"description,noindex" yaml:"description"` +// ID string `json:"id" datastore:"id" yaml:"id,omitempty"` +// Name string `json:"name" datastore:"name" yaml:"name"` +// Example string `json:"example" datastore:"example,noindex" yaml:"example"` +// Value string `json:"value" datastore:"value,noindex" yaml:"value,omitempty"` +// Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` +// Options []string `json:"options" datastore:"options" yaml:"options"` +// ActionField string `json:"action_field" datastore:"action_field" yaml:"actionfield,omitempty"` +// Variant string `json:"variant" datastore:"variant" yaml:"variant,omitempty"` +// Required bool `json:"required" datastore:"required" yaml:"required"` +// Configuration bool `json:"configuration" datastore:"configuration" yaml:"configuration"` +// Tags []string `json:"tags" datastore:"tags" yaml:"tags"` +// Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` +// SkipMulticheck bool `json:"skip_multicheck" datastore:"skip_multicheck" yaml:"skip_multicheck"` +// ValueReplace []Valuereplace `json:"value_replace" datastore:"value_replace,noindex" yaml:"value_replace,omitempty"` +// UniqueToggled bool `json:"unique_toggled" datastore:"unique_toggled" yaml:"unique_toggled"` +//} +// +//type Valuereplace struct { +// Key string `json:"key" datastore:"key" yaml:"key"` +// Value string `json:"value" datastore:"value" yaml:"value"` +//} +// +//type SchemaDefinition struct { +// Type string `json:"type" datastore:"type"` +//} +// +//type WorkflowAppAction struct { +// Description string `json:"description" datastore:"description,noindex"` +// ID string `json:"id" datastore:"id" yaml:"id,omitempty"` +// Name string `json:"name" datastore:"name"` +// Label string `json:"label" datastore:"label"` +// NodeType string `json:"node_type" datastore:"node_type"` +// Environment string `json:"environment" datastore:"environment"` +// Sharing bool `json:"sharing" datastore:"sharing"` +// PrivateID string `json:"private_id" datastore:"private_id"` +// AppID string `json:"app_id" datastore:"app_id"` +// Tags []string `json:"tags" datastore:"tags" yaml:"tags"` +// Authentication []AuthenticationStore `json:"authentication" datastore:"authentication,noindex" yaml:"authentication,omitempty"` +// Tested bool `json:"tested" datastore:"tested" yaml:"tested"` +// Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters"` +// ExecutionVariable struct { +// Description string `json:"description" datastore:"description,noindex"` +// ID string `json:"id" datastore:"id"` +// Name string `json:"name" datastore:"name"` +// Value string `json:"value" datastore:"value,noindex"` +// } `json:"execution_variable" datastore:"execution_variables"` +// Returns struct { +// Description string `json:"description" datastore:"returns" yaml:"description,omitempty"` +// Example string `json:"example" datastore:"example,noindex" yaml:"example"` +// ID string `json:"id" datastore:"id" yaml:"id,omitempty"` +// Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` +// } `json:"returns" datastore:"returns"` +// AuthenticationId string `json:"authentication_id" datastore:"authentication_id"` +// Example string `json:"example,noindex" datastore:"example" yaml:"example"` +// AuthNotRequired bool `json:"auth_not_required" datastore:"auth_not_required" yaml:"auth_not_required"` +//} // FIXME: Generate a callback authentication ID? // FIXME: Add org check .. -type WorkflowExecution struct { - Type string `json:"type" datastore:"type"` - Status string `json:"status" datastore:"status"` - Start string `json:"start" datastore:"start"` - ExecutionArgument string `json:"execution_argument" datastore:"execution_argument,noindex"` - ExecutionId string `json:"execution_id" datastore:"execution_id"` - ExecutionSource string `json:"execution_source" datastore:"execution_source"` - ExecutionParent string `json:"execution_parent" datastore:"execution_parent"` - ExecutionOrg string `json:"execution_org" datastore:"execution_org"` - WorkflowId string `json:"workflow_id" datastore:"workflow_id"` - LastNode string `json:"last_node" datastore:"last_node"` - Authorization string `json:"authorization" datastore:"authorization"` - Result string `json:"result" datastore:"result,noindex"` - StartedAt int64 `json:"started_at" datastore:"started_at"` - CompletedAt int64 `json:"completed_at" datastore:"completed_at"` - ProjectId string `json:"project_id" datastore:"project_id"` - Locations []string `json:"locations" datastore:"locations"` - Workflow Workflow `json:"workflow" datastore:"workflow,noindex"` - Results []ActionResult `json:"results" datastore:"results,noindex"` - ExecutionVariables []struct { - Description string `json:"description" datastore:"description,noindex"` - ID string `json:"id" datastore:"id"` - Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value,noindex"` - } `json:"execution_variables,omitempty" datastore:"execution_variables,omitempty"` - OrgId string `json:"org_id" datastore:"org_id"` -} +//type WorkflowExecution struct { +// Type string `json:"type" datastore:"type"` +// Status string `json:"status" datastore:"status"` +// Start string `json:"start" datastore:"start"` +// ExecutionArgument string `json:"execution_argument" datastore:"execution_argument,noindex"` +// ExecutionId string `json:"execution_id" datastore:"execution_id"` +// ExecutionSource string `json:"execution_source" datastore:"execution_source"` +// ExecutionParent string `json:"execution_parent" datastore:"execution_parent"` +// ExecutionOrg string `json:"execution_org" datastore:"execution_org"` +// WorkflowId string `json:"workflow_id" datastore:"workflow_id"` +// LastNode string `json:"last_node" datastore:"last_node"` +// Authorization string `json:"authorization" datastore:"authorization"` +// Result string `json:"result" datastore:"result,noindex"` +// StartedAt int64 `json:"started_at" datastore:"started_at"` +// CompletedAt int64 `json:"completed_at" datastore:"completed_at"` +// ProjectId string `json:"project_id" datastore:"project_id"` +// Locations []string `json:"locations" datastore:"locations"` +// Workflow Workflow `json:"workflow" datastore:"workflow,noindex"` +// Results []ActionResult `json:"results" datastore:"results,noindex"` +// ExecutionVariables []struct { +// Description string `json:"description" datastore:"description,noindex"` +// ID string `json:"id" datastore:"id"` +// Name string `json:"name" datastore:"name"` +// Value string `json:"value" datastore:"value,noindex"` +// } `json:"execution_variables,omitempty" datastore:"execution_variables,omitempty"` +// OrgId string `json:"org_id" datastore:"org_id"` +//} // This is for the nodes in a workflow, NOT the app action itself. -type Action struct { - AppName string `json:"app_name" datastore:"app_name"` - AppVersion string `json:"app_version" datastore:"app_version"` - AppID string `json:"app_id" datastore:"app_id"` - Errors []string `json:"errors" datastore:"errors"` - ID string `json:"id" datastore:"id"` - IsValid bool `json:"is_valid" datastore:"is_valid"` - IsStartNode bool `json:"isStartNode,omitempty" datastore:"isStartNode"` - Sharing bool `json:"sharing,omitempty" datastore:"sharing"` - PrivateID string `json:"private_id,omitempty" datastore:"private_id"` - Label string `json:"label,omitempty" datastore:"label"` - SmallImage string `json:"small_image,omitempty" datastore:"small_image,noindex" required:false yaml:"small_image"` - LargeImage string `json:"large_image,omitempty" datastore:"large_image,noindex" yaml:"large_image" required:false` - Environment string `json:"environment,omitempty" datastore:"environment"` - Name string `json:"name" datastore:"name"` - Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"` - ExecutionVariable struct { - Description string `json:"description,omitempty" datastore:"description,noindex"` - ID string `json:"id,omitempty" datastore:"id"` - Name string `json:"name,omitempty" datastore:"name"` - Value string `json:"value,omitempty" datastore:"value,noindex"` - } `json:"execution_variable,omitempty" datastore:"execution_variable,omitempty"` - Position struct { - X float64 `json:"x,omitempty" datastore:"x"` - Y float64 `json:"y,omitempty" datastore:"y"` - } `json:"position,omitempty"` - Priority int `json:"priority,omitempty" datastore:"priority"` - AuthenticationId string `json:"authentication_id" datastore:"authentication_id"` - Example string `json:"example,omitempty" datastore:"example,noindex"` - AuthNotRequired bool `json:"auth_not_required,omitempty" datastore:"auth_not_required" yaml:"auth_not_required"` - Category string `json:"category" datastore:"category"` -} +//type Action struct { +// AppName string `json:"app_name" datastore:"app_name"` +// AppVersion string `json:"app_version" datastore:"app_version"` +// AppID string `json:"app_id" datastore:"app_id"` +// Errors []string `json:"errors" datastore:"errors"` +// ID string `json:"id" datastore:"id"` +// IsValid bool `json:"is_valid" datastore:"is_valid"` +// IsStartNode bool `json:"isStartNode,omitempty" datastore:"isStartNode"` +// Sharing bool `json:"sharing,omitempty" datastore:"sharing"` +// PrivateID string `json:"private_id,omitempty" datastore:"private_id"` +// Label string `json:"label,omitempty" datastore:"label"` +// SmallImage string `json:"small_image,omitempty" datastore:"small_image,noindex" required:false yaml:"small_image"` +// LargeImage string `json:"large_image,omitempty" datastore:"large_image,noindex" yaml:"large_image" required:false` +// Environment string `json:"environment,omitempty" datastore:"environment"` +// Name string `json:"name" datastore:"name"` +// Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"` +// ExecutionVariable struct { +// Description string `json:"description,omitempty" datastore:"description,noindex"` +// ID string `json:"id,omitempty" datastore:"id"` +// Name string `json:"name,omitempty" datastore:"name"` +// Value string `json:"value,omitempty" datastore:"value,noindex"` +// } `json:"execution_variable,omitempty" datastore:"execution_variable,omitempty"` +// Position struct { +// X float64 `json:"x,omitempty" datastore:"x"` +// Y float64 `json:"y,omitempty" datastore:"y"` +// } `json:"position,omitempty"` +// Priority int `json:"priority,omitempty" datastore:"priority"` +// AuthenticationId string `json:"authentication_id" datastore:"authentication_id"` +// Example string `json:"example,omitempty" datastore:"example,noindex"` +// AuthNotRequired bool `json:"auth_not_required,omitempty" datastore:"auth_not_required" yaml:"auth_not_required"` +// Category string `json:"category" datastore:"category"` +//} +// +//// Added environment for location to execute +//type Trigger struct { +// AppName string `json:"app_name" datastore:"app_name"` +// Description string `json:"description" datastore:"description,noindex"` +// LongDescription string `json:"long_description" datastore:"long_description"` +// Status string `json:"status" datastore:"status"` +// AppVersion string `json:"app_version" datastore:"app_version"` +// Errors []string `json:"errors" datastore:"errors"` +// ID string `json:"id" datastore:"id"` +// IsValid bool `json:"is_valid" datastore:"is_valid"` +// IsStartNode bool `json:"isStartNode" datastore:"isStartNode"` +// Label string `json:"label" datastore:"label"` +// SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"` +// LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false` +// Environment string `json:"environment" datastore:"environment"` +// TriggerType string `json:"trigger_type" datastore:"trigger_type"` +// Name string `json:"name" datastore:"name"` +// Tags []string `json:"tags" datastore:"tags" yaml:"tags"` +// Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"` +// Position struct { +// X float64 `json:"x" datastore:"x"` +// Y float64 `json:"y" datastore:"y"` +// } `json:"position"` +// Priority int `json:"priority" datastore:"priority"` +//} +// +//type Branch struct { +// DestinationID string `json:"destination_id" datastore:"destination_id"` +// ID string `json:"id" datastore:"id"` +// SourceID string `json:"source_id" datastore:"source_id"` +// Label string `json:"label" datastore:"label"` +// HasError bool `json:"has_errors" datastore: "has_errors"` +// Conditions []Condition `json:"conditions" datastore: "conditions,noindex"` +//} +// +//// Same format for a lot of stuff +//type Condition struct { +// Condition WorkflowAppActionParameter `json:"condition" datastore:"condition"` +// Source WorkflowAppActionParameter `json:"source" datastore:"source"` +// Destination WorkflowAppActionParameter `json:"destination" datastore:"destination"` +//} +// +//type Schedule struct { +// Name string `json:"name" datastore:"name"` +// Frequency string `json:"frequency" datastore:"frequency"` +// ExecutionArgument string `json:"execution_argument" datastore:"execution_argument,noindex"` +// Id string `json:"id" datastore:"id"` +// OrgId string `json:"org_id" datastore:"org_id"` +// Environment string `json:"environment" datastore:"environment"` +//} -// Added environment for location to execute -type Trigger struct { - AppName string `json:"app_name" datastore:"app_name"` - Description string `json:"description" datastore:"description,noindex"` - LongDescription string `json:"long_description" datastore:"long_description"` - Status string `json:"status" datastore:"status"` - AppVersion string `json:"app_version" datastore:"app_version"` - Errors []string `json:"errors" datastore:"errors"` - ID string `json:"id" datastore:"id"` - IsValid bool `json:"is_valid" datastore:"is_valid"` - IsStartNode bool `json:"isStartNode" datastore:"isStartNode"` - Label string `json:"label" datastore:"label"` - SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"` - LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false` - Environment string `json:"environment" datastore:"environment"` - TriggerType string `json:"trigger_type" datastore:"trigger_type"` - Name string `json:"name" datastore:"name"` - Tags []string `json:"tags" datastore:"tags" yaml:"tags"` - Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"` - Position struct { - X float64 `json:"x" datastore:"x"` - Y float64 `json:"y" datastore:"y"` - } `json:"position"` - Priority int `json:"priority" datastore:"priority"` -} +//type Workflow struct { +// Actions []Action `json:"actions" datastore:"actions,noindex"` +// Branches []Branch `json:"branches" datastore:"branches,noindex"` +// Triggers []Trigger `json:"triggers" datastore:"triggers,noindex"` +// Schedules []Schedule `json:"schedules" datastore:"schedules,noindex"` +// Configuration struct { +// ExitOnError bool `json:"exit_on_error" datastore:"exit_on_error"` +// StartFromTop bool `json:"start_from_top" datastore:"start_from_top"` +// } `json:"configuration,omitempty" datastore:"configuration"` +// Created int64 `json:"created" datastore:"created"` +// Edited int64 `json:"edited" datastore:"edited"` +// LastRuntime int64 `json:"last_runtime" datastore:"last_runtime"` +// Errors []string `json:"errors,omitempty" datastore:"errors"` +// Tags []string `json:"tags,omitempty" datastore:"tags"` +// ID string `json:"id" datastore:"id"` +// IsValid bool `json:"is_valid" datastore:"is_valid"` +// Name string `json:"name" datastore:"name"` +// Description string `json:"description" datastore:"description,noindex"` +// Start string `json:"start" datastore:"start"` +// Owner string `json:"owner" datastore:"owner"` +// Sharing string `json:"sharing" datastore:"sharing"` +// Org []Org `json:"org,omitempty" datastore:"org"` +// ExecutingOrg Org `json:"execution_org,omitempty" datastore:"execution_org"` +// OrgId string `json:"org_id,omitempty" datastore:"org_id"` +// WorkflowVariables []struct { +// Description string `json:"description" datastore:"description,noindex"` +// ID string `json:"id" datastore:"id"` +// Name string `json:"name" datastore:"name"` +// Value string `json:"value" datastore:"value,noindex"` +// } `json:"workflow_variables" datastore:"workflow_variables"` +// ExecutionVariables []struct { +// Description string `json:"description" datastore:"description,noindex"` +// ID string `json:"id" datastore:"id"` +// Name string `json:"name" datastore:"name"` +// Value string `json:"value" datastore:"value,noindex"` +// } `json:"execution_variables,omitempty" datastore:"execution_variables"` +// ExecutionEnvironment string `json:"execution_environment" datastore:"execution_environment"` +// PreviouslySaved bool `json:"previously_saved" datastore:"first_save"` +// Categories Categories `json:"categories" datastore:"categories"` +// ExampleArgument string `json:"example_argument" datastore:"example_argument,noindex"` +//} -type Branch struct { - DestinationID string `json:"destination_id" datastore:"destination_id"` - ID string `json:"id" datastore:"id"` - SourceID string `json:"source_id" datastore:"source_id"` - Label string `json:"label" datastore:"label"` - HasError bool `json:"has_errors" datastore: "has_errors"` - Conditions []Condition `json:"conditions" datastore: "conditions,noindex"` -} +//type Category struct { +// Name string `json:"name" datastore:"name"` +// Description string `json:"description" datastore:"description"` +// Count int64 `json:"count" datastore:"count"` +//} +// +//type Categories struct { +// SIEM Category `json:"siem" datastore:"siem"` +// Communication Category `json:"communication" datastore:"communication"` +// Assets Category `json:"assets" datastore:"assets"` +// Cases Category `json:"cases" datastore:"cases"` +// Network Category `json:"network" datastore:"network"` +// Intel Category `json:"intel" datastore:"intel"` +// EDR Category `json:"edr" datastore:"edr"` +// Other Category `json:"other" datastore:"other"` +//} -// Same format for a lot of stuff -type Condition struct { - Condition WorkflowAppActionParameter `json:"condition" datastore:"condition"` - Source WorkflowAppActionParameter `json:"source" datastore:"source"` - Destination WorkflowAppActionParameter `json:"destination" datastore:"destination"` -} - -type Schedule struct { - Name string `json:"name" datastore:"name"` - Frequency string `json:"frequency" datastore:"frequency"` - ExecutionArgument string `json:"execution_argument" datastore:"execution_argument,noindex"` - Id string `json:"id" datastore:"id"` - OrgId string `json:"org_id" datastore:"org_id"` - Environment string `json:"environment" datastore:"environment"` -} - -type Workflow struct { - Actions []Action `json:"actions" datastore:"actions,noindex"` - Branches []Branch `json:"branches" datastore:"branches,noindex"` - Triggers []Trigger `json:"triggers" datastore:"triggers,noindex"` - Schedules []Schedule `json:"schedules" datastore:"schedules,noindex"` - Configuration struct { - ExitOnError bool `json:"exit_on_error" datastore:"exit_on_error"` - StartFromTop bool `json:"start_from_top" datastore:"start_from_top"` - } `json:"configuration,omitempty" datastore:"configuration"` - Created int64 `json:"created" datastore:"created"` - Edited int64 `json:"edited" datastore:"edited"` - LastRuntime int64 `json:"last_runtime" datastore:"last_runtime"` - Errors []string `json:"errors,omitempty" datastore:"errors"` - Tags []string `json:"tags,omitempty" datastore:"tags"` - ID string `json:"id" datastore:"id"` - IsValid bool `json:"is_valid" datastore:"is_valid"` - Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description,noindex"` - Start string `json:"start" datastore:"start"` - Owner string `json:"owner" datastore:"owner"` - Sharing string `json:"sharing" datastore:"sharing"` - Org []Org `json:"org,omitempty" datastore:"org"` - ExecutingOrg Org `json:"execution_org,omitempty" datastore:"execution_org"` - OrgId string `json:"org_id,omitempty" datastore:"org_id"` - WorkflowVariables []struct { - Description string `json:"description" datastore:"description,noindex"` - ID string `json:"id" datastore:"id"` - Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value,noindex"` - } `json:"workflow_variables" datastore:"workflow_variables"` - ExecutionVariables []struct { - Description string `json:"description" datastore:"description,noindex"` - ID string `json:"id" datastore:"id"` - Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value,noindex"` - } `json:"execution_variables,omitempty" datastore:"execution_variables"` - ExecutionEnvironment string `json:"execution_environment" datastore:"execution_environment"` - PreviouslySaved bool `json:"previously_saved" datastore:"first_save"` - Categories Categories `json:"categories" datastore:"categories"` - ExampleArgument string `json:"example_argument" datastore:"example_argument,noindex"` -} - -type Category struct { - Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description"` - Count int64 `json:"count" datastore:"count"` -} - -type Categories struct { - SIEM Category `json:"siem" datastore:"siem"` - Communication Category `json:"communication" datastore:"communication"` - Assets Category `json:"assets" datastore:"assets"` - Cases Category `json:"cases" datastore:"cases"` - Network Category `json:"network" datastore:"network"` - Intel Category `json:"intel" datastore:"intel"` - EDR Category `json:"edr" datastore:"edr"` - Other Category `json:"other" datastore:"other"` -} - -type ActionResult struct { - Action Action `json:"action" datastore:"action,noindex"` - ExecutionId string `json:"execution_id" datastore:"execution_id"` - Authorization string `json:"authorization" datastore:"authorization"` - Result string `json:"result" datastore:"result,noindex"` - StartedAt int64 `json:"started_at" datastore:"started_at"` - CompletedAt int64 `json:"completed_at" datastore:"completed_at"` - Status string `json:"status" datastore:"status"` -} - -type Authentication struct { - Required bool `json:"required" datastore:"required" yaml:"required" ` - Parameters []AuthenticationParams `json:"parameters" datastore:"parameters" yaml:"parameters"` -} - -type AuthenticationParams struct { - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - ID string `json:"id" datastore:"id" yaml:"id"` - Name string `json:"name" datastore:"name" yaml:"name"` - Example string `json:"example" datastore:"example,noindex" yaml:"example"` - Value string `json:"value,omitempty" datastore:"value,noindex" yaml:"value"` - Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` - Required bool `json:"required" datastore:"required" yaml:"required"` - In string `json:"in" datastore:"in" yaml:"in"` - Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` - Scheme string `json:"scheme" datastore:"scheme" yaml:"scheme"` // Deprecated -} - -type AuthenticationStore struct { - Key string `json:"key" datastore:"key"` - Value string `json:"value" datastore:"value,noindex"` -} - -type ExecutionRequestWrapper struct { - Data []ExecutionRequest `json:"data"` -} - -type AppExecutionExample struct { - AppName string `json:"app_name" datastore:"app_name"` - AppVersion string `json:"app_version" datastore:"app_version"` - AppAction string `json:"app_action" datastore:"app_action"` - AppId string `json:"app_id" datastore:"app_id"` - ExampleId string `json:"example_id" datastore:"example_id"` - SuccessExamples []string `json:"success_examples" datastore:"success_examples,noindex"` - FailureExamples []string `json:"failure_examples" datastore:"failure_examples,noindex"` -} +//type ActionResult struct { +// Action Action `json:"action" datastore:"action,noindex"` +// ExecutionId string `json:"execution_id" datastore:"execution_id"` +// Authorization string `json:"authorization" datastore:"authorization"` +// Result string `json:"result" datastore:"result,noindex"` +// StartedAt int64 `json:"started_at" datastore:"started_at"` +// CompletedAt int64 `json:"completed_at" datastore:"completed_at"` +// Status string `json:"status" datastore:"status"` +//} +// +//type Authentication struct { +// Required bool `json:"required" datastore:"required" yaml:"required" ` +// Parameters []AuthenticationParams `json:"parameters" datastore:"parameters" yaml:"parameters"` +//} +// +//type AuthenticationParams struct { +// Description string `json:"description" datastore:"description,noindex" yaml:"description"` +// ID string `json:"id" datastore:"id" yaml:"id"` +// Name string `json:"name" datastore:"name" yaml:"name"` +// Example string `json:"example" datastore:"example,noindex" yaml:"example"` +// Value string `json:"value,omitempty" datastore:"value,noindex" yaml:"value"` +// Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` +// Required bool `json:"required" datastore:"required" yaml:"required"` +// In string `json:"in" datastore:"in" yaml:"in"` +// Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` +// Scheme string `json:"scheme" datastore:"scheme" yaml:"scheme"` // Deprecated +//} +// +//type AuthenticationStore struct { +// Key string `json:"key" datastore:"key"` +// Value string `json:"value" datastore:"value,noindex"` +//} +// +//type ExecutionRequestWrapper struct { +// Data []ExecutionRequest `json:"data"` +//} +// +//type AppExecutionExample struct { +// AppName string `json:"app_name" datastore:"app_name"` +// AppVersion string `json:"app_version" datastore:"app_version"` +// AppAction string `json:"app_action" datastore:"app_action"` +// AppId string `json:"app_id" datastore:"app_id"` +// ExampleId string `json:"example_id" datastore:"example_id"` +// SuccessExamples []string `json:"success_examples" datastore:"success_examples,noindex"` +// FailureExamples []string `json:"failure_examples" datastore:"failure_examples,noindex"` +//} // This might be... a bit off, but that's fine :) // This might also be stupid, as we want timelines and such @@ -551,7 +534,7 @@ func increaseStatisticsField(ctx context.Context, fieldname, id string, amount i return nil } -func setWorkflowQueue(ctx context.Context, executionRequest ExecutionRequest, env string) error { +func setWorkflowQueue(ctx context.Context, executionRequest shuffle.ExecutionRequest, env string) error { orgKey := fmt.Sprintf("workflowqueue-%s", env) key := datastore.NameKey(orgKey, executionRequest.ExecutionId, nil) @@ -577,16 +560,16 @@ func setWorkflowQueue(ctx context.Context, executionRequest ExecutionRequest, en // return nil //} -func getWorkflowQueue(ctx context.Context, id string) (ExecutionRequestWrapper, error) { +func getWorkflowQueue(ctx context.Context, id string) (shuffle.ExecutionRequestWrapper, error) { orgId := fmt.Sprintf("workflowqueue-%s", id) q := datastore.NewQuery(orgId).Limit(10) - executions := []ExecutionRequest{} + executions := []shuffle.ExecutionRequest{} _, err := dbclient.GetAll(ctx, q, &executions) if err != nil { - return ExecutionRequestWrapper{}, err + return shuffle.ExecutionRequestWrapper{}, err } - return ExecutionRequestWrapper{Data: executions}, nil + return shuffle.ExecutionRequestWrapper{Data: executions}, nil //key := datastore.NameKey("workflowqueue", id, nil) //executions := ExecutionRequestWrapper{} @@ -660,7 +643,7 @@ func createSchedule(ctx context.Context, scheduleId, workflowId, name, startNode Body: ioutil.NopCloser(strings.NewReader(bodyWrapper)), } - _, _, err := handleExecution(workflowId, Workflow{ExecutingOrg: Org{Id: orgId}}, request) + _, _, err := handleExecution(workflowId, shuffle.Workflow{ExecutingOrg: shuffle.Org{Id: orgId}}, request) if err != nil { log.Printf("Failed to execute %s: %s", workflowId, err) } @@ -747,7 +730,7 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque // Getting from the request //log.Println(string(body)) - var removeExecutionRequests ExecutionRequestWrapper + var removeExecutionRequests shuffle.ExecutionRequestWrapper err = json.Unmarshal(body, &removeExecutionRequests) if err != nil { log.Printf("Failed executionrequest in queue unmarshaling: %s", err) @@ -831,7 +814,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { } if len(executionRequests.Data) == 0 { - executionRequests.Data = []ExecutionRequest{} + executionRequests.Data = []shuffle.ExecutionRequest{} } else { log.Printf("[INFO] Executionrequests (%s): %d", id, len(executionRequests.Data)) } @@ -861,7 +844,7 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { return } - var actionResult ActionResult + var actionResult shuffle.ActionResult err = json.Unmarshal(body, &actionResult) if err != nil { log.Printf("Failed ActionResult unmarshaling: %s", err) @@ -871,7 +854,7 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - workflowExecution, err := getWorkflowExecution(ctx, actionResult.ExecutionId) + workflowExecution, err := shuffle.GetWorkflowExecution(ctx, actionResult.ExecutionId) if err != nil { //log.Printf("Failed getting execution (streamresult) %s: %s", actionResult.ExecutionId, err) resp.WriteHeader(401) @@ -901,7 +884,7 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { // Finds the child nodes of a node in execution and returns them // Used if e.g. a node in a branch is exited, and all children have to be stopped -func findChildNodes(workflowExecution WorkflowExecution, nodeId string) []string { +func findChildNodes(workflowExecution shuffle.WorkflowExecution, nodeId string) []string { //log.Printf("\nNODE TO FIX: %s\n\n", nodeId) allChildren := []string{nodeId} @@ -955,14 +938,14 @@ func validateNewWorkerExecution(body []byte) error { //} ctx := context.Background() - var execution WorkflowExecution + var execution shuffle.WorkflowExecution err := json.Unmarshal(body, &execution) if err != nil { log.Printf("[WARNING] Failed execution unmarshaling: %s", err) return err } - baseExecution, err := getWorkflowExecution(ctx, execution.ExecutionId) + baseExecution, err := shuffle.GetWorkflowExecution(ctx, execution.ExecutionId) if err != nil { log.Printf("[ERROR] Failed getting execution (workflowqueue) %s: %s", execution.ExecutionId, err) return err @@ -1044,7 +1027,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { //log.Printf("[WARNING] Handling other execution variant: %s", err) } - var actionResult ActionResult + var actionResult shuffle.ActionResult err = json.Unmarshal(body, &actionResult) if err != nil { log.Printf("Failed ActionResult unmarshaling: %s", err) @@ -1060,7 +1043,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { // IF FAIL: Set executionstatus: abort or cancel ctx := context.Background() - workflowExecution, err := getWorkflowExecution(ctx, actionResult.ExecutionId) + workflowExecution, err := shuffle.GetWorkflowExecution(ctx, actionResult.ExecutionId) if err != nil { log.Printf("[ERROR] Failed getting execution (workflowqueue) %s: %s", actionResult.ExecutionId, err) resp.WriteHeader(401) @@ -1099,7 +1082,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { if actionResult.Status == "WAITING" && actionResult.Action.AppName == "User Input" { log.Printf("SHOULD WAIT A BIT AND RUN CLOUD STUFF WITH USER INPUT! WAITING!") - var trigger Trigger + var trigger shuffle.Trigger err = json.Unmarshal([]byte(actionResult.Result), &trigger) if err != nil { log.Printf("Failed unmarshaling actionresult for user input: %s", err) @@ -1152,9 +1135,9 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { } // Will make sure transactions are always ran for an execution. This is recursive if it fails. Allowed to fail up to 5 times -func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workflowExecutionId string, actionResult ActionResult, resp http.ResponseWriter) { +func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workflowExecutionId string, actionResult shuffle.ActionResult, resp http.ResponseWriter) { // Should start a tx for the execution here - workflowExecution, err := getWorkflowExecution(ctx, workflowExecutionId) + workflowExecution, err := shuffle.GetWorkflowExecution(ctx, workflowExecutionId) if err != nil { log.Printf("[ERROR] Failed getting execution cache: %s", err) resp.WriteHeader(401) @@ -1192,7 +1175,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl if actionResult.Status == "ABORTED" || actionResult.Status == "FAILURE" { dbSave = true - newResults := []ActionResult{} + newResults := []shuffle.ActionResult{} childNodes := []string{} if workflowExecution.Workflow.Configuration.ExitOnError { log.Printf("[WARNING] Actionresult is %s for node %s in %s. Should set workflowExecution and exit all running functions", actionResult.Status, actionResult.Action.ID, workflowExecution.ExecutionId) @@ -1214,7 +1197,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // 1. Find the action itself // 2. Create an actionresult - curAction := Action{ID: ""} + curAction := shuffle.Action{ID: ""} for _, action := range workflowExecution.Workflow.Actions { if action.ID == nodeId { curAction = action @@ -1258,7 +1241,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } if !skipNodeAdd { - newAction := Action{ + newAction := shuffle.Action{ AppName: curAction.AppName, AppVersion: curAction.AppVersion, Label: curAction.Label, @@ -1266,7 +1249,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl ID: curAction.ID, } - newResult := ActionResult{ + newResult := shuffle.ActionResult{ Action: newAction, ExecutionId: actionResult.ExecutionId, Authorization: actionResult.Authorization, @@ -1474,7 +1457,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } // FIXME - why isn't this how it works otherwise, wtf? - //workflow, err := getWorkflow(workflowExecution.Workflow.ID) + //workflow, err := shuffle.GetWorkflow(workflowExecution.Workflow.ID) //newActions := []Action{} //for _, action := range workflowExecution.Workflow.Actions { // log.Printf("Name: %s, Env: %s", action.Name, action.Environment) @@ -1489,7 +1472,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // Result string `json:"result" datastore:"result,noindex"` // Arbitrary reduction size maxSize := 500000 - newResults := []ActionResult{} + newResults := []shuffle.ActionResult{} for _, item := range workflowExecution.Results { if len(item.Result) > maxSize { item.Result = "[ERROR] Result too large to handle (https://github.com/frikky/shuffle/issues/171)" @@ -1506,7 +1489,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // Handled using cachhing, so actually pretty fast cacheKey := fmt.Sprintf("workflowexecution-%s", workflowExecution.ExecutionId) if value, found := requestCache.Get(cacheKey); found { - parsedValue := value.(*WorkflowExecution) + parsedValue := value.(*shuffle.WorkflowExecution) if len(parsedValue.Results) > 0 && len(parsedValue.Results) != resultLength { setExecution = false if attempts > 5 { @@ -1587,10 +1570,10 @@ func JSONCheck(str string) bool { return json.Unmarshal([]byte(str), &jsonStr) == nil } -func handleExecutionStatistics(execution WorkflowExecution) { +func handleExecutionStatistics(execution shuffle.WorkflowExecution) { // FIXME: CLEAN UP THE JSON THAT'S SAVED. // https://github.com/frikky/Shuffle/issues/172 - appResults := []AppExecutionExample{} + appResults := []shuffle.AppExecutionExample{} for _, result := range execution.Results { resultCheck := JSONCheck(result.Result) if !resultCheck { @@ -1625,7 +1608,7 @@ func handleExecutionStatistics(execution WorkflowExecution) { } else { // CREATE SuccessExamples or FailureExamples - executionExample := AppExecutionExample{ + executionExample := shuffle.AppExecutionExample{ AppName: result.Action.AppName, AppVersion: result.Action.AppVersion, AppAction: result.Action.Name, @@ -1672,7 +1655,7 @@ func getWorkflows(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in getworkflows: %s", err) resp.WriteHeader(401) @@ -1703,7 +1686,7 @@ func getWorkflows(resp http.ResponseWriter, request *http.Request) { q = q.Order("-edited") - var workflows []Workflow + var workflows []shuffle.Workflow _, err = dbclient.GetAll(ctx, q, &workflows) if err != nil { if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") { @@ -1765,7 +1748,7 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -1781,7 +1764,7 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { return } - var workflow Workflow + var workflow shuffle.Workflow err = json.Unmarshal(body, &workflow) if err != nil { log.Printf("Failed unmarshaling: %s", err) @@ -1793,7 +1776,7 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { workflow.ID = uuid.NewV4().String() workflow.Owner = user.Id workflow.Sharing = "private" - user.ActiveOrg.Users = []User{} + user.ActiveOrg.Users = []shuffle.User{} workflow.ExecutingOrg = user.ActiveOrg workflow.OrgId = user.ActiveOrg.Id //log.Printf("TRIGGERS: %d", len(workflow.Triggers)) @@ -1805,19 +1788,19 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { //} if len(workflow.Actions) == 0 { - workflow.Actions = []Action{} + workflow.Actions = []shuffle.Action{} } if len(workflow.Branches) == 0 { - workflow.Branches = []Branch{} + workflow.Branches = []shuffle.Branch{} } if len(workflow.Triggers) == 0 { - workflow.Triggers = []Trigger{} + workflow.Triggers = []shuffle.Trigger{} } if len(workflow.Errors) == 0 { workflow.Errors = []string{} } - newActions := []Action{} + newActions := []shuffle.Action{} for _, action := range workflow.Actions { if action.Environment == "" { //action.Environment = baseEnvironment @@ -1833,11 +1816,11 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { //log.Printf("APPENDING NEW APP FOR NEW WORKFLOW") // Adds the Testing app if it's a new workflow - workflowapps, err := getAllWorkflowApps(ctx, 500) + workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500) if err == nil { // FIXME: Add real env envName := "Shuffle" - environments, err := getEnvironments(ctx, user.ActiveOrg.Id) + environments, err := shuffle.GetEnvironments(ctx, user.ActiveOrg.Id) if err == nil { for _, env := range environments { if env.Default { @@ -1851,11 +1834,11 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { if item.Name == "Testing" && item.AppVersion == "1.0.0" { nodeId := "40447f30-fa44-4a4f-a133-4ee710368737" workflow.Start = nodeId - newActions = append(newActions, Action{ + newActions = append(newActions, shuffle.Action{ Label: "Start node", Name: "hello_world", Environment: envName, - Parameters: []WorkflowAppActionParameter{}, + Parameters: []shuffle.WorkflowAppActionParameter{}, Position: struct { X float64 "json:\"x,omitempty\" datastore:\"x\"" Y float64 "json:\"y,omitempty\" datastore:\"y\"" @@ -1882,7 +1865,7 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { // FIXME: Check if they require authentication and if they exist locally //log.Printf("\n\nSHOULD VALIDATE AUTHENTICATION") //AuthenticationId string `json:"authentication_id,omitempty" datastore:"authentication_id"` - //allAuths, err := getAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) + //allAuths, err := shuffle.GetAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) //if err == nil { // log.Printf("AUTH: %#v", allAuths) // for _, action := range newActions { @@ -1891,7 +1874,7 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { //} } - workflow.Actions = []Action{} + workflow.Actions = []shuffle.Action{} for _, item := range workflow.Actions { oldId := item.ID sourceIndexes := []int{} @@ -1918,7 +1901,7 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { newActions = append(newActions, item) } - newTriggers := []Trigger{} + newTriggers := []shuffle.Trigger{} for _, item := range workflow.Triggers { oldId := item.ID sourceIndexes := []int{} @@ -1946,7 +1929,7 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { newTriggers = append(newTriggers, item) } - newSchedules := []Schedule{} + newSchedules := []shuffle.Schedule{} for _, item := range workflow.Schedules { item.Id = uuid.NewV4().String() newSchedules = append(newSchedules, item) @@ -1991,7 +1974,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in deleting workflow: %s", err) resp.WriteHeader(401) @@ -2019,7 +2002,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - workflow, err := getWorkflow(ctx, fileId) + workflow, err := shuffle.GetWorkflow(ctx, fileId) if err != nil { log.Printf("Failed getting the workflow locally (delete workflow): %s", err) resp.WriteHeader(401) @@ -2084,7 +2067,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { } // Adds app auth tracking -func updateAppAuth(auth AppAuthenticationStorage, workflowId, nodeId string, add bool) error { +func updateAppAuth(auth shuffle.AppAuthenticationStorage, workflowId, nodeId string, add bool) error { workflowFound := false workflowIndex := 0 nodeFound := false @@ -2108,7 +2091,7 @@ func updateAppAuth(auth AppAuthenticationStorage, workflowId, nodeId string, add updateAuth := false if !workflowFound && add { log.Printf("[INFO] Adding workflow things to auth!") - usageItem := AuthenticationUsage{ + usageItem := shuffle.AuthenticationUsage{ WorkflowId: workflowId, Nodes: []string{nodeId}, } @@ -2127,7 +2110,7 @@ func updateAppAuth(auth AppAuthenticationStorage, workflowId, nodeId string, add if updateAuth { log.Printf("[INFO] Updating auth!") ctx := context.Background() - err := setWorkflowAppAuthDatastore(ctx, auth, auth.Id) + err := shuffle.SetWorkflowAppAuthDatastore(ctx, auth, auth.Id) if err != nil { log.Printf("Failed setting up app auth %s: %s", auth.Id, err) return err @@ -2138,7 +2121,7 @@ func updateAppAuth(auth AppAuthenticationStorage, workflowId, nodeId string, add } // Identifies what a category defined really is -func handleCategoryIncrease(categories Categories, action Action, workflowapps []WorkflowApp) Categories { +func handleCategoryIncrease(categories shuffle.Categories, action shuffle.Action, workflowapps []shuffle.WorkflowApp) shuffle.Categories { if action.Category == "" { appName := action.AppName for _, app := range workflowapps { @@ -2191,7 +2174,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } //log.Println("Start") - user, userErr := handleApiAuthentication(resp, request) + user, userErr := shuffle.HandleApiAuthentication(resp, request) if userErr != nil { log.Printf("Api authentication failed in edit workflow: %s", userErr) resp.WriteHeader(401) @@ -2222,7 +2205,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // Here to check access rights ctx := context.Background() - tmpworkflow, err := getWorkflow(ctx, fileId) + tmpworkflow, err := shuffle.GetWorkflow(ctx, fileId) if err != nil { log.Printf("Failed getting the workflow locally (save workflow): %s", err) resp.WriteHeader(401) @@ -2247,7 +2230,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { return } - var workflow Workflow + var workflow shuffle.Workflow err = json.Unmarshal([]byte(body), &workflow) //log.Printf(string(body)) if err != nil { @@ -2275,17 +2258,17 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { if len(workflow.ExecutingOrg.Id) == 0 { log.Printf("[INFO] Setting executing org for workflow") - user.ActiveOrg.Users = []User{} + user.ActiveOrg.Users = []shuffle.User{} workflow.ExecutingOrg = user.ActiveOrg } // FIXME - this shouldn't be necessary with proper API checks - newActions := []Action{} + newActions := []shuffle.Action{} allNodes := []string{} - workflow.Categories = Categories{} + workflow.Categories = shuffle.Categories{} //log.Printf("PRE APPS") - workflowapps, apperr := getAllWorkflowApps(ctx, 500) + workflowapps, apperr := shuffle.GetAllWorkflowApps(ctx, 500) //log.Printf("Action: %#v", action.Authentication) for _, action := range workflow.Actions { @@ -2322,14 +2305,14 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { newActions = append(newActions, action) } - newTriggers := []Trigger{} + newTriggers := []shuffle.Trigger{} for _, trigger := range workflow.Triggers { log.Printf("[INFO] Trigger %s: %s", trigger.TriggerType, trigger.Status) // Check if it's actually running // FIXME: Do this for other triggers too if trigger.TriggerType == "SCHEDULE" && trigger.Status != "uninitialized" { - schedule, err := getSchedule(ctx, trigger.ID) + schedule, err := shuffle.GetSchedule(ctx, trigger.ID) if err != nil { trigger.Status = "stopped" } else if schedule.Id == "" { @@ -2344,7 +2327,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { if len(user.ApiKey) > 0 { apikey = user.ApiKey } else { - user, err = generateApikey(ctx, user) + user, err = shuffle.GenerateApikey(ctx, user) if err != nil { workflow.IsValid = false workflow.Errors = []string{"Trigger is missing a parameter: %s", param.Name} @@ -2448,13 +2431,13 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { workflow.Triggers = newTriggers if len(workflow.Actions) == 0 { - workflow.Actions = []Action{} + workflow.Actions = []shuffle.Action{} } if len(workflow.Branches) == 0 { - workflow.Branches = []Branch{} + workflow.Branches = []shuffle.Branch{} } if len(workflow.Triggers) == 0 { - workflow.Triggers = []Trigger{} + workflow.Triggers = []shuffle.Trigger{} } if len(workflow.Errors) == 0 { workflow.Errors = []string{} @@ -2500,7 +2483,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { if len(foundNodes) != len(allNodes) || len(workflow.Actions) <= 0 { // This shit takes a few seconds lol if !workflow.IsValid { - oldworkflow, err := getWorkflow(ctx, fileId) + oldworkflow, err := shuffle.GetWorkflow(ctx, fileId) if err != nil { log.Printf("Workflow %s doesn't exist - oldworkflow.", fileId) if workflow.PreviouslySaved { @@ -2542,7 +2525,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // Have to do it like this to add the user's apps //log.Println("Apps set starting") //log.Printf("EXIT ON ERROR: %#v", workflow.Configuration.ExitOnError) - //workflowapps, apperr := getAllWorkflowApps(ctx, 500) + //workflowapps, apperr := shuffle.GetAllWorkflowApps(ctx, 500) // Started getting the single apps, but if it's weird, this is faster // 1. Check workflow.Start @@ -2567,7 +2550,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } } - allAuths, err := getAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) + allAuths, err := shuffle.GetAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) if userErr != nil { log.Printf("Api authentication failed in get all apps: %s", userErr) if workflow.PreviouslySaved { @@ -2579,8 +2562,8 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // Check every app action and param to see whether they exist //log.Printf("PRE ACTIONS 2") - allAuths, autherr := getAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) - newActions = []Action{} + allAuths, autherr := shuffle.GetAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) + newActions = []shuffle.Action{} for _, action := range workflow.Actions { reservedApps := []string{ "0ca8887e-b4af-4e3e-887c-87e9d3bc3d3e", @@ -2629,7 +2612,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { if builtin { newActions = append(newActions, action) } else { - curapp := WorkflowApp{} + curapp := shuffle.WorkflowApp{} // FIXME - can this work with ONLY AppID? for _, app := range workflowapps { if app.ID == action.AppID { @@ -2659,7 +2642,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { //return } else { // Check tosee if the appaction is valid - curappaction := WorkflowAppAction{} + curappaction := shuffle.WorkflowAppAction{} for _, curAction := range curapp.Actions { if action.Name == curAction.Name { curappaction = curAction @@ -2685,7 +2668,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // FIXME - check all parameters to see if they're valid // Includes checking required fields - selectedAuth := AppAuthenticationStorage{} + selectedAuth := shuffle.AppAuthenticationStorage{} if len(action.AuthenticationId) > 0 && autherr == nil { for _, auth := range allAuths { if auth.Id == action.AuthenticationId { @@ -2695,7 +2678,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } } - newParams := []WorkflowAppActionParameter{} + newParams := []shuffle.WorkflowAppActionParameter{} for _, param := range curappaction.Parameters { paramFound := false @@ -2765,7 +2748,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { if autherr == nil && len(workflowapps) > 0 && apperr == nil { //log.Printf("Setting actions") - actionFixing := []Action{} + actionFixing := []shuffle.Action{} appsAdded := []string{} for _, action := range newActions { setAuthentication := false @@ -2813,7 +2796,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // FIXME: Only o this IF there isn't another one for the app already if !authSet { //log.Printf("Validate if the app NEEDS auth or not") - outerapp := WorkflowApp{} + outerapp := shuffle.WorkflowApp{} for _, app := range workflowapps { if app.Name == action.AppName { outerapp = app @@ -2839,21 +2822,21 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // FIXME: Add app auth if !found { timeNow := int64(time.Now().Unix()) - authFields := []AuthenticationStore{} + authFields := []shuffle.AuthenticationStore{} for _, param := range outerapp.Authentication.Parameters { - authFields = append(authFields, AuthenticationStore{ + authFields = append(authFields, shuffle.AuthenticationStore{ Key: param.Name, Value: "", }) } - appAuth := AppAuthenticationStorage{ + appAuth := shuffle.AppAuthenticationStorage{ Active: true, Label: fmt.Sprintf("default_%s", outerapp.Name), Id: uuid.NewV4().String(), App: outerapp, Fields: authFields, - Usage: []AuthenticationUsage{}, + Usage: []shuffle.AuthenticationUsage{}, WorkflowCount: 0, NodeCount: 0, OrgId: user.ActiveOrg.Id, @@ -2861,7 +2844,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { Edited: timeNow, } - err = setWorkflowAppAuthDatastore(ctx, appAuth, appAuth.Id) + err = shuffle.SetWorkflowAppAuthDatastore(ctx, appAuth, appAuth.Id) if err != nil { log.Printf("Failed setting appauth for with name %s", appAuth.Label) } else { @@ -2876,7 +2859,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { //outerapp.Authentication.Required // Authentication Authentication `json:"authentication" yaml:"authentication" required:false datastore:"authentication"` - //workflowapps, apperr := getAllWorkflowApps(ctx, 100) + //workflowapps, apperr := shuffle.GetAllWorkflowApps(ctx, 100) } } @@ -2886,8 +2869,8 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { newActions = actionFixing } else { log.Printf("FirstSave error: %s - %s", err, apperr) - //workflowapps, apperr := getAllWorkflowApps(ctx, 100) - //allAuths, err := getAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) + //workflowapps, apperr := shuffle.GetAllWorkflowApps(ctx, 100) + //allAuths, err := shuffle.GetAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) } workflow.PreviouslySaved = true @@ -3022,7 +3005,7 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - workflowExecution, err := getWorkflowExecution(ctx, executionId) + workflowExecution, err := shuffle.GetWorkflowExecution(ctx, executionId) if err != nil { log.Printf("[ERROR] Failed getting execution (abort) %s: %s", executionId, err) resp.WriteHeader(401) @@ -3041,7 +3024,7 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) { if workflowExecution.Authorization != parsedKey { // FIXME: Check the execution if this fails. - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in abort workflow: %s", err) resp.WriteHeader(401) @@ -3074,7 +3057,7 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) { log.Printf("[INFO] Running shutdown of %s", workflowExecution.ExecutionId) lastResult := "" - newResults := []ActionResult{} + newResults := []shuffle.ActionResult{} // type ActionResult struct { for _, result := range workflowExecution.Results { if result.Status == "EXECUTING" { @@ -3116,7 +3099,7 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) { } if len(workflowExecution.Results) == 0 || addResult { - newaction := Action{ + newaction := shuffle.Action{ ID: workflowExecution.Start, } @@ -3127,7 +3110,7 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) { } } - workflowExecution.Results = append(workflowExecution.Results, ActionResult{ + workflowExecution.Results = append(workflowExecution.Results, shuffle.ActionResult{ Action: newaction, ExecutionId: workflowExecution.ExecutionId, Authorization: workflowExecution.Authorization, @@ -3144,7 +3127,7 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) { if nodeok { nodeId := node[0] log.Printf("[INFO] Found abort node %s", nodeId) - newaction := Action{ + newaction := shuffle.Action{ ID: nodeId, } @@ -3155,7 +3138,7 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) { } } - workflowExecution.Results = append(workflowExecution.Results, ActionResult{ + workflowExecution.Results = append(workflowExecution.Results, shuffle.ActionResult{ Action: newaction, ExecutionId: workflowExecution.ExecutionId, Authorization: workflowExecution.Authorization, @@ -3201,7 +3184,7 @@ func cleanupExecutions(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("[INFO] Api authentication failed in cleanup executions: %s", err) resp.WriteHeader(401) @@ -3221,7 +3204,7 @@ func cleanupExecutions(resp http.ResponseWriter, request *http.Request) { timestamp := int64(time.Now().AddDate(0, -2, 0).Unix()) log.Println(timestamp) q := datastore.NewQuery("workflowexecution").Filter("started_at <", timestamp) - var workflowExecutions []WorkflowExecution + var workflowExecutions []shuffle.WorkflowExecution _, err = dbclient.GetAll(ctx, q, &workflowExecutions) if err != nil { log.Printf("Error getting workflowexec (cleanup): %s", err) @@ -3234,13 +3217,13 @@ func cleanupExecutions(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(`{"success": true}`)) } -func handleExecution(id string, workflow Workflow, request *http.Request) (WorkflowExecution, string, error) { +func handleExecution(id string, workflow shuffle.Workflow, request *http.Request) (shuffle.WorkflowExecution, string, error) { ctx := context.Background() if workflow.ID == "" || workflow.ID != id { - tmpworkflow, err := getWorkflow(ctx, id) + tmpworkflow, err := shuffle.GetWorkflow(ctx, id) if err != nil { log.Printf("Failed getting the workflow locally (execution cleanup): %s", err) - return WorkflowExecution{}, "Failed getting workflow", err + return shuffle.WorkflowExecution{}, "Failed getting workflow", err } workflow = *tmpworkflow @@ -3248,13 +3231,13 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if len(workflow.ExecutingOrg.Id) == 0 { log.Printf("[INFO] Stopped execution because there is no executing org for workflow %s", workflow.ID) - return WorkflowExecution{}, fmt.Sprintf("Workflow has no executing org defined"), errors.New("Workflow has no executing org defined") + return shuffle.WorkflowExecution{}, fmt.Sprintf("Workflow has no executing org defined"), errors.New("Workflow has no executing org defined") } if len(workflow.Actions) == 0 { - workflow.Actions = []Action{} + workflow.Actions = []shuffle.Action{} } else { - newactions := []Action{} + newactions := []shuffle.Action{} for _, action := range workflow.Actions { action.LargeImage = "" action.SmallImage = "" @@ -3266,12 +3249,12 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } if len(workflow.Branches) == 0 { - workflow.Branches = []Branch{} + workflow.Branches = []shuffle.Branch{} } if len(workflow.Triggers) == 0 { - workflow.Triggers = []Trigger{} + workflow.Triggers = []shuffle.Trigger{} } else { - newtriggers := []Trigger{} + newtriggers := []shuffle.Trigger{} for _, trigger := range workflow.Triggers { trigger.LargeImage = "" trigger.SmallImage = "" @@ -3288,21 +3271,21 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if !workflow.IsValid { log.Printf("[ERROR] Stopped execution as workflow %s is not valid.", workflow.ID) - return WorkflowExecution{}, fmt.Sprintf(`workflow %s is invalid`, workflow.ID), errors.New("Failed getting workflow") + return shuffle.WorkflowExecution{}, fmt.Sprintf(`workflow %s is invalid`, workflow.ID), errors.New("Failed getting workflow") } workflowBytes, err := json.Marshal(workflow) if err != nil { log.Printf("Failed workflow unmarshal in execution: %s", err) - return WorkflowExecution{}, "", err + return shuffle.WorkflowExecution{}, "", err } //log.Println(workflow) - var workflowExecution WorkflowExecution + var workflowExecution shuffle.WorkflowExecution err = json.Unmarshal(workflowBytes, &workflowExecution.Workflow) if err != nil { log.Printf("Failed execution unmarshaling: %s", err) - return WorkflowExecution{}, "Failed unmarshal during execution", err + return shuffle.WorkflowExecution{}, "Failed unmarshal during execution", err } makeNew := true @@ -3311,7 +3294,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf body, err := ioutil.ReadAll(request.Body) if err != nil { log.Printf("[ERROR] Failed request POST read: %s", err) - return WorkflowExecution{}, "Failed getting body", err + return shuffle.WorkflowExecution{}, "Failed getting body", err } // This one doesn't really matter. @@ -3353,11 +3336,11 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf log.Printf("Body: %s", string(body)) } - var execution ExecutionRequest + var execution shuffle.ExecutionRequest err = json.Unmarshal(body, &execution) if err != nil { log.Printf("[WARNING] Failed execution POST unmarshaling - continuing anyway: %s", err) - //return WorkflowExecution{}, "", err + //return shuffle.WorkflowExecution{}, "", err } if execution.Start == "" && len(body) > 0 { @@ -3388,12 +3371,12 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if !found { log.Printf("[ERROR] ACTION %s WAS NOT FOUND!", workflow.Start) - return WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", workflow.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", workflow.Start)) + return shuffle.WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", workflow.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", workflow.Start)) } } else if len(execution.Start) > 0 { log.Printf("[ERROR] START ACTION %s IS WRONG ID LENGTH %d!", execution.Start, len(execution.Start)) - return WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", execution.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", execution.Start)) + return shuffle.WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", execution.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", execution.Start)) } if len(execution.ExecutionId) == 36 { @@ -3415,18 +3398,18 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf log.Printf("Should update reference and return, no need for further execution!") // Get the reference execution - oldExecution, err := getWorkflowExecution(ctx, referenceId[0]) + oldExecution, err := shuffle.GetWorkflowExecution(ctx, referenceId[0]) if err != nil { log.Printf("Failed getting execution (execution) %s: %s", referenceId[0], err) - return WorkflowExecution{}, fmt.Sprintf("Failed getting execution ID %s because it doesn't exist.", referenceId[0]), err + return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed getting execution ID %s because it doesn't exist.", referenceId[0]), err } if oldExecution.Workflow.ID != id { log.Println("Wrong workflowid!") - return WorkflowExecution{}, fmt.Sprintf("Bad ID %s", referenceId), errors.New("Bad ID") + return shuffle.WorkflowExecution{}, fmt.Sprintf("Bad ID %s", referenceId), errors.New("Bad ID") } - newResults := []ActionResult{} + newResults := []shuffle.ActionResult{} //log.Printf("%#v", oldExecution.Results) for _, result := range oldExecution.Results { log.Printf("%s - %s", result.Action.ID, start[0]) @@ -3453,20 +3436,20 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf err = setWorkflowExecution(ctx, *oldExecution, true) if err != nil { log.Printf("Error saving workflow execution actionresult setting: %s", err) - return WorkflowExecution{}, fmt.Sprintf("Failed setting workflowexecution actionresult in execution: %s", err), err + return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed setting workflowexecution actionresult in execution: %s", err), err } - return WorkflowExecution{}, "", nil + return shuffle.WorkflowExecution{}, "", nil } } if referenceok { log.Printf("Handling an old execution continuation!") // Will use the old name, but still continue with NEW ID - oldExecution, err := getWorkflowExecution(ctx, referenceId[0]) + oldExecution, err := shuffle.GetWorkflowExecution(ctx, referenceId[0]) if err != nil { log.Printf("Failed getting execution (execution) %s: %s", referenceId[0], err) - return WorkflowExecution{}, fmt.Sprintf("Failed getting execution ID %s because it doesn't exist.", referenceId[0]), err + return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed getting execution ID %s because it doesn't exist.", referenceId[0]), err } workflowExecution = *oldExecution @@ -3492,7 +3475,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf // FIXME - regex uuid, and check if already exists? if len(workflowExecution.ExecutionId) != 36 { log.Printf("Invalid uuid: %s", workflowExecution.ExecutionId) - return WorkflowExecution{}, "Invalid uuid", err + return shuffle.WorkflowExecution{}, "Invalid uuid", err } // FIXME - find owner of workflow @@ -3553,10 +3536,10 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf topic := "workflows" startFound := false // FIXME - remove this? - newActions := []Action{} - defaultResults := []ActionResult{} + newActions := []shuffle.Action{} + defaultResults := []shuffle.ActionResult{} - allAuths := []AppAuthenticationStorage{} + allAuths := []shuffle.AppAuthenticationStorage{} for _, action := range workflowExecution.Workflow.Actions { //action.LargeImage = "" if action.ID == workflowExecution.Start { @@ -3565,20 +3548,20 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf //log.Println(action.Environment) if action.Environment == "" { - return WorkflowExecution{}, fmt.Sprintf("Environment is not defined for %s", action.Name), errors.New("Environment not defined!") + return shuffle.WorkflowExecution{}, fmt.Sprintf("Environment is not defined for %s", action.Name), errors.New("Environment not defined!") } // FIXME: Authentication parameters if len(action.AuthenticationId) > 0 { if len(allAuths) == 0 { - allAuths, err = getAllWorkflowAppAuth(ctx, workflow.ExecutingOrg.Id) + allAuths, err = shuffle.GetAllWorkflowAppAuth(ctx, workflow.ExecutingOrg.Id) if err != nil { log.Printf("Api authentication failed in get all app auth: %s", err) - return WorkflowExecution{}, fmt.Sprintf("Api authentication failed in get all app auth: %s", err), err + return shuffle.WorkflowExecution{}, fmt.Sprintf("Api authentication failed in get all app auth: %s", err), err } } - curAuth := AppAuthenticationStorage{Id: ""} + curAuth := shuffle.AppAuthenticationStorage{Id: ""} for _, auth := range allAuths { if auth.Id == action.AuthenticationId { curAuth = auth @@ -3587,11 +3570,11 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } if len(curAuth.Id) == 0 { - return WorkflowExecution{}, fmt.Sprintf("Auth ID %s doesn't exist", action.AuthenticationId), errors.New(fmt.Sprintf("Auth ID %s doesn't exist", action.AuthenticationId)) + return shuffle.WorkflowExecution{}, fmt.Sprintf("Auth ID %s doesn't exist", action.AuthenticationId), errors.New(fmt.Sprintf("Auth ID %s doesn't exist", action.AuthenticationId)) } // Rebuild params with the right data. This is to prevent issues on the frontend - newParams := []WorkflowAppActionParameter{} + newParams := []shuffle.WorkflowAppActionParameter{} for _, param := range action.Parameters { for _, authparam := range curAuth.Fields { @@ -3620,7 +3603,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf // as it's not a childnode of the startnode // This is a configuration item for the workflow itself. if len(workflowExecution.Results) > 0 { - defaultResults = []ActionResult{} + defaultResults = []shuffle.ActionResult{} for _, result := range workflowExecution.Results { if result.Status == "WAITING" { result.Status = "FINISHED" @@ -3644,7 +3627,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } //log.Printf("[WARNING] Set %s to SKIPPED as it's NOT a childnode of the startnode.", action.ID) - curaction := Action{ + curaction := shuffle.Action{ AppName: action.AppName, AppVersion: action.AppVersion, Label: action.Label, @@ -3653,7 +3636,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } //action //curaction.Parameters = [] - defaultResults = append(defaultResults, ActionResult{ + defaultResults = append(defaultResults, shuffle.ActionResult{ Action: curaction, ExecutionId: workflowExecution.ExecutionId, Authorization: workflowExecution.Authorization, @@ -3687,7 +3670,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if !found { //log.Printf("SHOULD SET TRIGGER %s TO BE SKIPPED", trigger.ID) - curaction := Action{ + curaction := shuffle.Action{ AppName: "shuffle-subflow", AppVersion: trigger.AppVersion, Label: trigger.Label, @@ -3695,7 +3678,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf ID: trigger.ID, } - defaultResults = append(defaultResults, ActionResult{ + defaultResults = append(defaultResults, shuffle.ActionResult{ Action: curaction, ExecutionId: workflowExecution.ExecutionId, Authorization: workflowExecution.Authorization, @@ -3713,7 +3696,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if !startFound { log.Printf("[ERROR] Startnode %s doesn't exist!!", workflowExecution.Start) - return WorkflowExecution{}, fmt.Sprintf("Workflow action %s doesn't exist in workflow", workflowExecution.Start), errors.New(fmt.Sprintf(`Workflow start node "%s" doesn't exist. Exiting!`, workflowExecution.Start)) + return shuffle.WorkflowExecution{}, fmt.Sprintf("Workflow action %s doesn't exist in workflow", workflowExecution.Start), errors.New(fmt.Sprintf(`Workflow start node "%s" doesn't exist. Exiting!`, workflowExecution.Start)) } // Verification for execution environments @@ -3726,14 +3709,14 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf workflowExecution.ExecutionOrg = workflow.ExecutingOrg.Id } - var allEnvs []Environment + var allEnvs []shuffle.Environment if len(workflowExecution.ExecutionOrg) > 0 { //log.Printf("[INFO] Executing ORG: %s", workflowExecution.ExecutionOrg) - allEnvironments, err := getEnvironments(ctx, workflowExecution.ExecutionOrg) + allEnvironments, err := shuffle.GetEnvironments(ctx, workflowExecution.ExecutionOrg) if err != nil { log.Printf("Failed finding environments: %s", err) - return WorkflowExecution{}, fmt.Sprintf("Workflow environments not found for this org"), errors.New(fmt.Sprintf("Workflow environments not found for this org")) + return shuffle.WorkflowExecution{}, fmt.Sprintf("Workflow environments not found for this org"), errors.New(fmt.Sprintf("Workflow environments not found for this org")) } for _, curenv := range allEnvironments { @@ -3745,12 +3728,12 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } } else { log.Printf("[ERROR] No org identified for execution of %s. Returning", workflowExecution.Workflow.ID) - return WorkflowExecution{}, "No org identified for execution", errors.New("No org identified for execution") + return shuffle.WorkflowExecution{}, "No org identified for execution", errors.New("No org identified for execution") } if len(allEnvs) == 0 { log.Printf("[ERROR] No active environments found for org: %s", workflowExecution.ExecutionOrg) - return WorkflowExecution{}, "No active environments found", errors.New(fmt.Sprintf("No active env found for org %s", workflowExecution.ExecutionOrg)) + return shuffle.WorkflowExecution{}, "No active environments found", errors.New(fmt.Sprintf("No active env found for org %s", workflowExecution.ExecutionOrg)) } // Check if the actions are children of the startnode? @@ -3769,7 +3752,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf onpremExecution = true } else { log.Printf("[ERROR] No handler for environment type %s", env.Type) - return WorkflowExecution{}, "No active environments found", errors.New(fmt.Sprintf("No handler for environment type %s", env.Type)) + return shuffle.WorkflowExecution{}, "No active environments found", errors.New(fmt.Sprintf("No handler for environment type %s", env.Type)) } break } @@ -3777,7 +3760,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if !found { log.Printf("[ERROR] Couldn't find environment %s. Maybe it's inactive?", action.Environment) - return WorkflowExecution{}, "Couldn't find the environment", errors.New(fmt.Sprintf("Couldn't find env %s in org %s", action.Environment, workflowExecution.ExecutionOrg)) + return shuffle.WorkflowExecution{}, "Couldn't find the environment", errors.New(fmt.Sprintf("Couldn't find env %s in org %s", action.Environment, workflowExecution.ExecutionOrg)) } found = false @@ -3802,7 +3785,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf err = imageCheckBuilder(imageNames) if err != nil { log.Printf("[ERROR] Failed building the required images from %#v: %s", imageNames, err) - return WorkflowExecution{}, "Failed building missing Docker images", err + return shuffle.WorkflowExecution{}, "Failed building missing Docker images", err } //b, err := json.Marshal(workflowExecution) @@ -3812,17 +3795,17 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf // //workflowExecution.ExecutionOrg.SyncFeatures = Org{} //} - workflowExecution.Workflow.ExecutingOrg = Org{ + workflowExecution.Workflow.ExecutingOrg = shuffle.Org{ Id: workflowExecution.Workflow.ExecutingOrg.Id, } - workflowExecution.Workflow.Org = []Org{ + workflowExecution.Workflow.Org = []shuffle.Org{ workflowExecution.Workflow.ExecutingOrg, } //Org []Org `json:"org,omitempty" datastore:"org"` err = setWorkflowExecution(ctx, workflowExecution, true) if err != nil { log.Printf("Error saving workflow execution for updates %s: %s", topic, err) - return WorkflowExecution{}, "Failed getting workflowexecution", err + return shuffle.WorkflowExecution{}, "Failed getting workflowexecution", err } // Adds queue for onprem execution @@ -3833,7 +3816,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf for _, environment := range environments { log.Printf("[INFO] Execution: %s should execute onprem with execution environment \"%s\". Workflow: %s", workflowExecution.ExecutionId, environment, workflowExecution.Workflow.ID) - executionRequest := ExecutionRequest{ + executionRequest := shuffle.ExecutionRequest{ ExecutionId: workflowExecution.ExecutionId, WorkflowId: workflowExecution.Workflow.ID, Authorization: workflowExecution.Authorization, @@ -3863,7 +3846,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if !featuresList.Workflows.Active || err != nil { log.Printf("Error: %s", err) log.Printf("[ERROR] Cloud not implemented yet. May need to work on app checking and such") - return WorkflowExecution{}, "Cloud not implemented yet", errors.New("Cloud not implemented yet") + return shuffle.WorkflowExecution{}, "Cloud not implemented yet", errors.New("Cloud not implemented yet") } // What it needs to know: @@ -3873,11 +3856,11 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf //cloudExecuteAction(workflowExecution.ExecutionId, workflowExecution.Workflow.Actions[0], workflowExecution.ExecutionOrg, workflowExecution.Workflow.ID) cloudExecuteAction(workflowExecution) - return WorkflowExecution{}, "Cloud not implemented yet (1)", errors.New("Cloud not implemented yet") + return shuffle.WorkflowExecution{}, "Cloud not implemented yet (1)", errors.New("Cloud not implemented yet") } else { // If it's here, it should be controlled by Worker. // If worker, should this backend be a proxy? I think so. - return WorkflowExecution{}, "Cloud not implemented yet (2)", errors.New("Cloud not implemented yet") + return shuffle.WorkflowExecution{}, "Cloud not implemented yet (2)", errors.New("Cloud not implemented yet") } } @@ -3890,21 +3873,21 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } // This updates stuff locally from remote executions -func cloudExecuteAction(execution WorkflowExecution) error { +func cloudExecuteAction(execution shuffle.WorkflowExecution) error { ctx := context.Background() - org, err := getOrg(ctx, execution.ExecutionOrg) + org, err := shuffle.GetOrg(ctx, execution.ExecutionOrg) if err != nil { return err } type ExecutionStruct struct { - ExecutionId string `json:"execution_id" datastore:"execution_id"` - Action Action `json:"action" datastore:"action"` - Authorization string `json:"authorization" datastore:"authorization"` - Results []ActionResult `json:"results" datastore:"results,noindex"` - ExecutionArgument string `json:"execution_argument" datastore:"execution_argument,noindex"` - WorkflowId string `json:"workflow_id" datastore:"workflow_id"` - ExecutionSource string `json:"execution_source" datastore:"execution_source"` + ExecutionId string `json:"execution_id" datastore:"execution_id"` + Action shuffle.Action `json:"action" datastore:"action"` + Authorization string `json:"authorization" datastore:"authorization"` + Results []shuffle.ActionResult `json:"results" datastore:"results,noindex"` + ExecutionArgument string `json:"execution_argument" datastore:"execution_argument,noindex"` + WorkflowId string `json:"workflow_id" datastore:"workflow_id"` + ExecutionSource string `json:"execution_source" datastore:"execution_source"` } data := ExecutionStruct{ @@ -3966,7 +3949,7 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("[INFO] Api authentication failed in execute workflow: %s", err) resp.WriteHeader(401) @@ -3995,7 +3978,7 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { //memcacheName := fmt.Sprintf("%s_%s", user.Username, fileId) ctx := context.Background() - workflow, err := getWorkflow(ctx, fileId) + workflow, err := shuffle.GetWorkflow(ctx, fileId) if err != nil { log.Printf("Failed getting the workflow locally (execute workflow): %s", err) resp.WriteHeader(401) @@ -4014,7 +3997,7 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { log.Printf("[INFO] Starting execution of %s!", fileId) - user.ActiveOrg.Users = []User{} + user.ActiveOrg.Users = []shuffle.User{} workflow.ExecutingOrg = user.ActiveOrg workflowExecution, executionResp, err := handleExecution(fileId, *workflow, request) @@ -4034,7 +4017,7 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in schedule workflow: %s", err) resp.WriteHeader(401) @@ -4070,7 +4053,7 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - workflow, err := getWorkflow(ctx, fileId) + workflow, err := shuffle.GetWorkflow(ctx, fileId) if err != nil { log.Printf("[WARNING] Failed getting the workflow locally (stop schedule): %s", err) resp.WriteHeader(401) @@ -4087,7 +4070,7 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { return } - schedule, err := getSchedule(ctx, scheduleId) + schedule, err := shuffle.GetSchedule(ctx, scheduleId) if err != nil { log.Printf("[WARNING] Failed finding schedule %s", scheduleId) resp.WriteHeader(401) @@ -4100,7 +4083,7 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { if schedule.Environment == "cloud" { log.Printf("[INFO] Should STOP a cloud schedule for workflow %s with schedule ID %s", fileId, scheduleId) // https://shuffler.io/v1/hooks/webhook_80184973-3e82-4852-842e-0290f7f34d7c - org, err := getOrg(ctx, user.ActiveOrg.Id) + org, err := shuffle.GetOrg(ctx, user.ActiveOrg.Id) if err != nil { log.Printf("Failed finding org %s: %s", org.Id, err) return @@ -4163,7 +4146,7 @@ func stopScheduleGCP(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in schedule workflow: %s", err) resp.WriteHeader(401) @@ -4199,7 +4182,7 @@ func stopScheduleGCP(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - workflow, err := getWorkflow(ctx, fileId) + workflow, err := shuffle.GetWorkflow(ctx, fileId) if err != nil { log.Printf("Failed getting the workflow locally (stop schedule GCP): %s", err) resp.WriteHeader(401) @@ -4217,13 +4200,13 @@ func stopScheduleGCP(resp http.ResponseWriter, request *http.Request) { } if len(workflow.Actions) == 0 { - workflow.Actions = []Action{} + workflow.Actions = []shuffle.Action{} } if len(workflow.Branches) == 0 { - workflow.Branches = []Branch{} + workflow.Branches = []shuffle.Branch{} } if len(workflow.Triggers) == 0 { - workflow.Triggers = []Trigger{} + workflow.Triggers = []shuffle.Trigger{} } if len(workflow.Errors) == 0 { workflow.Errors = []string{} @@ -4292,7 +4275,7 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in schedule workflow: %s", err) resp.WriteHeader(401) @@ -4320,7 +4303,7 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - workflow, err := getWorkflow(ctx, fileId) + workflow, err := shuffle.GetWorkflow(ctx, fileId) if err != nil { log.Printf("Failed getting the workflow locally (schedule workflow): %s", err) resp.WriteHeader(401) @@ -4338,13 +4321,13 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { } if len(workflow.Actions) == 0 { - workflow.Actions = []Action{} + workflow.Actions = []shuffle.Action{} } if len(workflow.Branches) == 0 { - workflow.Branches = []Branch{} + workflow.Branches = []shuffle.Branch{} } if len(workflow.Triggers) == 0 { - workflow.Triggers = []Trigger{} + workflow.Triggers = []shuffle.Trigger{} } if len(workflow.Errors) == 0 { workflow.Errors = []string{} @@ -4358,7 +4341,7 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { return } - var schedule Schedule + var schedule shuffle.Schedule err = json.Unmarshal(body, &schedule) if err != nil { log.Printf("Failed schedule POST unmarshaling: %s", err) @@ -4422,7 +4405,7 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { if schedule.Environment == "cloud" { log.Printf("[INFO] Should START a cloud schedule for workflow %s with schedule ID %s", workflow.ID, schedule.Id) // https://shuffler.io/v1/hooks/webhook_80184973-3e82-4852-842e-0290f7f34d7c - org, err := getOrg(ctx, user.ActiveOrg.Id) + org, err := shuffle.GetOrg(ctx, user.ActiveOrg.Id) if err != nil { log.Printf("Failed finding org %s: %s", org.Id, err) return @@ -4523,7 +4506,7 @@ func getSpecificWorkflow(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in getting specific workflow: %s", err) resp.WriteHeader(401) @@ -4569,7 +4552,7 @@ func getSpecificWorkflow(resp http.ResponseWriter, request *http.Request) { // return //} - workflow, err := getWorkflow(ctx, fileId) + workflow, err := shuffle.GetWorkflow(ctx, fileId) if err != nil { log.Printf("Workflow %s doesn't exist.", fileId) resp.WriteHeader(401) @@ -4588,13 +4571,13 @@ func getSpecificWorkflow(resp http.ResponseWriter, request *http.Request) { } if len(workflow.Actions) == 0 { - workflow.Actions = []Action{} + workflow.Actions = []shuffle.Action{} } if len(workflow.Branches) == 0 { - workflow.Branches = []Branch{} + workflow.Branches = []shuffle.Branch{} } if len(workflow.Triggers) == 0 { - workflow.Triggers = []Trigger{} + workflow.Triggers = []shuffle.Trigger{} } if len(workflow.Errors) == 0 { workflow.Errors = []string{} @@ -4643,7 +4626,7 @@ func getSpecificWorkflow(resp http.ResponseWriter, request *http.Request) { resp.Write(body) } -func setWorkflowExecution(ctx context.Context, workflowExecution WorkflowExecution, dbSave bool) error { +func setWorkflowExecution(ctx context.Context, workflowExecution shuffle.WorkflowExecution, dbSave bool) error { //log.Printf("\n\n\nRESULT: %s\n\n\n", workflowExecution.Status) if len(workflowExecution.ExecutionId) == 0 { log.Printf("Workflowexeciton executionId can't be empty.") @@ -4667,11 +4650,11 @@ func setWorkflowExecution(ctx context.Context, workflowExecution WorkflowExecuti return nil } -func getWorkflowExecution(ctx context.Context, id string) (*WorkflowExecution, error) { - workflowExecution := &WorkflowExecution{} +func getWorkflowExecution(ctx context.Context, id string) (*shuffle.WorkflowExecution, error) { + workflowExecution := &shuffle.WorkflowExecution{} cacheKey := fmt.Sprintf("workflowexecution-%s", id) if value, found := requestCache.Get(cacheKey); found { - parsedValue := value.(*WorkflowExecution) + parsedValue := value.(*shuffle.WorkflowExecution) //log.Printf("Found execution for id %s with %d results", parsedValue.ExecutionId, len(parsedValue.Results)) return parsedValue, nil @@ -4688,58 +4671,46 @@ func getWorkflowExecution(ctx context.Context, id string) (*WorkflowExecution, e key := datastore.NameKey("workflowexecution", strings.ToLower(id), nil) if err := dbclient.Get(ctx, key, workflowExecution); err != nil { - return &WorkflowExecution{}, err + return &shuffle.WorkflowExecution{}, err } return workflowExecution, nil } -func getApp(ctx context.Context, id string) (*WorkflowApp, error) { - key := datastore.NameKey("workflowapp", strings.ToLower(id), nil) - workflowApp := &WorkflowApp{} - if err := dbclient.Get(ctx, key, workflowApp); err != nil { - return &WorkflowApp{}, err +//func shuffle.GetApp(ctx context.Context, id string) (*WorkflowApp, error) { +// key := datastore.NameKey("workflowapp", strings.ToLower(id), nil) +// workflowApp := &WorkflowApp{} +// if err := dbclient.Get(ctx, key, workflowApp); err != nil { +// return &WorkflowApp{}, err +// +// } +// +// return workflowApp, nil +//} +// +//func shuffle.GetWorkflow(ctx context.Context, id string) (*shuffle.Workflow, error) { +// key := datastore.NameKey("workflow", strings.ToLower(id), nil) +// workflow := &Workflow{} +// if err := dbclient.Get(ctx, key, workflow); err != nil { +// return &Workflow{}, err +// } +// +// return workflow, nil +//} +// +//func shuffle.GetEnvironments(ctx context.Context, orgId string) ([]Environment, error) { +// var environments []Environment +// q := datastore.NewQuery("Environments").Filter("org_id =", orgId) +// +// _, err := dbclient.GetAll(ctx, q, &environments) +// if err != nil { +// return []Environment{}, err +// } +// +// return environments, nil +//} - } - - return workflowApp, nil -} - -func getWorkflow(ctx context.Context, id string) (*Workflow, error) { - key := datastore.NameKey("workflow", strings.ToLower(id), nil) - workflow := &Workflow{} - if err := dbclient.Get(ctx, key, workflow); err != nil { - return &Workflow{}, err - } - - return workflow, nil -} - -func getEnvironments(ctx context.Context, orgId string) ([]Environment, error) { - var environments []Environment - q := datastore.NewQuery("Environments").Filter("org_id =", orgId) - - _, err := dbclient.GetAll(ctx, q, &environments) - if err != nil { - return []Environment{}, err - } - - return environments, nil -} - -func getAllWorkflows(ctx context.Context, orgId string) ([]Workflow, error) { - var allworkflows []Workflow - q := datastore.NewQuery("workflow").Filter("org_id = ", orgId) - - _, err := dbclient.GetAll(ctx, q, &allworkflows) - if err != nil { - return []Workflow{}, err - } - - return allworkflows, nil -} - -func setExampleresult(ctx context.Context, result AppExecutionExample) error { +func setExampleresult(ctx context.Context, result shuffle.AppExecutionExample) error { // FIXME: Reintroduce this for stats //key := datastore.NameKey("example_result", result.ExampleId, nil) @@ -4754,7 +4725,7 @@ func setExampleresult(ctx context.Context, result AppExecutionExample) error { // Hmm, so I guess this should use uuid :( // Consistency PLX -func setWorkflow(ctx context.Context, workflow Workflow, id string, optionalEditedSecondsOffset ...int) error { +func setWorkflow(ctx context.Context, workflow shuffle.Workflow, id string, optionalEditedSecondsOffset ...int) error { workflow.Edited = int64(time.Now().Unix()) if len(optionalEditedSecondsOffset) > 0 { workflow.Edited += int64(optionalEditedSecondsOffset[0]) @@ -4777,7 +4748,7 @@ func deleteAppAuthentication(resp http.ResponseWriter, request *http.Request) { return } - user, userErr := handleApiAuthentication(resp, request) + user, userErr := shuffle.HandleApiAuthentication(resp, request) if userErr != nil { log.Printf("Api authentication failed in edit workflow: %s", userErr) resp.WriteHeader(401) @@ -4831,7 +4802,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { return } - user, userErr := handleApiAuthentication(resp, request) + user, userErr := shuffle.HandleApiAuthentication(resp, request) if userErr != nil { log.Printf("Api authentication failed in edit workflow: %s", userErr) resp.WriteHeader(401) @@ -4854,7 +4825,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() log.Printf("ID: %s", fileId) - app, err := getApp(ctx, fileId) + app, err := shuffle.GetApp(ctx, fileId) if err != nil { log.Printf("Error getting app (delete) %s: %s", fileId, err) resp.WriteHeader(401) @@ -4879,7 +4850,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { // FIXME: Make workflows track themself INSIDE apps, or with a reference q := datastore.NewQuery("workflow").Filter("org_id = ", user.ActiveOrg.Id).Limit(30) - var workflows []Workflow + var workflows []shuffle.Workflow _, err = dbclient.GetAll(ctx, q, &workflows) if err != nil { log.Printf("Failed getting related workflows for the app: %s", err) @@ -4893,7 +4864,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { for _, workflow := range workflows { found := false - newActions := []Action{} + newActions := []shuffle.Action{} for _, action := range workflow.Actions { if action.AppName == app.Name && action.AppVersion == app.AppVersion { found = true @@ -4944,7 +4915,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { // Not really deleting it, just removing from user cache if private { log.Printf("[INFO] Deleting private app") - var privateApps []WorkflowApp + var privateApps []shuffle.WorkflowApp for _, item := range user.PrivateApps { if item.ID == fileId { continue @@ -4954,7 +4925,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { } user.PrivateApps = privateApps - err = setUser(ctx, &user) + err = shuffle.SetUser(ctx, &user) if err != nil { log.Printf("[ERROR] Failed removing %s app for user %s: %s", app.Name, user.Username, err) resp.WriteHeader(401) @@ -5006,7 +4977,7 @@ func getWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { fileId = location[4] } - app, err := getApp(ctx, fileId) + app, err := shuffle.GetApp(ctx, fileId) if err != nil { log.Printf("[WARNING] Error getting app %s (app config): %s", fileId, err) resp.WriteHeader(401) @@ -5051,7 +5022,7 @@ func getWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { return } - user, userErr := handleApiAuthentication(resp, request) + user, userErr := shuffle.HandleApiAuthentication(resp, request) if userErr != nil { log.Printf("[WARNING] Api authentication failed in get app: %s", userErr) resp.WriteHeader(401) @@ -5099,7 +5070,7 @@ func setAuthenticationConfig(resp http.ResponseWriter, request *http.Request) { return } - user, userErr := handleApiAuthentication(resp, request) + user, userErr := shuffle.HandleApiAuthentication(resp, request) if userErr != nil { log.Printf("Api authentication failed in get all apps: %s", userErr) resp.WriteHeader(401) @@ -5155,7 +5126,7 @@ func setAuthenticationConfig(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - auth, err := getWorkflowAppAuthDatastore(ctx, fileId) + auth, err := shuffle.GetWorkflowAppAuthDatastore(ctx, fileId) if err != nil { log.Printf("Authget error: %s", err) resp.WriteHeader(401) @@ -5174,7 +5145,7 @@ func setAuthenticationConfig(resp http.ResponseWriter, request *http.Request) { q := datastore.NewQuery("workflow").Filter("org_id =", user.ActiveOrg.Id) q = q.Order("-edited").Limit(35) - var workflows []Workflow + var workflows []shuffle.Workflow _, err = dbclient.GetAll(ctx, q, &workflows) if err != nil { log.Printf("Getall error in auth update: %s", err) @@ -5186,11 +5157,11 @@ func setAuthenticationConfig(resp http.ResponseWriter, request *http.Request) { // FIXME: Add function to remove auth from other auth's actionCnt := 0 workflowCnt := 0 - authenticationUsage := []AuthenticationUsage{} + authenticationUsage := []shuffle.AuthenticationUsage{} for _, workflow := range workflows { - newActions := []Action{} + newActions := []shuffle.Action{} edited := false - usage := AuthenticationUsage{ + usage := shuffle.AuthenticationUsage{ WorkflowId: workflow.ID, Nodes: []string{}, } @@ -5229,7 +5200,7 @@ func setAuthenticationConfig(resp http.ResponseWriter, request *http.Request) { auth.Usage = authenticationUsage auth.Defined = true - err = setWorkflowAppAuthDatastore(ctx, *auth, auth.Id) + err = shuffle.SetWorkflowAppAuthDatastore(ctx, *auth, auth.Id) if err != nil { log.Printf("Failed setting appauth: %s", err) resp.WriteHeader(401) @@ -5254,7 +5225,7 @@ func addAppAuthentication(resp http.ResponseWriter, request *http.Request) { return } - user, userErr := handleApiAuthentication(resp, request) + user, userErr := shuffle.HandleApiAuthentication(resp, request) if userErr != nil { log.Printf("Api authentication failed in get all apps: %s", userErr) resp.WriteHeader(401) @@ -5270,7 +5241,7 @@ func addAppAuthentication(resp http.ResponseWriter, request *http.Request) { return } - var appAuth AppAuthenticationStorage + var appAuth shuffle.AppAuthenticationStorage err = json.Unmarshal(body, &appAuth) if err != nil { log.Printf("Failed unmarshaling (appauth): %s", err) @@ -5283,7 +5254,7 @@ func addAppAuthentication(resp http.ResponseWriter, request *http.Request) { if len(appAuth.Id) == 0 { appAuth.Id = uuid.NewV4().String() } else { - auth, err := getWorkflowAppAuthDatastore(ctx, appAuth.Id) + auth, err := shuffle.GetWorkflowAppAuthDatastore(ctx, appAuth.Id) if err == nil { // OrgId string `json:"org_id" datastore:"org_id"` if auth.OrgId != user.ActiveOrg.Id { @@ -5331,10 +5302,10 @@ func addAppAuthentication(resp http.ResponseWriter, request *http.Request) { } // FIXME: Doens't validate Org - app, err := getApp(ctx, appAuth.App.ID) + app, err := shuffle.GetApp(ctx, appAuth.App.ID) if err != nil { log.Printf("[WARNING] Failed finding app %s while setting auth. Finding it by looping apps.", appAuth.App.ID) - workflowapps, err := getAllWorkflowApps(ctx, 500) + workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500) if err != nil { resp.WriteHeader(409) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) @@ -5379,7 +5350,7 @@ func addAppAuthentication(resp http.ResponseWriter, request *http.Request) { //appAuth.LargeImage = "" appAuth.OrgId = user.ActiveOrg.Id appAuth.Defined = true - err = setWorkflowAppAuthDatastore(ctx, appAuth, appAuth.Id) + err = shuffle.SetWorkflowAppAuthDatastore(ctx, appAuth, appAuth.Id) if err != nil { log.Printf("Failed setting up app auth %s: %s", appAuth.Id, err) resp.WriteHeader(409) @@ -5397,7 +5368,7 @@ func getAppAuthentication(resp http.ResponseWriter, request *http.Request) { return } - user, userErr := handleApiAuthentication(resp, request) + user, userErr := shuffle.HandleApiAuthentication(resp, request) if userErr != nil { log.Printf("Api authentication failed in get all apps: %s", userErr) resp.WriteHeader(401) @@ -5413,7 +5384,7 @@ func getAppAuthentication(resp http.ResponseWriter, request *http.Request) { // return //} ctx := context.Background() - allAuths, err := getAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) + allAuths, err := shuffle.GetAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) if err != nil { log.Printf("Api authentication failed in get all app auth: %s", err) resp.WriteHeader(401) @@ -5428,7 +5399,7 @@ func getAppAuthentication(resp http.ResponseWriter, request *http.Request) { } // Cleanup for frontend usage. User shouldn't be able to get the data. - newAuth := []AppAuthenticationStorage{} + newAuth := []shuffle.AppAuthenticationStorage{} for _, auth := range allAuths { newAuthField := auth for index, _ := range auth.Fields { @@ -5509,7 +5480,7 @@ func updateWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { return } - user, userErr := handleApiAuthentication(resp, request) + user, userErr := shuffle.HandleApiAuthentication(resp, request) if userErr != nil { log.Printf("Api authentication failed in get all apps: %s", userErr) resp.WriteHeader(401) @@ -5530,7 +5501,7 @@ func updateWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - app, err := getApp(ctx, fileId) + app, err := shuffle.GetApp(ctx, fileId) if err != nil { log.Printf("Error getting app (update app): %s", fileId) resp.WriteHeader(401) @@ -5575,7 +5546,7 @@ func updateWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { app.SharingConfig = tmpfields.SharingConfig } - err = setWorkflowAppDatastore(ctx, *app, app.ID) + err = shuffle.SetWorkflowAppDatastore(ctx, *app, app.ID) if err != nil { log.Printf("Failed patching workflowapp: %s", err) resp.WriteHeader(401) @@ -5606,7 +5577,7 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() // Just need to be logged in // FIXME - need to be logged in? - user, userErr := handleApiAuthentication(resp, request) + user, userErr := shuffle.HandleApiAuthentication(resp, request) if userErr != nil { log.Printf("Continuing with apps even without auth") //log.Printf("Api authentication failed in get all apps: %s", userErr) @@ -5643,7 +5614,7 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { // return //} - workflowapps, err := getAllWorkflowApps(ctx, 500) + workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500) if err != nil { log.Printf("Failed getting apps (getworkflowapps): %s", err) resp.WriteHeader(401) @@ -5756,7 +5727,7 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { // Bad check for workflowapps :) // FIXME - use tags and struct reflection -func checkWorkflowApp(workflowApp WorkflowApp) error { +func checkWorkflowApp(workflowApp shuffle.WorkflowApp) error { // Validate fields if workflowApp.Name == "" { return errors.New("App field name doesn't exist") @@ -5804,7 +5775,7 @@ func getSpecificApps(resp http.ResponseWriter, request *http.Request) { // Just need to be logged in // FIXME - should have some permissions? - _, err := handleApiAuthentication(resp, request) + _, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new app: %s", err) resp.WriteHeader(401) @@ -5836,7 +5807,7 @@ func getSpecificApps(resp http.ResponseWriter, request *http.Request) { // FIXME - continue the search here with github repos etc. // Caching might be smart :D ctx := context.Background() - workflowapps, err := getAllWorkflowApps(ctx, 500) + workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500) if err != nil { log.Printf("Error: Failed getting workflowapps: %s", err) resp.WriteHeader(401) @@ -5844,7 +5815,7 @@ func getSpecificApps(resp http.ResponseWriter, request *http.Request) { return } - returnValues := []WorkflowApp{} + returnValues := []shuffle.WorkflowApp{} search := strings.ToLower(tmpBody.Search) for _, app := range workflowapps { if !app.Activated && app.Generated { @@ -5879,7 +5850,7 @@ func validateAppInput(resp http.ResponseWriter, request *http.Request) { // Just need to be logged in // FIXME - should have some permissions? - _, err := handleApiAuthentication(resp, request) + _, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new app: %s", err) resp.WriteHeader(401) @@ -6138,7 +6109,7 @@ func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) { // Just need to be logged in // FIXME - should have some permissions? - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in load apps: %s", err) resp.WriteHeader(401) @@ -6205,7 +6176,7 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { // Just need to be logged in // FIXME - should have some permissions? - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in app hotload: %s", err) resp.WriteHeader(401) @@ -6252,7 +6223,7 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) { // Just need to be logged in // FIXME - should have some permissions? - _, err := handleApiAuthentication(resp, request) + _, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in load specific apps: %s", err) resp.WriteHeader(401) @@ -6378,7 +6349,7 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) { func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error { ctx := context.Background() - workflowapps, err := getAllWorkflowApps(ctx, 500) + workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500) appCounter := 0 if err != nil { log.Printf("Failed to get existing generated apps") @@ -6465,7 +6436,7 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, } //log.Printf("Should generate yaml") - swagger, api, _, err := generateYaml(swagger, parsedOpenApi.ID) + swagger, api, _, err := shuffle.GenerateYaml(swagger, parsedOpenApi.ID) if err != nil { log.Printf("Failed building and generating yaml in loop (%s): %s", filename, err) continue @@ -6490,7 +6461,7 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, } if !found { - err = setWorkflowAppDatastore(ctx, api, api.ID) + err = shuffle.SetWorkflowAppDatastore(ctx, api, api.ID) if err != nil { log.Printf("Failed setting workflowapp in loop: %s", err) continue @@ -6582,7 +6553,7 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra continue } - var workflow Workflow + var workflow shuffle.Workflow err = json.Unmarshal(readFile, &workflow) if err != nil { continue @@ -6595,11 +6566,11 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra workflow.ID = uuid.NewV4().String() workflow.OrgId = orgId - workflow.ExecutingOrg = Org{ + workflow.ExecutingOrg = shuffle.Org{ Id: orgId, } - workflow.Org = append(workflow.Org, Org{ + workflow.Org = append(workflow.Org, shuffle.Org{ Id: orgId, }) workflow.IsValid = false @@ -6646,7 +6617,7 @@ type buildLaterStruct struct { func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate bool) ([]buildLaterStruct, []buildLaterStruct, error) { var err error - allapps := []WorkflowApp{} + allapps := []shuffle.WorkflowApp{} // These are slow apps to build with some funky mechanisms reservedNames := []string{ @@ -6757,7 +6728,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin combined = append(combined, dockerfileData...) md5 := md5sum(combined) - var workflowapp WorkflowApp + var workflowapp shuffle.WorkflowApp err = gyaml.Unmarshal(appfileData, &workflowapp) if err != nil { log.Printf("Failed unmarshaling workflowapp %s: %s", fullPath, err) @@ -6772,7 +6743,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin } if len(allapps) == 0 { - allapps, err = getAllWorkflowApps(ctx, 500) + allapps, err = shuffle.GetAllWorkflowApps(ctx, 500) if err != nil { log.Printf("Failed getting apps to verify: %s", err) continue @@ -6816,7 +6787,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin // 2. Check if they're present in the action // 3. Add them IF they DONT exist // 4. Fix python code with reflection (FIXME) - appendParams := []WorkflowAppActionParameter{} + appendParams := []shuffle.WorkflowAppActionParameter{} for _, fieldname := range workflowapp.Authentication.Parameters { found := false for index, param := range action.Parameters { @@ -6830,7 +6801,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin } if !found { - appendParams = append(appendParams, WorkflowAppActionParameter{ + appendParams = append(appendParams, shuffle.WorkflowAppActionParameter{ Name: fieldname.Name, Description: fieldname.Description, Example: fieldname.Example, @@ -6872,7 +6843,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin workflowapp.Downloaded = true workflowapp.Hash = md5 - err = setWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) + err = shuffle.SetWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) if err != nil { log.Printf("Failed setting workflowapp: %s", err) continue @@ -6972,7 +6943,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { // Just need to be logged in // FIXME - should have some permissions? - _, err := handleApiAuthentication(resp, request) + _, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new app: %s", err) resp.WriteHeader(401) @@ -6988,7 +6959,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { return } - var workflowapp WorkflowApp + var workflowapp shuffle.WorkflowApp err = json.Unmarshal(body, &workflowapp) if err != nil { log.Printf("Failed unmarshaling: %s", err) @@ -6998,7 +6969,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - allapps, err := getAllWorkflowApps(ctx, 500) + allapps, err := shuffle.GetAllWorkflowApps(ctx, 500) if err != nil { log.Printf("Failed getting apps to verify: %s", err) resp.WriteHeader(401) @@ -7039,7 +7010,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { workflowapp.Generated = false workflowapp.Activated = true - err = setWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) + err = shuffle.SetWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) if err != nil { log.Printf("Failed setting workflowapp: %s", err) resp.WriteHeader(401) @@ -7065,7 +7036,7 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in getting specific workflow: %s", err) resp.WriteHeader(401) @@ -7093,7 +7064,7 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - workflow, err := getWorkflow(ctx, fileId) + workflow, err := shuffle.GetWorkflow(ctx, fileId) if err != nil { log.Printf("Failed getting the workflow %s locally (get executions): %s", fileId, err) resp.WriteHeader(401) @@ -7112,7 +7083,7 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) { // Query for the specifci workflowId maxAmount := 30 q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(maxAmount) - var workflowExecutions []WorkflowExecution + var workflowExecutions []shuffle.WorkflowExecution _, err = dbclient.GetAll(ctx, q, &workflowExecutions) if err != nil { @@ -7133,7 +7104,7 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) { it := dbclient.Run(ctx, q) //_, err = it.Next(&app) for { - var workflowExecution WorkflowExecution + var workflowExecution shuffle.WorkflowExecution _, err := it.Next(&workflowExecution) if err != nil { break @@ -7227,169 +7198,169 @@ func getAllSchedules(ctx context.Context, orgId string) ([]ScheduleOld, error) { } //FIXME: Add cursor -func getAllWorkflowApps(ctx context.Context, maxLen int) ([]WorkflowApp, error) { - var apps []WorkflowApp - query := datastore.NewQuery("workflowapp").Order("-edited").Limit(10) - //query := datastore.NewQuery("workflowapp").Order("-edited").Limit(40) +//func shuffle.GetAllWorkflowApps(ctx context.Context, maxLen int) ([]shuffle.WorkflowApp, error) { +// var apps []WorkflowApp +// query := datastore.NewQuery("workflowapp").Order("-edited").Limit(10) +// //query := datastore.NewQuery("workflowapp").Order("-edited").Limit(40) +// +// cacheKey := fmt.Sprintf("workflowapps-sorted-%d", maxLen) +// if value, found := requestCache.Get(cacheKey); found { +// parsedValue := value.(*[]WorkflowApp) +// log.Printf("[INFO] Returning %d apps from cache", len(*parsedValue)) +// return *parsedValue, nil +// } +// +// cursorStr := "" +// +// // NOT BEING UPDATED +// // FIXME: Update the app with the correct actions. HOW DOES THIS WORK?? +// // Seems like only actions are wrong. Could get the app individually. +// // Guessing it's a memory issue. +// //Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions,noindex"` +// //errors.New(nil) +// var err error +// for { +// it := dbclient.Run(ctx, query) +// //_, err = it.Next(&app) +// for { +// var app WorkflowApp +// _, err := it.Next(&app) +// if err != nil { +// break +// } +// +// if app.Name == "Shuffle Subflow" { +// continue +// } +// +// found := false +// //log.Printf("ACTIONS: %d - %s", len(app.Actions), app.Name) +// for _, innerapp := range apps { +// if innerapp.Name == app.Name { +// found = true +// break +// } +// } +// +// if !found { +// apps = append(apps, app) +// } +// } +// +// if err != iterator.Done { +// //log.Printf("[INFO] Failed fetching results: %v", err) +// //break +// } +// +// // Get the cursor for the next page of results. +// nextCursor, err := it.Cursor() +// if err != nil { +// log.Printf("Cursorerror: %s", err) +// break +// } else { +// //log.Printf("NEXTCURSOR: %s", nextCursor) +// nextStr := fmt.Sprintf("%s", nextCursor) +// if cursorStr == nextStr { +// break +// } +// +// cursorStr = nextStr +// query = query.Start(nextCursor) +// //cursorStr = nextCursor +// //break +// } +// +// if len(apps) > maxLen { +// break +// } +// } +// +// if len(apps) > 20 { +// log.Printf("[INFO] Setting %d apps in cache", len(apps)) +// requestCache.Set(cacheKey, &apps, cache.DefaultExpiration) +// } +// +// //var allworkflowapps []WorkflowApp +// //_, err := dbclient.GetAll(ctx, query, &allworkflowapps) +// //if err != nil { +// // if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") { +// // //datastore.NewQuery("workflowapp").Limit(30).Order("-edited") +// // query = datastore.NewQuery("workflowapp").Order("-edited").Limit(25) +// // //q := q.Limit(25) +// // _, err := dbclient.GetAll(ctx, query, &allworkflowapps) +// // if err != nil { +// // return []WorkflowApp{}, err +// // } +// // } else { +// // return []WorkflowApp{}, err +// // } +// //} +// +// return apps, nil +//} - cacheKey := fmt.Sprintf("workflowapps-sorted-%d", maxLen) - if value, found := requestCache.Get(cacheKey); found { - parsedValue := value.(*[]WorkflowApp) - log.Printf("[INFO] Returning %d apps from cache", len(*parsedValue)) - return *parsedValue, nil - } - - cursorStr := "" - - // NOT BEING UPDATED - // FIXME: Update the app with the correct actions. HOW DOES THIS WORK?? - // Seems like only actions are wrong. Could get the app individually. - // Guessing it's a memory issue. - //Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions,noindex"` - //errors.New(nil) - var err error - for { - it := dbclient.Run(ctx, query) - //_, err = it.Next(&app) - for { - var app WorkflowApp - _, err := it.Next(&app) - if err != nil { - break - } - - if app.Name == "Shuffle Subflow" { - continue - } - - found := false - //log.Printf("ACTIONS: %d - %s", len(app.Actions), app.Name) - for _, innerapp := range apps { - if innerapp.Name == app.Name { - found = true - break - } - } - - if !found { - apps = append(apps, app) - } - } - - if err != iterator.Done { - //log.Printf("[INFO] Failed fetching results: %v", err) - //break - } - - // Get the cursor for the next page of results. - nextCursor, err := it.Cursor() - if err != nil { - log.Printf("Cursorerror: %s", err) - break - } else { - //log.Printf("NEXTCURSOR: %s", nextCursor) - nextStr := fmt.Sprintf("%s", nextCursor) - if cursorStr == nextStr { - break - } - - cursorStr = nextStr - query = query.Start(nextCursor) - //cursorStr = nextCursor - //break - } - - if len(apps) > maxLen { - break - } - } - - if len(apps) > 20 { - log.Printf("[INFO] Setting %d apps in cache", len(apps)) - requestCache.Set(cacheKey, &apps, cache.DefaultExpiration) - } - - //var allworkflowapps []WorkflowApp - //_, err := dbclient.GetAll(ctx, query, &allworkflowapps) - //if err != nil { - // if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") { - // //datastore.NewQuery("workflowapp").Limit(30).Order("-edited") - // query = datastore.NewQuery("workflowapp").Order("-edited").Limit(25) - // //q := q.Limit(25) - // _, err := dbclient.GetAll(ctx, query, &allworkflowapps) - // if err != nil { - // return []WorkflowApp{}, err - // } - // } else { - // return []WorkflowApp{}, err - // } - //} - - return apps, nil -} - -func getAllWorkflowAppAuth(ctx context.Context, OrgId string) ([]AppAuthenticationStorage, error) { - var allworkflowapps []AppAuthenticationStorage - q := datastore.NewQuery("workflowappauth").Filter("org_id = ", OrgId) - - _, err := dbclient.GetAll(ctx, q, &allworkflowapps) - if err != nil { - return []AppAuthenticationStorage{}, err - } - - return allworkflowapps, nil -} - -func getWorkflowAppAuthDatastore(ctx context.Context, id string) (*AppAuthenticationStorage, error) { - - key := datastore.NameKey("workflowappauth", id, nil) - appAuth := &AppAuthenticationStorage{} - // New struct, to not add body, author etc - if err := dbclient.Get(ctx, key, appAuth); err != nil { - return &AppAuthenticationStorage{}, err - } - - return appAuth, nil -} - -func setWorkflowAppAuthDatastore(ctx context.Context, workflowappauth AppAuthenticationStorage, id string) error { - timeNow := int64(time.Now().Unix()) - if workflowappauth.Created == 0 { - workflowappauth.Created = timeNow - } - - workflowappauth.Edited = timeNow - - key := datastore.NameKey("workflowappauth", id, nil) - - // New struct, to not add body, author etc - if _, err := dbclient.Put(ctx, key, &workflowappauth); err != nil { - log.Printf("Error adding workflow app auth: %s", err) - return err - } - - return nil -} - -// Hmm, so I guess this should use uuid :( -// Consistency PLX -func setWorkflowAppDatastore(ctx context.Context, workflowapp WorkflowApp, id string) error { - timeNow := int64(time.Now().Unix()) - if workflowapp.Created == 0 { - workflowapp.Created = timeNow - } - - workflowapp.Edited = timeNow - key := datastore.NameKey("workflowapp", id, nil) - - // New struct, to not add body, author etc - if _, err := dbclient.Put(ctx, key, &workflowapp); err != nil { - log.Printf("Error adding workflow app: %s", err) - return err - } - - return nil -} +//func shuffle.GetAllWorkflowAppAuth(ctx context.Context, OrgId string) ([]shuffle.AppAuthenticationStorage, error) { +// var allworkflowapps []AppAuthenticationStorage +// q := datastore.NewQuery("workflowappauth").Filter("org_id = ", OrgId) +// +// _, err := dbclient.GetAll(ctx, q, &allworkflowapps) +// if err != nil { +// return []AppAuthenticationStorage{}, err +// } +// +// return allworkflowapps, nil +//} +// +//func getWorkflowAppAuthDatastore(ctx context.Context, id string) (*AppAuthenticationStorage, error) { +// +// key := datastore.NameKey("workflowappauth", id, nil) +// appAuth := &AppAuthenticationStorage{} +// // New struct, to not add body, author etc +// if err := dbclient.Get(ctx, key, appAuth); err != nil { +// return &AppAuthenticationStorage{}, err +// } +// +// return appAuth, nil +//} +// +//func shuffle.SetWorkflowAppAuthDatastore(ctx context.Context, workflowappauth AppAuthenticationStorage, id string) error { +// timeNow := int64(time.Now().Unix()) +// if workflowappauth.Created == 0 { +// workflowappauth.Created = timeNow +// } +// +// workflowappauth.Edited = timeNow +// +// key := datastore.NameKey("workflowappauth", id, nil) +// +// // New struct, to not add body, author etc +// if _, err := dbclient.Put(ctx, key, &workflowappauth); err != nil { +// log.Printf("Error adding workflow app auth: %s", err) +// return err +// } +// +// return nil +//} +// +//// Hmm, so I guess this should use uuid :( +//// Consistency PLX +//func SetWorkflowAppDatastore(ctx context.Context, workflowapp WorkflowApp, id string) error { +// timeNow := int64(time.Now().Unix()) +// if workflowapp.Created == 0 { +// workflowapp.Created = timeNow +// } +// +// workflowapp.Edited = timeNow +// key := datastore.NameKey("workflowapp", id, nil) +// +// // New struct, to not add body, author etc +// if _, err := dbclient.Put(ctx, key, &workflowapp); err != nil { +// log.Printf("Error adding workflow app: %s", err) +// return err +// } +// +// return nil +//} // Starts a new webhook func handleStopHook(resp http.ResponseWriter, request *http.Request) { @@ -7398,7 +7369,7 @@ func handleStopHook(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -7481,7 +7452,7 @@ func handleDeleteHook(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -7543,7 +7514,7 @@ func handleDeleteHook(resp http.ResponseWriter, request *http.Request) { log.Printf("Hook: %#v", hook) if hook.Environment == "cloud" { log.Printf("[INFO] Should STOP cloud webhook https://shuffler.io/api/v1/hooks/webhook_%s", hook.Id) - org, err := getOrg(ctx, user.ActiveOrg.Id) + org, err := shuffle.GetOrg(ctx, user.ActiveOrg.Id) if err != nil { log.Printf("Failed finding org %s: %s", org.Id, err) return @@ -7622,7 +7593,7 @@ func handleStartHook(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -7730,7 +7701,7 @@ func removeOutlookTriggerFunction(ctx context.Context, triggerId string) error { return nil } -func handleUserInput(trigger Trigger, organizationId string, workflowId string, referenceExecution string) error { +func handleUserInput(trigger shuffle.Trigger, organizationId string, workflowId string, referenceExecution string) error { // E.g. check email sms := "" email := "" @@ -7768,7 +7739,7 @@ func handleUserInput(trigger Trigger, organizationId string, workflowId string, FifthItem: referenceExecution, } - org, err := getOrg(ctx, organizationId) + org, err := shuffle.GetOrg(ctx, organizationId) if err != nil { log.Printf("Failed email send to cloud (1): %s", err) return err @@ -7794,7 +7765,7 @@ func handleUserInput(trigger Trigger, organizationId string, workflowId string, FifthItem: referenceExecution, } - org, err := getOrg(ctx, organizationId) + org, err := shuffle.GetOrg(ctx, organizationId) if err != nil { log.Printf("Failed sms send to cloud (3): %s", err) return err From 7e422c4d8b76334d6e7762ab1e3cfa0c0f2d3bd5 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 23 Mar 2021 09:58:04 +0100 Subject: [PATCH 47/57] More cloud sync migrations --- backend/go-app/go.mod | 4 +- backend/go-app/main.go | 658 +-------- backend/go-app/walkoff.go | 1878 +----------------------- frontend/src/views/AngularWorkflow.jsx | 31 +- frontend/src/views/Workflows.jsx | 3 +- functions/onprem/worker/worker.go | 2 +- 6 files changed, 110 insertions(+), 2466 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index e403c8bd..f41eccf3 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -3,7 +3,6 @@ module shuffle go 1.13 replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared - replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi require ( @@ -13,14 +12,13 @@ require ( cloud.google.com/go/storage v1.12.0 github.com/Microsoft/go-winio v0.4.14 // indirect github.com/basgys/goxml2json v1.1.0 + github.com/frikky/kin-openapi v0.38.0 github.com/carlescere/scheduler v0.0.0-20170109141437-ee74d2f83d82 github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/docker v1.13.1 github.com/docker/go-connections v0.4.0 github.com/docker/go-units v0.4.0 // indirect github.com/frikky/shuffle-shared v0.0.12 // indirect - github.com/getkin/kin-openapi v0.52.0 // indirect - //github.com/getkin/kin-openapi v0.8.0 github.com/ghodss/yaml v1.0.0 github.com/go-git/go-billy/v5 v5.0.0 github.com/go-git/go-git/v5 v5.0.0 diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 11df5536..01676100 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -63,7 +63,6 @@ import ( // Web "github.com/gorilla/mux" - "github.com/patrickmn/go-cache" "google.golang.org/api/option" "google.golang.org/grpc" http2 "gopkg.in/src-d/go-git.v4/plumbing/transport/http" @@ -84,7 +83,6 @@ var syncSubUrl = "https://shuffler.io" //var syncSubUrl = "https://050196912a9d.ngrok.io" var dbclient *datastore.Client -var requestCache *cache.Cache type Userapi struct { Username string `datastore:"username"` @@ -667,36 +665,6 @@ func parseLoginParameters(resp http.ResponseWriter, request *http.Request) (logi return t, nil } -// Can check against HIBP etc? -// Removed for localhost -func checkPasswordStrength(password string) error { - // Check password strength here - if len(password) < 3 { - return errors.New("Minimum password length is 3.") - } - - //if len(password) > 128 { - // return errors.New("Maximum password length is 128.") - //} - - //re := regexp.MustCompile("[0-9]+") - //if len(re.FindAllString(password, -1)) == 0 { - // return errors.New("Password must contain a number") - //} - - //re = regexp.MustCompile("[a-z]+") - //if len(re.FindAllString(password, -1)) == 0 { - // return errors.New("Password must contain a lower case char") - //} - - //re = regexp.MustCompile("[A-Z]+") - //if len(re.FindAllString(password, -1)) == 0 { - // return errors.New("Password must contain an upper case char") - //} - - return nil -} - func deleteUser(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -959,7 +927,7 @@ func handleSetEnvironments(resp http.ResponseWriter, request *http.Request) { func createNewUser(username, password, role, apikey string, org shuffle.Org) error { // Returns false if there is an issue // Use this for register - err := checkPasswordStrength(password) + err := shuffle.CheckPasswordStrength(password) if err != nil { log.Printf("Bad password strength: %s", err) return err @@ -1145,104 +1113,6 @@ func handleCookie(request *http.Request) bool { return true } -func handleLogout(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - http.SetCookie(resp, &http.Cookie{ - Name: "session_token", - Value: "", - Path: "/", - Expires: time.Unix(0, 0), - }) - - userInfo, err := shuffle.HandleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in handleLogout: %s", err) - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true, "reason": "Not logged in"}`)) - return - } - - ctx := context.Background() - session, err := shuffle.GetSession(ctx, userInfo.Session) - if err != nil { - log.Printf("Session %#v doesn't exist: %s", session, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "No session"}`)) - return - } - - // Check cookie - //c, err := request.Cookie("session_token") - //if err != nil { - // resp.WriteHeader(200) - // resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - // return - //} else { - // log.Printf("Session cookie is set to %s!", c.Value) - //} - - //var Userdata User - //ctx := context.Background() - //sessionToken = c.Value - //session, err := getSession(ctx, sessionToken) - //if err != nil { - // log.Printf("[WARNING] Session %s doesn't exist (logout): %s", sessionToken, err) - // resp.WriteHeader(401) - // resp.Write([]byte(`{"success": false, "reason": "Couldn't find your session"}`)) - // return - //} - - // Get session first - // Should basically never happen - //_, err = shuffle.GetUser(ctx, session.Id) - //if err != nil { - // log.Printf("Username %s doesn't exist (logout): %s", session.Username, err) - // resp.WriteHeader(401) - // resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) - // return - //} - - // Userdata = *tmpdata - //} - - // FIXME - // Session might delete someone elses here? - // No need to think about before possible scale..? - err = shuffle.SetSession(ctx, userInfo, "") - if err != nil { - log.Printf("Error removing session for: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) - return - } - - err = DeleteKey(ctx, "sessions", userInfo.Session) - if err != nil { - log.Printf("Error deleting key %s for %s: %s", userInfo.Session, userInfo.Username, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) - return - } - - userInfo.Session = "" - err = shuffle.SetUser(ctx, &userInfo) - if err != nil { - log.Printf("Failed updating user: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Failed updating apikey"}`)) - return - } - - //memcache.Delete(request.Context(), sessionToken) - - resp.WriteHeader(200) - resp.Write([]byte(`{"success": false, "reason": "Successfully logged out"}`)) -} - func handleUpdateUser(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -1370,105 +1240,6 @@ func handleUpdateUser(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } -func handleApiGeneration(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - userInfo, err := shuffle.HandleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in apigen: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - ctx := context.Background() - if request.Method == "GET" { - newUserInfo, err := shuffle.GenerateApikey(ctx, userInfo) - if err != nil { - log.Printf("Failed to generate apikey for user %s: %s", userInfo.Username, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": ""}`)) - return - } - userInfo = newUserInfo - log.Printf("Updated apikey for user %s", userInfo.Username) - } else if request.Method == "POST" { - log.Printf("Handling post!") - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Println("Failed reading body") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Missing field: user_id"}`))) - return - } - - type userId struct { - UserId string `json:"user_id"` - } - - var t userId - err = json.Unmarshal(body, &t) - if err != nil { - log.Printf("Failed unmarshaling userId: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unmarshaling. Missing field: user_id"}`))) - return - } - - if userInfo.Role != "admin" { - log.Printf("%s tried and failed to change apikey for %s", userInfo.Username, t.UserId) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "You need to be admin to change others' apikey"}`))) - return - } - - foundUser, err := shuffle.GetUser(ctx, t.UserId) - if err != nil { - log.Printf("Can't find user %s (apikey gen): %s", t.UserId, err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false}`))) - return - } - - newUserInfo, err := shuffle.GenerateApikey(ctx, *foundUser) - if err != nil { - log.Printf("Failed to generate apikey for user %s: %s", foundUser.Username, err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return - } - foundUser = &newUserInfo - - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "username": "%s", "verified": %t, "apikey": "%s"}`, foundUser.Username, foundUser.Verified, foundUser.ApiKey))) - return - } - - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "username": "%s", "verified": %t, "apikey": "%s"}`, userInfo.Username, userInfo.Verified, userInfo.ApiKey))) -} - -func handleSettings(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - userInfo, err := shuffle.HandleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in apigen: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "username": "%s", "verified": %t, "apikey": "%s"}`, userInfo.Username, userInfo.Verified, userInfo.ApiKey))) -} - func handleInfo(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -1645,13 +1416,6 @@ type passwordReset struct { Reference string `json:"reference"` } -type passwordChange struct { - Username string `json:"username"` - Newpassword string `json:"newpassword"` - Newpassword2 string `json:"newpassword2"` - Currentpassword string `json:"currentpassword"` -} - func handlePasswordReset(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -1740,159 +1504,6 @@ func handlePasswordReset(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, defaultMessage))) } -func handlePasswordChange(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - log.Println("Handling password change") - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Println("Failed reading body") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false}`))) - return - } - - // Get the current user - check if they're admin or the "username" user. - var t passwordChange - err = json.Unmarshal(body, &t) - if err != nil { - log.Println("Failed unmarshaling") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false}`))) - return - } - - userInfo, err := shuffle.HandleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in set new workflowhandler: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - curUserFound := false - if t.Username != userInfo.Username && userInfo.Role != "admin" { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Admin required to change others' passwords"}`)) - return - } else if t.Username == userInfo.Username { - curUserFound = true - } - - if userInfo.Role != "admin" { - if t.Newpassword != t.Newpassword2 { - err := "Passwords don't match" - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return - } - - if len(t.Newpassword) < 10 || len(t.Newpassword2) < 10 { - err := "Passwords too short - 2" - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return - } - } else { - // Check ORG HERE? - } - - // Current password - err = checkPasswordStrength(t.Newpassword) - if err != nil { - log.Printf("Bad password strength: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return - } - - ctx := context.Background() - foundUser := shuffle.User{} - if !curUserFound { - log.Printf("Have to find a different user") - q := datastore.NewQuery("Users").Filter("Username =", strings.ToLower(t.Username)) - var users []shuffle.User - _, err = dbclient.GetAll(ctx, q, &users) - if err != nil { - log.Printf("Failed getting user %s", t.Username) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) - return - } - - if len(users) != 1 { - log.Printf(`Found multiple or no users with the same username: %s: %d`, t.Username, len(users)) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Found %d users with the same username: %s"}`, len(users), t.Username))) - return - } - - foundUser = users[0] - orgFound := false - if userInfo.ActiveOrg.Id == foundUser.ActiveOrg.Id { - orgFound = true - } else { - log.Printf("FoundUser: %#v", foundUser.Orgs) - for _, item := range foundUser.Orgs { - if item == userInfo.ActiveOrg.Id { - orgFound = true - break - } - } - } - - if !orgFound { - log.Printf("User %s is admin, but can't change user's passowrd outside their own org.", userInfo.Id) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't change users outside your org."}`))) - return - } - } else { - // Admins can re-generate others' passwords as well. - if userInfo.Role != "admin" { - err = bcrypt.CompareHashAndPassword([]byte(userInfo.Password), []byte(t.Newpassword)) - if err != nil { - log.Printf("Bad password for %s: %s", userInfo.Username, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) - return - } - } - } - - if len(foundUser.Id) == 0 { - log.Printf("Something went wrong in password reset: couldn't find user.") - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false}`)) - return - } - - hashedPassword, err := bcrypt.GenerateFromPassword([]byte(t.Newpassword), 8) - if err != nil { - log.Printf("New password failure for %s: %s", userInfo.Username, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) - return - } - - userInfo.Password = string(hashedPassword) - err = shuffle.SetUser(ctx, &foundUser) - if err != nil { - log.Printf("Error fixing password for user %s: %s", userInfo.Username, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) - return - } - - //memcache.Delete(ctx, sessionToken) - - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) -} - // FIXME - forward this to emails or whatever CRM system in use func handleContact(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) @@ -2162,65 +1773,6 @@ func handleGetOrgs(resp http.ResponseWriter, request *http.Request) { resp.Write(newjson) } -func handleGetUsers(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - user, err := shuffle.HandleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in set new workflowhandler: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if user.Role != "admin" { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Not admin"}`)) - return - } - - // FIXME: Check by org. - ctx := context.Background() - org, err := shuffle.GetOrg(ctx, user.ActiveOrg.Id) - if err != nil { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Failed getting org users"}`)) - return - } - - newUsers := []shuffle.User{} - for _, item := range org.Users { - if len(item.Username) == 0 { - continue - } - - //for _, tmpUser := range newUsers { - // if tmpUser.Name - //} - - item.Password = "" - item.Session = "" - item.VerificationToken = "" - item.Orgs = []string{} - - newUsers = append(newUsers, item) - } - - newjson, err := json.Marshal(newUsers) - if err != nil { - log.Printf("Failed unmarshal: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking"}`))) - return - } - - resp.WriteHeader(200) - resp.Write(newjson) -} - func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -2307,7 +1859,7 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) { // FIXME - have timeout here loginData := `{"success": true}` if len(Userdata.Session) != 0 { - log.Println("[INFO] User session exists - resetting") + log.Println("[INFO] User session already exists - resetting it") expiration := time.Now().Add(3600 * time.Second) http.SetCookie(resp, &http.Cookie{ @@ -3350,7 +2902,7 @@ func executeCloudAction(action CloudSyncJob, apikey string) error { // Starts a new webhook func handleNewHook(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -3506,137 +3058,6 @@ func handleNewHook(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(`{"success": true}`)) } -func sendHookResult(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - user, err := shuffle.HandleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in set new workflowhandler: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - _ = user - - location := strings.Split(request.URL.String(), "/") - - var workflowId string - if location[1] == "api" { - if len(location) <= 4 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - workflowId = location[4] - } - - if len(workflowId) != 32 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "message": "ID not valid"}`)) - return - } - - ctx := context.Background() - hook, err := getHook(ctx, workflowId) - if err != nil { - log.Printf("Failed getting hook %s (send): %s", workflowId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Printf("Body data error: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - log.Printf("SET the hook results for %s to %s", workflowId, body) - // FIXME - set the hook result in the DB somehow as interface{} - // FIXME - should the hook do the transform? Hmm - - b, err := json.Marshal(hook) - if err != nil { - log.Printf("Failed marshalling: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - resp.WriteHeader(200) - resp.Write([]byte(b)) - return -} - -func handleGetHook(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - user, err := shuffle.HandleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in set new workflowhandler: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - location := strings.Split(request.URL.String(), "/") - - var workflowId string - if location[1] == "api" { - if len(location) <= 4 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - workflowId = location[4] - } - - if len(workflowId) != 36 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "message": "ID not valid"}`)) - return - } - - ctx := context.Background() - hook, err := getHook(ctx, workflowId) - if err != nil { - log.Printf("Failed getting hook %s (get hook): %s", workflowId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if user.Id != hook.Owner && user.Role != "admin" && user.Role != "scheduler" { - log.Printf("Wrong user (%s) for hook %s", user.Username, hook.Id) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - b, err := json.Marshal(hook) - if err != nil { - log.Printf("Failed marshalling: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - // FIXME - get some real data? - resp.WriteHeader(200) - resp.Write([]byte(b)) - return -} - func getSpecificSchedule(resp http.ResponseWriter, request *http.Request) { if request.Method != "GET" { setSpecificSchedule(resp, request) @@ -5465,9 +4886,9 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { } cacheKey := fmt.Sprintf("workflowapps-sorted-100") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) cacheKey = fmt.Sprintf("workflowapps-sorted-500") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s"}`, api.ID))) @@ -5533,7 +4954,7 @@ func createFs(basepath, pathname string) (billy.Filesystem, error) { } // Hotloads new apps from a folder -func handleAppHotload(location string, forceUpdate bool) error { +func handleAppHotload(ctx context.Context, location string, forceUpdate bool) error { basepath := "base" fs, err := createFs(basepath, location) @@ -5557,12 +4978,12 @@ func handleAppHotload(location string, forceUpdate bool) error { return err } - cacheKey := fmt.Sprintf("workflowapps-sorted-100") - requestCache.Delete(cacheKey) + cacheKey := fmt.Sprintf("workflowapps-sorted") + shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-100") + shuffle.DeleteCache(ctx, cacheKey) cacheKey = fmt.Sprintf("workflowapps-sorted-500") - requestCache.Delete(cacheKey) - cacheKey = fmt.Sprintf("workflowapps-sorted") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) return nil } @@ -5731,7 +5152,7 @@ func handleCloudJob(job CloudSyncJob) error { log.Printf("Should handle user_input CONTINUE for workflow %s with start node %s and execution ID %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem) // FIXME: Handle authorization ctx := context.Background() - workflowExecution, err := getWorkflowExecution(ctx, job.ThirdItem) + workflowExecution, err := shuffle.GetWorkflowExecution(ctx, job.ThirdItem) if err != nil { return err } @@ -5741,7 +5162,7 @@ func handleCloudJob(job CloudSyncJob) error { } workflowExecution.Status = "EXECUTING" - err = setWorkflowExecution(ctx, *workflowExecution, true) + err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, true) if err != nil { return err } @@ -5767,7 +5188,7 @@ func handleCloudJob(job CloudSyncJob) error { } else if job.Action == "stop" { log.Printf("Should handle user_input STOP for workflow %s with start node %s and execution ID %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem) ctx := context.Background() - workflowExecution, err := getWorkflowExecution(ctx, job.ThirdItem) + workflowExecution, err := shuffle.GetWorkflowExecution(ctx, job.ThirdItem) if err != nil { return err } @@ -5794,7 +5215,7 @@ func handleCloudJob(job CloudSyncJob) error { workflowExecution.Results = newResults workflowExecution.Status = "ABORTED" - err = setWorkflowExecution(ctx, *workflowExecution, true) + err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, true) if err != nil { return err } @@ -5925,7 +5346,7 @@ func runInit(ctx context.Context) { log.Printf("Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy) } - requestCache = cache.New(5*time.Minute, 10*time.Minute) + //requestCache = cache.New(5*time.Minute, 10*time.Minute) /* proxyUrl, err := url.Parse(httpProxy) @@ -6201,7 +5622,7 @@ func runInit(ctx context.Context) { } if setLocal { - err = setWorkflow(ctx, workflow, workflow.ID) + err = shuffle.SetWorkflow(ctx, workflow, workflow.ID) if err != nil { log.Printf("Failed setting workflow in init: %s", err) } else { @@ -6444,7 +5865,7 @@ func runInit(ctx context.Context) { // Hotloads locally location := os.Getenv("SHUFFLE_APP_HOTLOAD_FOLDER") if len(location) != 0 { - handleAppHotload(location, false) + handleAppHotload(ctx, location, false) } } @@ -6713,7 +6134,7 @@ func handleKeyValueCheck(resp http.ResponseWriter, request *http.Request) { return } - workflowExecution, err := getWorkflowExecution(ctx, tmpData.ExecutionRef) + workflowExecution, err := shuffle.GetWorkflowExecution(ctx, tmpData.ExecutionRef) if err != nil { log.Printf("[INFO] User can't edit the org") resp.WriteHeader(401) @@ -7467,7 +6888,7 @@ func initHandlers() { ctx := context.Background() log.Printf("Starting Shuffle backend - initializing database connection") - requestCache = cache.New(5*time.Minute, 10*time.Minute) + //requestCache = cache.New(5*time.Minute, 10*time.Minute) dbclient, err = datastore.NewClient(ctx, gceProject, option.WithGRPCDialOption(grpc.WithNoProxy())) if err != nil { panic(fmt.Sprintf("DBclient error during init: %s", err)) @@ -7488,29 +6909,30 @@ func initHandlers() { // Make user related locations // Fix user changes with org - r.HandleFunc("/api/v1/users/generateapikey", handleApiGeneration).Methods("GET", "POST", "OPTIONS") r.HandleFunc("/api/v1/users/login", handleLogin).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/users/logout", handleLogout).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/users/register", handleRegister).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/users/checkusers", checkAdminLogin).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/users/getinfo", handleInfo).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/users/getsettings", handleSettings).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/users/getusers", handleGetUsers).Methods("GET", "OPTIONS") + + r.HandleFunc("/api/v1/users/generateapikey", shuffle.HandleApiGeneration).Methods("GET", "POST", "OPTIONS") + r.HandleFunc("/api/v1/users/logout", shuffle.HandleLogout).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/users/getsettings", shuffle.HandleSettings).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/users/getusers", shuffle.HandleGetUsers).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/users/updateuser", handleUpdateUser).Methods("PUT", "OPTIONS") r.HandleFunc("/api/v1/users/{user}", deleteUser).Methods("DELETE", "OPTIONS") - r.HandleFunc("/api/v1/users/passwordchange", handlePasswordChange).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/users", handleGetUsers).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/users/passwordchange", shuffle.HandlePasswordChange).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/users", shuffle.HandleGetUsers).Methods("GET", "OPTIONS") // General - duplicates and old. + r.HandleFunc("/api/v1/getusers", shuffle.HandleGetUsers).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/login", handleLogin).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/logout", handleLogout).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/logout", shuffle.HandleLogout).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/register", handleRegister).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/checkusers", checkAdminLogin).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/getusers", handleGetUsers).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/getinfo", handleInfo).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/getsettings", handleSettings).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/generateapikey", handleApiGeneration).Methods("GET", "POST", "OPTIONS") - r.HandleFunc("/api/v1/passwordchange", handlePasswordChange).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/getsettings", shuffle.HandleSettings).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/generateapikey", shuffle.HandleApiGeneration).Methods("GET", "POST", "OPTIONS") + r.HandleFunc("/api/v1/passwordchange", shuffle.HandlePasswordChange).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/getenvironments", handleGetEnvironments).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/setenvironments", handleSetEnvironments).Methods("PUT", "OPTIONS") @@ -7542,11 +6964,11 @@ func initHandlers() { r.HandleFunc("/api/v1/apps", setNewWorkflowApp).Methods("PUT", "OPTIONS") r.HandleFunc("/api/v1/apps/search", getSpecificApps).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/apps/authentication", getAppAuthentication).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/apps/authentication", addAppAuthentication).Methods("PUT", "OPTIONS") - r.HandleFunc("/api/v1/apps/authentication/{appauthId}/config", setAuthenticationConfig).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/apps/authentication", shuffle.GetAppAuthentication).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/apps/authentication", shuffle.AddAppAuthentication).Methods("PUT", "OPTIONS") + r.HandleFunc("/api/v1/apps/authentication/{appauthId}/config", shuffle.SetAuthenticationConfig).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/apps/authentication/{appauthId}", deleteAppAuthentication).Methods("DELETE", "OPTIONS") + r.HandleFunc("/api/v1/apps/authentication/{appauthId}", shuffle.DeleteAppAuthentication).Methods("DELETE", "OPTIONS") // Legacy app things r.HandleFunc("/api/v1/workflows/apps/validate", validateAppInput).Methods("POST", "OPTIONS") @@ -7556,8 +6978,8 @@ func initHandlers() { // Workflows // FIXME - implement the queue counter lol /* Everything below here increases the counters*/ - r.HandleFunc("/api/v1/workflows", getWorkflows).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/workflows", setNewWorkflow).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/workflows", shuffle.GetWorkflows).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/workflows", shuffle.SetNewWorkflow).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/schedules", handleGetSchedules).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/schedule", scheduleWorkflow).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/download_remote", loadSpecificWorkflows).Methods("POST", "OPTIONS") @@ -7567,8 +6989,8 @@ func initHandlers() { r.HandleFunc("/api/v1/workflows/{key}/outlook/{triggerId}", handleDeleteOutlookSub).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/executions", getWorkflowExecutions).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/executions/{key}/abort", abortExecution).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/workflows/{key}", getSpecificWorkflow).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/workflows/{key}", saveWorkflow).Methods("PUT", "OPTIONS") + r.HandleFunc("/api/v1/workflows/{key}", shuffle.GetSpecificWorkflow).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/workflows/{key}", shuffle.SaveWorkflow).Methods("PUT", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}", deleteWorkflow).Methods("DELETE", "OPTIONS") // Triggers diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 10b96f19..f20d15ee 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -44,7 +44,6 @@ import ( //"cloud.google.com/go/firestore" // "google.golang.org/api/option" - "github.com/patrickmn/go-cache" "google.golang.org/api/iterator" ) @@ -992,7 +991,7 @@ func validateNewWorkerExecution(body []byte) error { //} //log.Printf("\n\nSHOULD SET BACKEND DATA FOR EXEC \n\n") - err = setWorkflowExecution(ctx, execution, true) + err = shuffle.SetWorkflowExecution(ctx, execution, true) if err == nil { log.Printf("[INFO] Set workflowexecution based on new worker (>0.8.53) for execution %s. Actions: %d, Triggers: %d, Results: %d", execution.ExecutionId, len(execution.Workflow.Actions), len(execution.Workflow.Triggers), len(execution.Results)) //log.Printf("[INFO] Successfully set the execution to wait.") @@ -1102,7 +1101,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { actionResult.Result = fmt.Sprintf("Cloud error: %s", err) workflowExecution.Results = append(workflowExecution.Results, actionResult) workflowExecution.Status = "ABORTED" - err = setWorkflowExecution(ctx, *workflowExecution, true) + err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, true) if err != nil { log.Printf("Failed to set execution during wait") } else { @@ -1120,7 +1119,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { workflowExecution.Results = append(workflowExecution.Results, actionResult) workflowExecution.Status = actionResult.Status - err = setWorkflowExecution(ctx, *workflowExecution, true) + err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, true) if err != nil { log.Printf("Failed ") } else { @@ -1144,7 +1143,8 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting execution"}`))) return } - resultLength := len(workflowExecution.Results) + + //resultLength := len(workflowExecution.Results) dbSave := false setExecution := true @@ -1488,10 +1488,12 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // Validating that action results hasn't changed // Handled using cachhing, so actually pretty fast cacheKey := fmt.Sprintf("workflowexecution-%s", workflowExecution.ExecutionId) - if value, found := requestCache.Get(cacheKey); found { - parsedValue := value.(*shuffle.WorkflowExecution) - if len(parsedValue.Results) > 0 && len(parsedValue.Results) != resultLength { - setExecution = false + cache, err := shuffle.GetCache(ctx, cacheKey) + if err == nil { + cacheData := []byte(cache.([]uint8)) + //log.Printf("CACHEDATA: %#v", cacheData) + err = json.Unmarshal(cacheData, &workflowExecution) + if err == nil { if attempts > 5 { //log.Printf("\n\nSkipping execution input - %d vs %d. Attempts: (%d)\n\n", len(parsedValue.Results), resultLength, attempts) } @@ -1504,8 +1506,24 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } } + //if value, found := requestCache.Get(cacheKey); found { + // parsedValue := value.(*shuffle.WorkflowExecution) + // if len(parsedValue.Results) > 0 && len(parsedValue.Results) != resultLength { + // setExecution = false + // if attempts > 5 { + // //log.Printf("\n\nSkipping execution input - %d vs %d. Attempts: (%d)\n\n", len(parsedValue.Results), resultLength, attempts) + // } + + // attempts += 1 + // if len(workflowExecution.Results) <= len(workflowExecution.Workflow.Actions) { + // runWorkflowExecutionTransaction(ctx, attempts, workflowExecutionId, actionResult, resp) + // return + // } + // } + //} + if setExecution || workflowExecution.Status == "FINISHED" || workflowExecution.Status == "ABORTED" || workflowExecution.Status == "FAILURE" { - err = setWorkflowExecution(ctx, *workflowExecution, dbSave) + err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, dbSave) if err != nil { resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting workflowexecution actionresult: %s"}`, err))) @@ -1540,7 +1558,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // log.Printf("[ERROR] QUITTING: tx.Commit %d: %v", attempts, err) // workflowExecution.Status = "ABORTED" - // setWorkflowExecution(ctx, *workflowExecution, true) + // shuffle.SetWorkflowExecution(ctx, *workflowExecution, true) // resp.WriteHeader(401) // resp.Write([]byte(`{"success": false}`)) @@ -1741,233 +1759,6 @@ func getWorkflows(resp http.ResponseWriter, request *http.Request) { resp.Write(newjson) } -// FIXME - add to actual database etc -func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - user, err := shuffle.HandleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in set new workflowhandler: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Printf("Error with body read: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - var workflow shuffle.Workflow - err = json.Unmarshal(body, &workflow) - if err != nil { - log.Printf("Failed unmarshaling: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - workflow.ID = uuid.NewV4().String() - workflow.Owner = user.Id - workflow.Sharing = "private" - user.ActiveOrg.Users = []shuffle.User{} - workflow.ExecutingOrg = user.ActiveOrg - workflow.OrgId = user.ActiveOrg.Id - //log.Printf("TRIGGERS: %d", len(workflow.Triggers)) - - ctx := context.Background() - //err = increaseStatisticsField(ctx, "total_workflows", workflow.ID, 1, workflow.OrgId) - //if err != nil { - // log.Printf("Failed to increase total workflows stats: %s", err) - //} - - if len(workflow.Actions) == 0 { - workflow.Actions = []shuffle.Action{} - } - if len(workflow.Branches) == 0 { - workflow.Branches = []shuffle.Branch{} - } - if len(workflow.Triggers) == 0 { - workflow.Triggers = []shuffle.Trigger{} - } - if len(workflow.Errors) == 0 { - workflow.Errors = []string{} - } - - newActions := []shuffle.Action{} - for _, action := range workflow.Actions { - if action.Environment == "" { - //action.Environment = baseEnvironment - action.IsValid = true - } - - //action.LargeImage = "" - newActions = append(newActions, action) - } - - // Initialized without functions = adding a hello world node. - if len(newActions) == 0 { - //log.Printf("APPENDING NEW APP FOR NEW WORKFLOW") - - // Adds the Testing app if it's a new workflow - workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500) - if err == nil { - // FIXME: Add real env - envName := "Shuffle" - environments, err := shuffle.GetEnvironments(ctx, user.ActiveOrg.Id) - if err == nil { - for _, env := range environments { - if env.Default { - envName = env.Name - break - } - } - } - - for _, item := range workflowapps { - if item.Name == "Testing" && item.AppVersion == "1.0.0" { - nodeId := "40447f30-fa44-4a4f-a133-4ee710368737" - workflow.Start = nodeId - newActions = append(newActions, shuffle.Action{ - Label: "Start node", - Name: "hello_world", - Environment: envName, - Parameters: []shuffle.WorkflowAppActionParameter{}, - Position: struct { - X float64 "json:\"x,omitempty\" datastore:\"x\"" - Y float64 "json:\"y,omitempty\" datastore:\"y\"" - }{X: 449.5, Y: 446}, - Priority: 0, - Errors: []string{}, - ID: nodeId, - IsValid: true, - IsStartNode: true, - Sharing: true, - PrivateID: "", - SmallImage: "", - AppName: item.Name, - AppVersion: item.AppVersion, - AppID: item.ID, - LargeImage: item.LargeImage, - }) - break - } - } - } - } else { - log.Printf("[INFO] Has %d actions already", len(newActions)) - // FIXME: Check if they require authentication and if they exist locally - //log.Printf("\n\nSHOULD VALIDATE AUTHENTICATION") - //AuthenticationId string `json:"authentication_id,omitempty" datastore:"authentication_id"` - //allAuths, err := shuffle.GetAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) - //if err == nil { - // log.Printf("AUTH: %#v", allAuths) - // for _, action := range newActions { - // log.Printf("ACTION: %#v", action) - // } - //} - } - - workflow.Actions = []shuffle.Action{} - for _, item := range workflow.Actions { - oldId := item.ID - sourceIndexes := []int{} - destinationIndexes := []int{} - for branchIndex, branch := range workflow.Branches { - if branch.SourceID == oldId { - sourceIndexes = append(sourceIndexes, branchIndex) - } - - if branch.DestinationID == oldId { - destinationIndexes = append(destinationIndexes, branchIndex) - } - } - - item.ID = uuid.NewV4().String() - for _, index := range sourceIndexes { - workflow.Branches[index].SourceID = item.ID - } - - for _, index := range destinationIndexes { - workflow.Branches[index].DestinationID = item.ID - } - - newActions = append(newActions, item) - } - - newTriggers := []shuffle.Trigger{} - for _, item := range workflow.Triggers { - oldId := item.ID - sourceIndexes := []int{} - destinationIndexes := []int{} - for branchIndex, branch := range workflow.Branches { - if branch.SourceID == oldId { - sourceIndexes = append(sourceIndexes, branchIndex) - } - - if branch.DestinationID == oldId { - destinationIndexes = append(destinationIndexes, branchIndex) - } - } - - item.ID = uuid.NewV4().String() - for _, index := range sourceIndexes { - workflow.Branches[index].SourceID = item.ID - } - - for _, index := range destinationIndexes { - workflow.Branches[index].DestinationID = item.ID - } - - item.Status = "uninitialized" - newTriggers = append(newTriggers, item) - } - - newSchedules := []shuffle.Schedule{} - for _, item := range workflow.Schedules { - item.Id = uuid.NewV4().String() - newSchedules = append(newSchedules, item) - } - - timeNow := int64(time.Now().Unix()) - workflow.Actions = newActions - workflow.Triggers = newTriggers - workflow.Schedules = newSchedules - workflow.IsValid = true - workflow.Configuration.ExitOnError = false - workflow.Created = timeNow - - workflowjson, err := json.Marshal(workflow) - if err != nil { - log.Printf("Failed workflow json setting marshalling: %s", err) - resp.WriteHeader(http.StatusInternalServerError) - resp.Write([]byte(`{"success": false}`)) - return - } - - err = setWorkflow(ctx, workflow, workflow.ID) - if err != nil { - log.Printf("Failed setting workflow: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - log.Printf("[INFO] Saved new workflow %s with name %s", workflow.ID, workflow.Name) - //memcacheName := fmt.Sprintf("%s_workflows", user.Username) - //memcache.Delete(ctx, memcacheName) - - resp.WriteHeader(200) - //log.Println(string(workflowjson)) - resp.Write(workflowjson) -} - func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -2066,875 +1857,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(`{"success": true}`)) } -// Adds app auth tracking -func updateAppAuth(auth shuffle.AppAuthenticationStorage, workflowId, nodeId string, add bool) error { - workflowFound := false - workflowIndex := 0 - nodeFound := false - for index, workflow := range auth.Usage { - if workflow.WorkflowId == workflowId { - // Check if node exists - workflowFound = true - workflowIndex = index - for _, actionId := range workflow.Nodes { - if actionId == nodeId { - nodeFound = true - break - } - } - - break - } - } - - // FIXME: Add a way to use !add to remove - updateAuth := false - if !workflowFound && add { - log.Printf("[INFO] Adding workflow things to auth!") - usageItem := shuffle.AuthenticationUsage{ - WorkflowId: workflowId, - Nodes: []string{nodeId}, - } - - auth.Usage = append(auth.Usage, usageItem) - auth.WorkflowCount += 1 - auth.NodeCount += 1 - updateAuth = true - } else if !nodeFound && add { - log.Printf("[INFO] Adding node things to auth!") - auth.Usage[workflowIndex].Nodes = append(auth.Usage[workflowIndex].Nodes, nodeId) - auth.NodeCount += 1 - updateAuth = true - } - - if updateAuth { - log.Printf("[INFO] Updating auth!") - ctx := context.Background() - err := shuffle.SetWorkflowAppAuthDatastore(ctx, auth, auth.Id) - if err != nil { - log.Printf("Failed setting up app auth %s: %s", auth.Id, err) - return err - } - } - - return nil -} - // Identifies what a category defined really is -func handleCategoryIncrease(categories shuffle.Categories, action shuffle.Action, workflowapps []shuffle.WorkflowApp) shuffle.Categories { - if action.Category == "" { - appName := action.AppName - for _, app := range workflowapps { - if appName != strings.ToLower(app.Name) { - continue - } - - if len(app.Categories) > 0 { - log.Printf("[INFO] Setting category for %s: %s", app.Name, app.Categories) - action.Category = app.Categories[0] - break - } - } - - //log.Printf("Should find app's categories as it's empty during save") - return categories - } - - //log.Printf("Action: %s, category: %s", action.AppName, action.Category) - // FIXME: Make this an "autodiscover" that's controlled by the category itself - // Should just be a list that's looped against :) - newCategory := strings.ToLower(action.Category) - if strings.Contains(newCategory, "case") || strings.Contains(newCategory, "ticket") || strings.Contains(newCategory, "alert") || strings.Contains(newCategory, "mssp") { - categories.Cases.Count += 1 - } else if strings.Contains(newCategory, "siem") || strings.Contains(newCategory, "event") || strings.Contains(newCategory, "log") || strings.Contains(newCategory, "search") { - categories.SIEM.Count += 1 - } else if strings.Contains(newCategory, "sms") || strings.Contains(newCategory, "comm") || strings.Contains(newCategory, "phone") || strings.Contains(newCategory, "call") || strings.Contains(newCategory, "chat") || strings.Contains(newCategory, "mail") || strings.Contains(newCategory, "phish") { - categories.Communication.Count += 1 - } else if strings.Contains(newCategory, "intel") || strings.Contains(newCategory, "crim") || strings.Contains(newCategory, "ti") { - categories.Intel.Count += 1 - } else if strings.Contains(newCategory, "sand") || strings.Contains(newCategory, "virus") || strings.Contains(newCategory, "malware") || strings.Contains(newCategory, "scan") || strings.Contains(newCategory, "edr") || strings.Contains(newCategory, "endpoint detection") { - // Sandbox lol - categories.EDR.Count += 1 - } else if strings.Contains(newCategory, "vuln") || strings.Contains(newCategory, "fim") || strings.Contains(newCategory, "fim") || strings.Contains(newCategory, "integrity") { - categories.Assets.Count += 1 - } else if strings.Contains(newCategory, "network") || strings.Contains(newCategory, "firewall") || strings.Contains(newCategory, "waf") || strings.Contains(newCategory, "switch") { - categories.Network.Count += 1 - } else { - categories.Other.Count += 1 - } - - return categories -} - -// Saves a workflow to an ID -func saveWorkflow(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - //log.Println("Start") - user, userErr := shuffle.HandleApiAuthentication(resp, request) - if userErr != nil { - log.Printf("Api authentication failed in edit workflow: %s", userErr) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - //log.Println("PostUser") - location := strings.Split(request.URL.String(), "/") - - var fileId string - if location[1] == "api" { - if len(location) <= 4 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - if len(fileId) != 36 { - log.Printf(`ID %s is not valid`, fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Workflow ID to save is not valid"}`)) - return - } - - // Here to check access rights - ctx := context.Background() - tmpworkflow, err := shuffle.GetWorkflow(ctx, fileId) - if err != nil { - log.Printf("Failed getting the workflow locally (save workflow): %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - // FIXME - have a check for org etc too.. - if user.Id != tmpworkflow.Owner && user.Role != "admin" { - log.Printf("Wrong user (%s) for workflow %s (save)", user.Username, tmpworkflow.ID) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - //log.Printf("PRE BODY") - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Printf("Failed hook unmarshaling: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - var workflow shuffle.Workflow - err = json.Unmarshal([]byte(body), &workflow) - //log.Printf(string(body)) - if err != nil { - log.Printf(string(body)) - log.Printf("[ERROR] Failed workflow unmarshaling: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return - } - - //log.Printf("SAVED: %#v", workflow.PreviouslySaved) - - // FIXME - auth and check if they should have access - if fileId != workflow.ID { - log.Printf("Path and request ID are not matching: %s:%s.", fileId, workflow.ID) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - // Fixing wrong owners when importing - if workflow.Owner == "" { - workflow.Owner = user.Id - } - - if len(workflow.ExecutingOrg.Id) == 0 { - log.Printf("[INFO] Setting executing org for workflow") - user.ActiveOrg.Users = []shuffle.User{} - workflow.ExecutingOrg = user.ActiveOrg - } - - // FIXME - this shouldn't be necessary with proper API checks - newActions := []shuffle.Action{} - allNodes := []string{} - workflow.Categories = shuffle.Categories{} - - //log.Printf("PRE APPS") - workflowapps, apperr := shuffle.GetAllWorkflowApps(ctx, 500) - - //log.Printf("Action: %#v", action.Authentication) - for _, action := range workflow.Actions { - allNodes = append(allNodes, action.ID) - - if len(action.Errors) > 0 || !action.IsValid { - action.IsValid = true - action.Errors = []string{} - } - - if action.Environment == "" { - if workflow.PreviouslySaved { - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "An environment for %s is required"}`, action.Label))) - return - } - action.IsValid = true - } - - // FIXME: Have a good way of tracking errors. ID's or similar. - if !action.IsValid && len(action.Errors) > 0 { - log.Printf("Node %s is invalid and needs to be remade. Errors: %s", action.Label, strings.Join(action.Errors, "\n")) - - if workflow.PreviouslySaved { - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Node %s is invalid and needs to be remade."}`, action.Label))) - return - } - action.IsValid = true - action.Errors = []string{} - } - - workflow.Categories = handleCategoryIncrease(workflow.Categories, action, workflowapps) - newActions = append(newActions, action) - } - - newTriggers := []shuffle.Trigger{} - for _, trigger := range workflow.Triggers { - log.Printf("[INFO] Trigger %s: %s", trigger.TriggerType, trigger.Status) - - // Check if it's actually running - // FIXME: Do this for other triggers too - if trigger.TriggerType == "SCHEDULE" && trigger.Status != "uninitialized" { - schedule, err := shuffle.GetSchedule(ctx, trigger.ID) - if err != nil { - trigger.Status = "stopped" - } else if schedule.Id == "" { - trigger.Status = "stopped" - } - } else if trigger.TriggerType == "SUBFLOW" { - for index, param := range trigger.Parameters { - if len(param.Value) == 0 && param.Name != "argument" { - //log.Printf("Param: %#v", param) - if param.Name == "user_apikey" { - apikey := "" - if len(user.ApiKey) > 0 { - apikey = user.ApiKey - } else { - user, err = shuffle.GenerateApikey(ctx, user) - if err != nil { - workflow.IsValid = false - workflow.Errors = []string{"Trigger is missing a parameter: %s", param.Name} - - log.Printf("No type specified for user input node") - - if workflow.PreviouslySaved { - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Trigger %s is missing the parameter %s"}`, trigger.Label, param.Name))) - return - } - } - - apikey = user.ApiKey - } - - log.Printf("[INFO] Set apikey in subflow trigger for user during save") - trigger.Parameters[index].Value = apikey - } else { - - workflow.IsValid = false - workflow.Errors = []string{"Trigger is missing a parameter: %s", param.Name} - - log.Printf("No type specified for user input node") - if workflow.PreviouslySaved { - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Trigger %s is missing the parameter %s"}`, trigger.Label, param.Name))) - return - } - } - } - } - } else if trigger.TriggerType == "WEBHOOK" && trigger.Status != "uninitialized" { - hook, err := getHook(ctx, trigger.ID) - if err != nil { - log.Printf("Failed getting webhook") - trigger.Status = "stopped" - } else if hook.Id == "" { - trigger.Status = "stopped" - } - } else if trigger.TriggerType == "USERINPUT" { - // E.g. check email - sms := "" - email := "" - triggerType := "" - triggerInformation := "" - for _, item := range trigger.Parameters { - if item.Name == "alertinfo" { - triggerInformation = item.Value - } else if item.Name == "type" { - triggerType = item.Value - } else if item.Name == "email" { - email = item.Value - } else if item.Name == "sms" { - sms = item.Value - } - } - - if len(triggerType) == 0 { - log.Printf("No type specified for user input node") - if workflow.PreviouslySaved { - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No contact option specified in user input"}`))) - return - } - } - - // FIXME: This is not the right time to send them, BUT it's well served for testing. Save -> send email / sms - _ = triggerInformation - if strings.Contains(triggerType, "email") { - if email == "test@test.com" { - log.Printf("Email isn't specified during save.") - if workflow.PreviouslySaved { - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Email field in user input can't be empty"}`))) - return - } - } - - log.Printf("Should send email to %s during execution.", email) - } - if strings.Contains(triggerType, "sms") { - if sms == "0000000" { - log.Printf("Email isn't specified during save.") - if workflow.PreviouslySaved { - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "SMS field in user input can't be empty"}`))) - return - } - } - - log.Printf("Should send SMS to %s during execution.", sms) - } - } - - //log.Println("TRIGGERS") - allNodes = append(allNodes, trigger.ID) - newTriggers = append(newTriggers, trigger) - } - - workflow.Triggers = newTriggers - - if len(workflow.Actions) == 0 { - workflow.Actions = []shuffle.Action{} - } - if len(workflow.Branches) == 0 { - workflow.Branches = []shuffle.Branch{} - } - if len(workflow.Triggers) == 0 { - workflow.Triggers = []shuffle.Trigger{} - } - if len(workflow.Errors) == 0 { - workflow.Errors = []string{} - } - - //log.Printf("PRE VARIABLES") - for _, variable := range workflow.WorkflowVariables { - if len(variable.Value) == 0 { - log.Printf("[WARNING] Variable %s is empty!", variable.Name) - workflow.Errors = append(workflow.Errors, fmt.Sprintf("Variable %s is empty!", variable.Name)) - //resp.WriteHeader(401) - //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Variable %s can't be empty"}`, variable.Name))) - //return - } - } - - if len(workflow.ExecutionVariables) > 0 { - log.Printf("[INFO] Found %d execution variable(s)", len(workflow.ExecutionVariables)) - } - - if len(workflow.WorkflowVariables) > 0 { - log.Printf("[INFO] Found %d workflow variable(s)", len(workflow.WorkflowVariables)) - } - - // FIXME - do actual checks ROFL - // FIXME - minor issues with e.g. hello world and self.console_logger - // Nodechecks - foundNodes := []string{} - for _, node := range allNodes { - for _, branch := range workflow.Branches { - //log.Println("branch") - //log.Println(node) - //log.Println(branch.DestinationID) - if node == branch.DestinationID || node == branch.SourceID { - foundNodes = append(foundNodes, node) - break - } - } - } - - // FIXME - append all nodes (actions, triggers etc) to one single array here - //log.Printf("PRE VARIABLES") - if len(foundNodes) != len(allNodes) || len(workflow.Actions) <= 0 { - // This shit takes a few seconds lol - if !workflow.IsValid { - oldworkflow, err := shuffle.GetWorkflow(ctx, fileId) - if err != nil { - log.Printf("Workflow %s doesn't exist - oldworkflow.", fileId) - if workflow.PreviouslySaved { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Item already exists."}`)) - return - } - } - - oldworkflow.IsValid = false - err = setWorkflow(ctx, *oldworkflow, fileId) - if err != nil { - log.Printf("Failed saving workflow to database: %s", err) - if workflow.PreviouslySaved { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - } - } - - // FIXME - more checks here - force reload of data or something - //if len(allNodes) == 0 { - // resp.WriteHeader(401) - // resp.Write([]byte(`{"success": false, "reason": "Please insert a node"}`)) - // return - //} - - // Allowed with only a start node - //if len(allNodes) != 1 { - // resp.WriteHeader(401) - // resp.Write([]byte(`{"success": false, "reason": "There are nodes with no branches"}`)) - // return - //} - } - - // FIXME - might be a sploit to run someone elses app if getAllWorkflowApps - // doesn't check sharing=true - // Have to do it like this to add the user's apps - //log.Println("Apps set starting") - //log.Printf("EXIT ON ERROR: %#v", workflow.Configuration.ExitOnError) - //workflowapps, apperr := shuffle.GetAllWorkflowApps(ctx, 500) - - // Started getting the single apps, but if it's weird, this is faster - // 1. Check workflow.Start - // 2. Check if any node has "isStartnode" - if len(workflow.Actions) > 0 { - index := -1 - for indexFound, action := range workflow.Actions { - //log.Println("Apps set done") - if workflow.Start == action.ID { - index = indexFound - } - } - - if index >= 0 { - workflow.Actions[0].IsStartNode = true - } else { - if workflow.PreviouslySaved { - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "You need to set a startnode."}`))) - return - } - } - } - - allAuths, err := shuffle.GetAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) - if userErr != nil { - log.Printf("Api authentication failed in get all apps: %s", userErr) - if workflow.PreviouslySaved { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - } - - // Check every app action and param to see whether they exist - //log.Printf("PRE ACTIONS 2") - allAuths, autherr := shuffle.GetAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) - newActions = []shuffle.Action{} - for _, action := range workflow.Actions { - reservedApps := []string{ - "0ca8887e-b4af-4e3e-887c-87e9d3bc3d3e", - } - - //log.Printf("%s Action execution var: %s", action.Label, action.ExecutionVariable.Name) - - builtin := false - for _, id := range reservedApps { - if id == action.AppID { - builtin = true - break - } - } - - // Check auth - // 1. Find the auth in question - // 2. Update the node and workflow info in the auth - // 3. Get the values in the auth and add them to the action values - if len(action.AuthenticationId) > 0 { - authFound := false - for _, auth := range allAuths { - if auth.Id == action.AuthenticationId { - authFound = true - - // Updates the auth item itself IF necessary - go updateAppAuth(auth, workflow.ID, action.ID, true) - break - } - } - - if !authFound { - log.Printf("App auth %s doesn't exist. Setting error", action.AuthenticationId) - workflow.Errors = append(workflow.Errors, fmt.Sprintf("App authentication for %s doesn't exist!", action.AppName)) - workflow.IsValid = false - - action.Errors = append(action.Errors, "App authentication doesn't exist") - action.IsValid = false - action.AuthenticationId = "" - //resp.WriteHeader(401) - //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "App auth %s doesn't exist"}`, action.AuthenticationId))) - //return - } - } - - if builtin { - newActions = append(newActions, action) - } else { - curapp := shuffle.WorkflowApp{} - // FIXME - can this work with ONLY AppID? - for _, app := range workflowapps { - if app.ID == action.AppID { - curapp = app - break - } - - // Has to NOT be generated - if app.Name == action.AppName && app.AppVersion == action.AppVersion { - curapp = app - break - } - } - - // Check to see if the whole app is valid - if curapp.Name != action.AppName { - workflow.Errors = append(workflow.Errors, fmt.Sprintf("App %s doesn't exist", action.AppName)) - action.Errors = append(action.Errors, "This app doesn't exist.") - action.IsValid = false - workflow.IsValid = false - - // Append with errors - newActions = append(newActions, action) - log.Printf("App %s doesn't exist. Adding as error.", action.AppName) - //resp.WriteHeader(401) - //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "App %s doesn't exist"}`, action.AppName))) - //return - } else { - // Check tosee if the appaction is valid - curappaction := shuffle.WorkflowAppAction{} - for _, curAction := range curapp.Actions { - if action.Name == curAction.Name { - curappaction = curAction - break - } - } - - // Check to see if the action is valid - if curappaction.Name != action.Name { - log.Printf("[ERROR] Action %s in app %s doesn't exist.", action.Name, curapp.Name) - thisError := fmt.Sprintf("%s: Action %s in app %s doesn't exist", action.Label, action.Name, action.AppName) - workflow.Errors = append(workflow.Errors, thisError) - workflow.IsValid = false - action.Errors = append(action.Errors, thisError) - action.IsValid = false - //if workflow.PreviouslySaved { - // resp.WriteHeader(401) - // resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Action %s in app %s doesn't exist"}`, action.Name, curapp.Name))) - // return - //} - } - - // FIXME - check all parameters to see if they're valid - // Includes checking required fields - - selectedAuth := shuffle.AppAuthenticationStorage{} - if len(action.AuthenticationId) > 0 && autherr == nil { - for _, auth := range allAuths { - if auth.Id == action.AuthenticationId { - selectedAuth = auth - break - } - } - } - - newParams := []shuffle.WorkflowAppActionParameter{} - for _, param := range curappaction.Parameters { - paramFound := false - - // Handles check for parameter exists + value not empty in used fields - for _, actionParam := range action.Parameters { - if actionParam.Name == param.Name { - paramFound = true - - if actionParam.Value == "" && actionParam.Variant == "STATIC_VALUE" && actionParam.Required == true { - // Validating if the field is an authentication field - if len(selectedAuth.Id) > 0 { - authFound := false - for _, field := range selectedAuth.Fields { - if field.Key == actionParam.Name { - authFound = true - //log.Printf("FOUND REQUIRED KEY %s IN AUTH", field.Key) - break - } - } - - if authFound { - newParams = append(newParams, actionParam) - continue - } - } - - log.Printf("[WARNING] Appaction %s with required param '%s' is empty. Can't save.", action.Name, param.Name) - thisError := fmt.Sprintf("%s is missing reqired parameter %s", action.Label, param.Name) - action.Errors = append(action.Errors, thisError) - workflow.Errors = append(workflow.Errors, thisError) - action.IsValid = false - } - - if actionParam.Variant == "" { - actionParam.Variant = "STATIC_VALUE" - } - - newParams = append(newParams, actionParam) - break - } - } - - // Handles check for required params - if !paramFound && param.Required { - log.Printf("Appaction %s with required param %s doesn't exist.", action.Name, param.Name) - thisError := fmt.Sprintf("Parameter %s is required", param.Name) - action.Errors = append(action.Errors, thisError) - - workflow.Errors = append(workflow.Errors, thisError) - action.IsValid = false - //newActions = append(newActions, action) - //resp.WriteHeader(401) - //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name))) - //return - } - - } - - action.Parameters = newParams - newActions = append(newActions, action) - } - } - } - - if !workflow.PreviouslySaved { - log.Printf("[WORKFLOW INIT] NOT PREVIOUSLY SAVED - SET ACTION AUTH!") - - if autherr == nil && len(workflowapps) > 0 && apperr == nil { - //log.Printf("Setting actions") - actionFixing := []shuffle.Action{} - appsAdded := []string{} - for _, action := range newActions { - setAuthentication := false - if len(action.AuthenticationId) > 0 { - //found := false - authenticationFound := false - for _, auth := range allAuths { - if auth.Id == action.AuthenticationId { - authenticationFound = true - break - } - } - - if !authenticationFound { - setAuthentication = true - } - } else { - // FIXME: 1. Validate if the app needs auth - // 1. Validate if auth for the app exists - // var appAuth AppAuthenticationStorage - setAuthentication = true - - //App WorkflowApp `json:"app" datastore:"app,noindex"` - } - - if setAuthentication { - authSet := false - for _, auth := range allAuths { - if !auth.Active { - continue - } - - if !auth.Defined { - continue - } - - if auth.App.Name == action.AppName { - //log.Printf("FOUND AUTH FOR APP %s: %s", auth.App.Name, auth.Id) - action.AuthenticationId = auth.Id - authSet = true - break - } - } - - // FIXME: Only o this IF there isn't another one for the app already - if !authSet { - //log.Printf("Validate if the app NEEDS auth or not") - outerapp := shuffle.WorkflowApp{} - for _, app := range workflowapps { - if app.Name == action.AppName { - outerapp = app - break - } - } - - if len(outerapp.ID) > 0 && outerapp.Authentication.Required { - found := false - for _, auth := range allAuths { - if auth.App.ID == outerapp.ID { - found = true - break - } - } - - for _, added := range appsAdded { - if outerapp.ID == added { - found = true - } - } - - // FIXME: Add app auth - if !found { - timeNow := int64(time.Now().Unix()) - authFields := []shuffle.AuthenticationStore{} - for _, param := range outerapp.Authentication.Parameters { - authFields = append(authFields, shuffle.AuthenticationStore{ - Key: param.Name, - Value: "", - }) - } - - appAuth := shuffle.AppAuthenticationStorage{ - Active: true, - Label: fmt.Sprintf("default_%s", outerapp.Name), - Id: uuid.NewV4().String(), - App: outerapp, - Fields: authFields, - Usage: []shuffle.AuthenticationUsage{}, - WorkflowCount: 0, - NodeCount: 0, - OrgId: user.ActiveOrg.Id, - Created: timeNow, - Edited: timeNow, - } - - err = shuffle.SetWorkflowAppAuthDatastore(ctx, appAuth, appAuth.Id) - if err != nil { - log.Printf("Failed setting appauth for with name %s", appAuth.Label) - } else { - appsAdded = append(appsAdded, outerapp.ID) - } - } - - action.Errors = append(action.Errors, "Requires authentication") - action.IsValid = false - workflow.IsValid = false - } - - //outerapp.Authentication.Required - // Authentication Authentication `json:"authentication" yaml:"authentication" required:false datastore:"authentication"` - //workflowapps, apperr := shuffle.GetAllWorkflowApps(ctx, 100) - } - } - - actionFixing = append(actionFixing, action) - } - - newActions = actionFixing - } else { - log.Printf("FirstSave error: %s - %s", err, apperr) - //workflowapps, apperr := shuffle.GetAllWorkflowApps(ctx, 100) - //allAuths, err := shuffle.GetAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) - } - - workflow.PreviouslySaved = true - } - - //log.Printf("PRE TRIGGERS") - //workflow.Actions = newActions - - workflow.Actions = newActions - workflow.IsValid = true - log.Printf("[INFO] Tags: %#v", workflow.Tags) - - // FIXME: Is this too drastic? May lead to issues in the future. - // Should maybe make a copy for the old org. - if workflow.OrgId != user.ActiveOrg.Id { - log.Printf("[WARNING] Editing workflow to be owned by %s", user.ActiveOrg.Id) - workflow.OrgId = user.ActiveOrg.Id - workflow.ExecutingOrg = user.ActiveOrg - workflow.Org = append(workflow.Org, user.ActiveOrg) - } - - err = setWorkflow(ctx, workflow, fileId) - if err != nil { - log.Printf("Failed saving workflow to database: %s", err) - if workflow.PreviouslySaved { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - } - - totalOldActions := len(tmpworkflow.Actions) - totalNewActions := len(workflow.Actions) - err = increaseStatisticsField(ctx, "total_workflow_actions", workflow.ID, int64(totalNewActions-totalOldActions), workflow.OrgId) - if err != nil { - log.Printf("Failed to change total actions data: %s", err) - } - - type returnData struct { - Success bool `json:"success"` - Errors []string `json:"errors"` - } - - returndata := returnData{ - Success: true, - Errors: workflow.Errors, - } - - // Really don't know why this was happening - //cacheKey := fmt.Sprintf("workflowapps-sorted-100") - //requestCache.Delete(cacheKey) - //cacheKey = fmt.Sprintf("workflowapps-sorted-500") - //requestCache.Delete(cacheKey) - - log.Printf("[INFO] Saved new version of workflow %s (%s) for org %s", workflow.Name, fileId, workflow.OrgId) - resp.WriteHeader(200) - newBody, err := json.Marshal(returndata) - if err != nil { - resp.Write([]byte(`{"success": true}`)) - return - } - - resp.Write(newBody) -} func getWorkflowLocal(fileId string, request *http.Request) ([]byte, error) { fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s", localBase, fileId) @@ -3150,7 +2073,7 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) { } } - err = setWorkflowExecution(ctx, *workflowExecution, true) + err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, true) if err != nil { log.Printf("Error saving workflow execution for updates when aborting %s: %s", topic, err) resp.WriteHeader(401) @@ -3433,7 +2356,7 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request } oldExecution.Results = newResults - err = setWorkflowExecution(ctx, *oldExecution, true) + err = shuffle.SetWorkflowExecution(ctx, *oldExecution, true) if err != nil { log.Printf("Error saving workflow execution actionresult setting: %s", err) return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed setting workflowexecution actionresult in execution: %s", err), err @@ -3801,8 +2724,9 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request workflowExecution.Workflow.Org = []shuffle.Org{ workflowExecution.Workflow.ExecutingOrg, } + //Org []Org `json:"org,omitempty" datastore:"org"` - err = setWorkflowExecution(ctx, workflowExecution, true) + err = shuffle.SetWorkflowExecution(ctx, workflowExecution, true) if err != nil { log.Printf("Error saving workflow execution for updates %s: %s", topic, err) return shuffle.WorkflowExecution{}, "Failed getting workflowexecution", err @@ -4486,7 +3410,7 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { } workflow.Schedules = append(workflow.Schedules, schedule) - err = setWorkflow(ctx, *workflow, workflow.ID) + err = shuffle.SetWorkflow(ctx, *workflow, workflow.ID) if err != nil { log.Printf("Failed setting workflow for schedule: %s", err) resp.WriteHeader(401) @@ -4499,217 +3423,6 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { return } -// FIXME - add to actual database etc -func getSpecificWorkflow(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - user, err := shuffle.HandleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in getting specific workflow: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - location := strings.Split(request.URL.String(), "/") - - var fileId string - if location[1] == "api" { - if len(location) <= 4 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - if strings.Contains(fileId, "?") { - fileId = strings.Split(fileId, "?")[0] - } - - if len(fileId) != 36 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Workflow ID when getting workflow is not valid"}`)) - return - } - - ctx := context.Background() - //memcacheName := fmt.Sprintf("%s_%s", user.Username, fileId) - //if item, err := memcache.Get(ctx, memcacheName); err == memcache.ErrCacheMiss { - // // Not in cache - // log.Printf("User %s not in cache.", memcacheName) - //} else if err != nil { - // log.Printf("Error getting item: %v", err) - //} else { - // log.Printf("Got workflow %s from cache", fileId) - // // FIXME - verify if value is ok? Can unmarshal etc. - // resp.WriteHeader(200) - // resp.Write(item.Value) - // return - //} - - workflow, err := shuffle.GetWorkflow(ctx, fileId) - if err != nil { - log.Printf("Workflow %s doesn't exist.", fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Item already exists."}`)) - return - } - - // CHECK orgs of user, or if user is owner - // FIXME - add org check too, and not just owner - // Check workflow.Sharing == private / public / org too - if user.Id != workflow.Owner && user.Role != "admin" { - log.Printf("Wrong user (%s) for workflow %s (get workflow)", user.Username, workflow.ID) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if len(workflow.Actions) == 0 { - workflow.Actions = []shuffle.Action{} - } - if len(workflow.Branches) == 0 { - workflow.Branches = []shuffle.Branch{} - } - if len(workflow.Triggers) == 0 { - workflow.Triggers = []shuffle.Trigger{} - } - if len(workflow.Errors) == 0 { - workflow.Errors = []string{} - } - - // Only required for individuals I think - //newactions := []Action{} - //for _, item := range workflow.Actions { - // item.LargeImage = "" - // item.SmallImage = "" - // newactions = append(newactions, item) - //} - //workflow.Actions = newactions - - //newtriggers := []Trigger{} - //for _, item := range workflow.Triggers { - // item.LargeImage = "" - // newtriggers = append(newtriggers, item) - //} - //workflow.Triggers = newtriggers - - body, err := json.Marshal(workflow) - if err != nil { - log.Printf("Failed workflow GET marshalling: %s", err) - resp.WriteHeader(http.StatusInternalServerError) - resp.Write([]byte(`{"success": false}`)) - return - } - - //item := &memcache.Item{ - // Key: memcacheName, - // Value: body, - // Expiration: time.Minute * 60, - //} - //if err := memcache.Add(ctx, item); err == memcache.ErrNotStored { - // if err := memcache.Set(ctx, item); err != nil { - // log.Printf("Error setting item: %v", err) - // } - //} else if err != nil { - // log.Printf("error adding item: %v", err) - //} else { - // //log.Printf("Set cache for %s", item.Key) - //} - - resp.WriteHeader(200) - resp.Write(body) -} - -func setWorkflowExecution(ctx context.Context, workflowExecution shuffle.WorkflowExecution, dbSave bool) error { - //log.Printf("\n\n\nRESULT: %s\n\n\n", workflowExecution.Status) - if len(workflowExecution.ExecutionId) == 0 { - log.Printf("Workflowexeciton executionId can't be empty.") - return errors.New("ExecutionId can't be empty.") - } - - cacheKey := fmt.Sprintf("workflowexecution-%s", workflowExecution.ExecutionId) - requestCache.Set(cacheKey, &workflowExecution, cache.DefaultExpiration) - if !dbSave && workflowExecution.Status == "EXECUTING" && len(workflowExecution.Results) > 1 { - //log.Printf("[WARNING] SHOULD skip DB saving for execution") - return nil - } - - // New struct, to not add body, author etc - key := datastore.NameKey("workflowexecution", workflowExecution.ExecutionId, nil) - if _, err := dbclient.Put(ctx, key, &workflowExecution); err != nil { - log.Printf("Error adding workflow_execution: %s", err) - return err - } - - return nil -} - -func getWorkflowExecution(ctx context.Context, id string) (*shuffle.WorkflowExecution, error) { - workflowExecution := &shuffle.WorkflowExecution{} - cacheKey := fmt.Sprintf("workflowexecution-%s", id) - if value, found := requestCache.Get(cacheKey); found { - parsedValue := value.(*shuffle.WorkflowExecution) - //log.Printf("Found execution for id %s with %d results", parsedValue.ExecutionId, len(parsedValue.Results)) - return parsedValue, nil - - //log.Printf("[INFO] FOUND key %s with value length %d", cacheKey, len(parsedValue)) - //err := json.Unmarshal([]byte(parsedValue), &workflowExecution) - //if err == nil { - // log.Printf("SHOULD RETURN CACHED EXECUTION of length %d", len(parsedValue)) - //} else { - // log.Printf("Failed unmarshalling cached value: %s", err) - //} - } else { - //log.Printf("[ERROR] Couldn't find key %s", cacheKey) - } - - key := datastore.NameKey("workflowexecution", strings.ToLower(id), nil) - if err := dbclient.Get(ctx, key, workflowExecution); err != nil { - return &shuffle.WorkflowExecution{}, err - } - - return workflowExecution, nil -} - -//func shuffle.GetApp(ctx context.Context, id string) (*WorkflowApp, error) { -// key := datastore.NameKey("workflowapp", strings.ToLower(id), nil) -// workflowApp := &WorkflowApp{} -// if err := dbclient.Get(ctx, key, workflowApp); err != nil { -// return &WorkflowApp{}, err -// -// } -// -// return workflowApp, nil -//} -// -//func shuffle.GetWorkflow(ctx context.Context, id string) (*shuffle.Workflow, error) { -// key := datastore.NameKey("workflow", strings.ToLower(id), nil) -// workflow := &Workflow{} -// if err := dbclient.Get(ctx, key, workflow); err != nil { -// return &Workflow{}, err -// } -// -// return workflow, nil -//} -// -//func shuffle.GetEnvironments(ctx context.Context, orgId string) ([]Environment, error) { -// var environments []Environment -// q := datastore.NewQuery("Environments").Filter("org_id =", orgId) -// -// _, err := dbclient.GetAll(ctx, q, &environments) -// if err != nil { -// return []Environment{}, err -// } -// -// return environments, nil -//} - func setExampleresult(ctx context.Context, result shuffle.AppExecutionExample) error { // FIXME: Reintroduce this for stats //key := datastore.NameKey("example_result", result.ExampleId, nil) @@ -4723,78 +3436,6 @@ func setExampleresult(ctx context.Context, result shuffle.AppExecutionExample) e return nil } -// Hmm, so I guess this should use uuid :( -// Consistency PLX -func setWorkflow(ctx context.Context, workflow shuffle.Workflow, id string, optionalEditedSecondsOffset ...int) error { - workflow.Edited = int64(time.Now().Unix()) - if len(optionalEditedSecondsOffset) > 0 { - workflow.Edited += int64(optionalEditedSecondsOffset[0]) - } - - key := datastore.NameKey("workflow", id, nil) - - // New struct, to not add body, author etc - if _, err := dbclient.Put(ctx, key, &workflow); err != nil { - log.Printf("Error adding workflow: %s", err) - return err - } - - return nil -} - -func deleteAppAuthentication(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - user, userErr := shuffle.HandleApiAuthentication(resp, request) - if userErr != nil { - log.Printf("Api authentication failed in edit workflow: %s", userErr) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if user.Role != "admin" { - log.Printf("Need to be admin to delete appauth") - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - location := strings.Split(request.URL.String(), "/") - log.Printf("%#v", location) - var fileId string - if location[1] == "api" { - if len(location) <= 5 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[5] - } - - // FIXME: Set affected workflows to have errors - // 1. Get the auth - // 2. Loop the workflows (.Usage) and set them to have errors - // 3. Loop the nodes in workflows and do the same - - log.Printf("ID: %s", fileId) - ctx := context.Background() - err := DeleteKey(ctx, "workflowappauth", fileId) - if err != nil { - log.Printf("Failed deleting workflowapp") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed deleting workflow app"}`))) - return - } - - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true}`)) -} - // FIXME: Not suitable for cloud right now :O func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) @@ -4896,7 +3537,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { //} } - err = setWorkflow(ctx, workflow, workflow.ID) + err = shuffle.SetWorkflow(ctx, workflow, workflow.ID) if err != nil { log.Printf("Failed setting workflow when deleting app: %s", err) continue @@ -4948,9 +3589,9 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { log.Printf("Failed to increase total apps loaded stats: %s", err) } cacheKey := fmt.Sprintf("workflowapps-sorted-100") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) cacheKey = fmt.Sprintf("workflowapps-sorted-500") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) //err = memcache.Delete(request.Context(), sessionToken) resp.WriteHeader(200) @@ -5064,416 +3705,6 @@ func getWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { resp.Write(data) } -func setAuthenticationConfig(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - user, userErr := shuffle.HandleApiAuthentication(resp, request) - if userErr != nil { - log.Printf("Api authentication failed in get all apps: %s", userErr) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if user.Role != "admin" { - log.Printf("[WARNING] User isn't admin during auth edit config") - resp.WriteHeader(409) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Must be admin to perform this action"}`))) - return - } - - var fileId string - location := strings.Split(request.URL.String(), "/") - if location[1] == "api" { - if len(location) <= 5 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[5] - } - - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Printf("Error with body read: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - type configAuth struct { - Id string `json:"id"` - Action string `json:"action"` - } - - var config configAuth - err = json.Unmarshal(body, &config) - if err != nil { - log.Printf("Failed unmarshaling (appauth): %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if config.Id != fileId { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Bad ID match"}`)) - return - } - - ctx := context.Background() - auth, err := shuffle.GetWorkflowAppAuthDatastore(ctx, fileId) - if err != nil { - log.Printf("Authget error: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": ":("}`)) - return - } - - if auth.OrgId != user.ActiveOrg.Id { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "User can't edit this org"}`)) - return - } - - if config.Action == "assign_everywhere" { - log.Printf("Should set authentication config") - q := datastore.NewQuery("workflow").Filter("org_id =", user.ActiveOrg.Id) - q = q.Order("-edited").Limit(35) - - var workflows []shuffle.Workflow - _, err = dbclient.GetAll(ctx, q, &workflows) - if err != nil { - log.Printf("Getall error in auth update: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Failed getting workflows to update"}`)) - return - } - - // FIXME: Add function to remove auth from other auth's - actionCnt := 0 - workflowCnt := 0 - authenticationUsage := []shuffle.AuthenticationUsage{} - for _, workflow := range workflows { - newActions := []shuffle.Action{} - edited := false - usage := shuffle.AuthenticationUsage{ - WorkflowId: workflow.ID, - Nodes: []string{}, - } - - for _, action := range workflow.Actions { - if action.AppName == auth.App.Name { - action.AuthenticationId = auth.Id - - edited = true - actionCnt += 1 - usage.Nodes = append(usage.Nodes, action.ID) - } - - newActions = append(newActions, action) - } - - workflow.Actions = newActions - if edited { - //auth.Usage = usage - authenticationUsage = append(authenticationUsage, usage) - err = setWorkflow(ctx, workflow, workflow.ID) - if err != nil { - log.Printf("Failed setting (authupdate) workflow: %s", err) - continue - } - - workflowCnt += 1 - } - } - - //Usage []AuthenticationUsage `json:"usage" datastore:"usage"` - log.Printf("[INFO] Found %d workflows, %d actions", workflowCnt, actionCnt) - if actionCnt > 0 && workflowCnt > 0 { - auth.WorkflowCount = int64(workflowCnt) - auth.NodeCount = int64(actionCnt) - auth.Usage = authenticationUsage - auth.Defined = true - - err = shuffle.SetWorkflowAppAuthDatastore(ctx, *auth, auth.Id) - if err != nil { - log.Printf("Failed setting appauth: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Failed setting app auth for all workflows"}`)) - return - } else { - // FIXME: Remove ALL workflows from other auths using the same - } - } - } - - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true}`)) - //var config configAuth - - //log.Printf("Should set %s -} - -func addAppAuthentication(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - user, userErr := shuffle.HandleApiAuthentication(resp, request) - if userErr != nil { - log.Printf("Api authentication failed in get all apps: %s", userErr) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Printf("Error with body read: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - var appAuth shuffle.AppAuthenticationStorage - err = json.Unmarshal(body, &appAuth) - if err != nil { - log.Printf("Failed unmarshaling (appauth): %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - ctx := context.Background() - if len(appAuth.Id) == 0 { - appAuth.Id = uuid.NewV4().String() - } else { - auth, err := shuffle.GetWorkflowAppAuthDatastore(ctx, appAuth.Id) - if err == nil { - // OrgId string `json:"org_id" datastore:"org_id"` - if auth.OrgId != user.ActiveOrg.Id { - log.Printf("[WARNING] User isn't a part of the right org during auth edit") - resp.WriteHeader(409) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": ":("}`))) - return - } - - if user.Role != "admin" { - log.Printf("[WARNING] User isn't admin during auth edit") - resp.WriteHeader(409) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": ":("}`))) - return - } - - if !auth.Active { - log.Printf("[WARNING] Auth isn't active for edit") - resp.WriteHeader(409) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't update an inactive auth"}`))) - return - } - - if auth.App.Name != appAuth.App.Name { - log.Printf("[WARNING] User tried to modify auth") - resp.WriteHeader(409) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad app configuration: need to specify correct name"}`))) - return - } - } - } - - if len(appAuth.Label) == 0 { - resp.WriteHeader(409) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Label can't be empty"}`))) - return - } - - // Super basic check - if len(appAuth.App.ID) != 36 && len(appAuth.App.ID) != 32 { - log.Printf("Bad ID for app: %s", appAuth.App.ID) - resp.WriteHeader(409) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "App has to be defined"}`))) - return - } - - // FIXME: Doens't validate Org - app, err := shuffle.GetApp(ctx, appAuth.App.ID) - if err != nil { - log.Printf("[WARNING] Failed finding app %s while setting auth. Finding it by looping apps.", appAuth.App.ID) - workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500) - if err != nil { - resp.WriteHeader(409) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return - } - - foundIndex := -1 - for i, workflowapp := range workflowapps { - if workflowapp.Name == appAuth.App.Name { - foundIndex = i - break - } - } - - if foundIndex >= 0 { - log.Printf("[INFO] Found app %s by looping auth", appAuth.App.ID) - } else { - log.Printf("[ERROR] Failed finding app %s which has auth after looping", appAuth.App.ID) - resp.WriteHeader(409) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return - } - } - - // Check if the items are correct - for _, field := range appAuth.Fields { - found := false - for _, param := range app.Authentication.Parameters { - if field.Key == param.Name { - found = true - } - } - - if !found { - log.Printf("Failed finding field %s in appauth fields", field.Key) - resp.WriteHeader(409) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "All auth fields required"}`))) - return - } - } - - //appAuth.LargeImage = "" - appAuth.OrgId = user.ActiveOrg.Id - appAuth.Defined = true - err = shuffle.SetWorkflowAppAuthDatastore(ctx, appAuth, appAuth.Id) - if err != nil { - log.Printf("Failed setting up app auth %s: %s", appAuth.Id, err) - resp.WriteHeader(409) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return - } - - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true}`)) -} - -func getAppAuthentication(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - user, userErr := shuffle.HandleApiAuthentication(resp, request) - if userErr != nil { - log.Printf("Api authentication failed in get all apps: %s", userErr) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - // FIXME: Auth to get the right ones only - //if user.Role != "admin" { - // log.Printf("User isn't admin") - // resp.WriteHeader(401) - // resp.Write([]byte(`{"success": false}`)) - // return - //} - ctx := context.Background() - allAuths, err := shuffle.GetAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) - if err != nil { - log.Printf("Api authentication failed in get all app auth: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if len(allAuths) == 0 { - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true, "data": []}`)) - return - } - - // Cleanup for frontend usage. User shouldn't be able to get the data. - newAuth := []shuffle.AppAuthenticationStorage{} - for _, auth := range allAuths { - newAuthField := auth - for index, _ := range auth.Fields { - newAuthField.Fields[index].Value = "auth placeholder (replaced during execution)" - } - - newAuth = append(newAuth, newAuthField) - } - - newbody, err := json.Marshal(allAuths) - if err != nil { - log.Printf("Failed unmarshalling all app auths: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking workflow app auth"}`))) - return - } - - data := fmt.Sprintf(`{"success": true, "data": %s}`, string(newbody)) - - resp.WriteHeader(200) - resp.Write([]byte(data)) - - /* - data := `{ - "success": true, - "data": [ - { - "app": { - "name": "thehive", - "description": "what", - "app_version": "1.0.0", - "id": "4f97da9d-1caf-41cc-aa13-67104d8d825c", - "large_image": "asd" - }, - "fields": { - "apikey": "hello", - "url": "url" - }, - "usage": [{ - "workflow_id": "asd", - "nodes": [{ - "node_id": "" - }] - }], - "label": "Original", - "id": "4f97da9d-1caf-41cc-aa13-67104d8d825d", - "active": true - }, - { - "app": { - "name": "thehive", - "description": "what", - "app_version": "1.0.0", - "id": "4f97da9d-1caf-41cc-aa13-67104d8d825c", - "large_image": "asd" - }, - "fields": { - "apikey": "hello", - "url": "url" - }, - "usage": [{ - "workflow_id": "asd", - "nodes": [{ - "node_id": "" - }] - }], - "label": "Number 2", - "id": "4f97da9d-1caf-41cc-aa13-67104d8d825d", - "active": true - } - ] - }` - */ -} func updateWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -5555,9 +3786,9 @@ func updateWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { } cacheKey := fmt.Sprintf("workflowapps-sorted-100") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) cacheKey = fmt.Sprintf("workflowapps-sorted-500") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) log.Printf("Changed workflow app %s", app.ID) resp.WriteHeader(200) @@ -6169,10 +4400,11 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { return } + ctx := context.Background() cacheKey := fmt.Sprintf("workflowapps-sorted-100") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) cacheKey = fmt.Sprintf("workflowapps-sorted-500") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) // Just need to be logged in // FIXME - should have some permissions? @@ -6198,7 +4430,7 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { } log.Printf("[INFO] Starting hotloading from %s", location) - err = handleAppHotload(location, true) + err = handleAppHotload(ctx, location, true) if err != nil { log.Printf("Failed app hotload: %s", err) resp.WriteHeader(500) @@ -6207,9 +4439,9 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { } cacheKey = fmt.Sprintf("workflowapps-sorted-100") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) cacheKey = fmt.Sprintf("workflowapps-sorted-500") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) @@ -6337,10 +4569,11 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) { return } + ctx := context.Background() cacheKey := fmt.Sprintf("workflowapps-sorted-100") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) cacheKey = fmt.Sprintf("workflowapps-sorted-500") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) @@ -6477,9 +4710,9 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, } cacheKey := fmt.Sprintf("workflowapps-sorted-100") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) cacheKey = fmt.Sprintf("workflowapps-sorted-500") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) } } else { //log.Printf("Skipped upload of %s (%s)", api.Name, api.ID) @@ -6593,7 +4826,7 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra log.Printf("Import workflow from file: %s", filename) ctx := context.Background() - err = setWorkflow(ctx, workflow, workflow.ID, secondsOffset) + err = shuffle.SetWorkflow(ctx, workflow, workflow.ID, secondsOffset) if err != nil { log.Printf("Failed setting (download) workflow: %s", err) continue @@ -6896,9 +5129,9 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin // This is getting silly cacheKey := fmt.Sprintf("workflowapps-sorted-100") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) cacheKey = fmt.Sprintf("workflowapps-sorted-500") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) //log.Printf("BUILDLATERFIRST: %d, BUILDLATERLIST: %d", len(buildLaterFirst), len(buildLaterList)) if len(extra) == 0 { @@ -7020,11 +5253,10 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { log.Printf("Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion) } - //memcache.Delete(ctx, "all_apps") cacheKey := fmt.Sprintf("workflowapps-sorted-100") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) cacheKey = fmt.Sprintf("workflowapps-sorted-500") - requestCache.Delete(cacheKey) + shuffle.DeleteCache(ctx, cacheKey) resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 4cd4939a..51d09de6 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -173,7 +173,6 @@ const AngularWorkflow = (props) => { const [selectedApp, setSelectedApp] = React.useState({}); const [selectedAction, setSelectedAction] = React.useState({}); - const [selectedActionName, setSelectedActionName] = React.useState({}); const [selectedActionEnvironment, setSelectedActionEnvironment] = React.useState({}); const [executionRequest, setExecutionRequest] = React.useState({}) @@ -1173,20 +1172,25 @@ const AngularWorkflow = (props) => { const onNodeSelect = (event, newAppAuth) => { const data = event.target.data() setLastSaved(false) - const branch = workflow.branches.filter(branch => branch.source_id === data.id || branch.destination_id === data.id) + //const branch = workflow.branches.filter(branch => branch.source_id === data.id || branch.destination_id === data.id) console.log("NODE: ", data) - console.log("BRANCHES: ", branch) + //console.log("BRANCHES: ", branch) if (data.type === "ACTION") { + + // FIXME - unselect //console.log(cy.elements('[_id!="${data._id}"]`)) // Does it choose the wrong action? var curaction = workflow.actions.find(a => a.id === data.id) if (!curaction || curaction === undefined) { + //event.target.unselect() //alert.error("Action not found. Please remake it.") return } + setSelectedAction(curaction) + const curapp = apps.find(a => a.name === curaction.app_name && a.app_version === curaction.app_version) if (!curapp || curapp === undefined) { alert.error("App "+curaction.app_name+" not found. Did someone delete it?") @@ -1202,7 +1206,7 @@ const AngularWorkflow = (props) => { } var tmpAuth = JSON.parse(JSON.stringify(newAppAuth)) - console.log("Checking authentication: ", tmpAuth) + //console.log("Checking authentication: ", tmpAuth) for (var key in tmpAuth) { var item = tmpAuth[key] @@ -1221,7 +1225,7 @@ const AngularWorkflow = (props) => { } curaction.authentication = authenticationOptions - console.log("Authentication: ", authenticationOptions) + //console.log("Authentication: ", authenticationOptions) if (curaction.selectedAuthentication === null || curaction.selectedAuthentication === undefined || curaction.selectedAuthentication.length === "") { curaction.selectedAuthentication = {} } @@ -1243,8 +1247,6 @@ const AngularWorkflow = (props) => { setSelectedActionEnvironment(env) } - setSelectedActionName(curaction.name) - setSelectedAction(curaction) /* var params = [] @@ -1273,7 +1275,6 @@ const AngularWorkflow = (props) => { const trigger_index = workflow.triggers.findIndex(a => a.id === data.id) setSelectedTriggerIndex(trigger_index) setSelectedTrigger(data) - setSelectedActionName(data.name) setSelectedActionEnvironment(data.env) if (data.app_name === "Shuffle Workflow") { @@ -1781,7 +1782,6 @@ const AngularWorkflow = (props) => { const removeNode = () => { setSelectedApp({}) setSelectedAction({}) - setSelectedActionName("") const selectedNode = cy.$(':selected') if (selectedNode.data() === undefined) { @@ -2757,7 +2757,6 @@ const AngularWorkflow = (props) => { //setSelectedActionEnvironment(env) setSelectedAction(selectedAction) - setSelectedActionName(e.target.value) } // APPSELECT at top @@ -2766,7 +2765,7 @@ const AngularWorkflow = (props) => { // ACTION select // const selectedNameChange = (event) => { - console.log("OLDNAME: ", selectedActionName) + //console.log("OLDNAME: ", selectedAction.name) event.target.value = event.target.value.replace("(", "") event.target.value = event.target.value.replace(")", "") event.target.value = event.target.value.replace("$", "") @@ -2777,7 +2776,7 @@ const AngularWorkflow = (props) => { selectedAction.label = event.target.value setSelectedAction(selectedAction) - console.log("SHOULD CHANGE NAME EVERYWHERE ITS USED TOO BASED ON OLD NAME!") + //console.log("SHOULD CHANGE NAME EVERYWHERE ITS USED TOO BASED ON OLD NAME!") /* if (nodeaction.label !== curaction.label) { @@ -3103,7 +3102,6 @@ const AngularWorkflow = (props) => { selectedActionParameters[count].action_field = fieldvalue selectedAction.parameters = selectedActionParameters - setSelectedActionName(selectedActionName) setSelectedApp(selectedApp) setSelectedAction(selectedAction) setUpdate(fieldvalue) @@ -3124,7 +3122,6 @@ const AngularWorkflow = (props) => { // FIXME - check if startnode // Set value - setSelectedActionName(selectedActionName) setSelectedApp(selectedApp) setSelectedAction(selectedAction) @@ -3151,7 +3148,6 @@ const AngularWorkflow = (props) => { selectedAction.parameters = selectedActionParameters // This is a stupid workaround to make it refresh rofl - setSelectedActionName({}) setSelectedAction({}) setSelectedTrigger({}) setSelectedApp({}) @@ -3159,7 +3155,6 @@ const AngularWorkflow = (props) => { // FIXME - check if startnode // Set value - setSelectedActionName(selectedActionName) setSelectedApp(selectedApp) setSelectedAction(selectedAction) } @@ -4199,7 +4194,7 @@ const AngularWorkflow = (props) => { Actions
} + > + + Admin + + + User + + + } style ={{ minWidth: 135, maxWidth: 135, marginRight: 15,}} /> { - {errorCode.length > 0 ? `Error: ${errorCode}` : null} + + {errorCode.length > 0 ? `Error: ${errorCode}` : null} +
diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 0b4cd611..b7bc7626 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -195,6 +195,10 @@ const Apps = (props) => { setIsLoading(false) if (response.status !== 200) { console.log("Status not 200 for apps :O!") + + if (isCloud) { + window.location.pathname = "/search" + } } return response.json() @@ -648,11 +652,17 @@ const Apps = (props) => { upload = ref} onChange={importFiles} /> {workflows.length > 0 ? From 1b428b301a8177b6fc7a150850a1fb9113c220a0 Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 29 Mar 2021 21:58:48 +0200 Subject: [PATCH 53/57] Fixed no-path issue with requirement of leading / --- backend/Dockerfile | 3 - backend/go-app/go.mod | 4 +- backend/go-app/main.go | 139 ++++++++++++++++- backend/go-app/walkoff.go | 3 +- docker-compose.yml | 12 +- frontend/Dockerfile | 2 +- frontend/src/defaultCytoscapeStyle.js | 21 ++- frontend/src/views/AngularWorkflow.jsx | 125 ++++++++++++--- frontend/src/views/AppCreator.jsx | 6 +- frontend/src/views/MyView.jsx | 205 +++++++++++++------------ frontend/src/views/Workflows.jsx | 24 +-- functions/onprem/orborus/build.sh | 2 +- functions/onprem/orborus/orborus.go | 2 +- functions/onprem/worker/Dockerfile | 5 +- functions/onprem/worker/build.sh | 2 +- functions/onprem/worker/worker.go | 25 +-- 16 files changed, 419 insertions(+), 161 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index ea93625e..5d7c484d 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -7,15 +7,12 @@ WORKDIR /app ADD ./go-app/main.go /app ADD ./go-app/walkoff.go /app ADD ./go-app/docker.go /app -ADD ./go-app/codegen.go /app -ADD ./go-app/files.go /app ADD ./go-app/oauth2.go /app ADD ./go-app/go.mod /app # Required files for code generation ADD ./app_sdk/app_base.py /app_sdk -ADD ./app_sdk/static_baseline.py /app_sdk ADD ./app_sdk_kali/app_base.py /app_sdk_kali ADD ./app_sdk_kali/static_baseline.py /app_sdk_kali ADD ./app_sdk_blackarch/app_base.py /app_sdk_blackarch diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 7d7f91f2..fa5a2e35 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -2,7 +2,7 @@ module shuffle go 1.13 -replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared +//replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi require ( @@ -18,7 +18,7 @@ require ( github.com/docker/docker v1.13.1 github.com/docker/go-connections v0.4.0 github.com/docker/go-units v0.4.0 // indirect - github.com/frikky/shuffle-shared v0.0.15 + github.com/frikky/shuffle-shared v0.0.20 github.com/ghodss/yaml v1.0.0 github.com/go-git/go-billy/v5 v5.0.0 github.com/go-git/go-git/v5 v5.0.0 diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 7d1de1b2..097bc190 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -77,6 +77,8 @@ var registryName = "registry.hub.docker.com" var runningEnvironment = "onprem" var syncUrl = "https://shuffler.io" + +//var syncUrl = "http://localhost:5002" var syncSubUrl = "https://shuffler.io" //var syncUrl = "http://localhost:5002" @@ -5463,12 +5465,12 @@ func runInit(ctx context.Context) { } iterateOpenApiGithub(fs, dir, "", "") - log.Printf("Finished downloading extra API samples") + log.Printf("[INFO] Finished downloading extra API samples") } workflowLocation := os.Getenv("SHUFFLE_DOWNLOAD_WORKFLOW_LOCATION") if len(workflowLocation) > 0 { - log.Printf("Downloading WORKFLOWS from %s if no workflows - EXTRA workflows", workflowLocation) + log.Printf("[INFO] Downloading WORKFLOWS from %s if no workflows - EXTRA workflows", workflowLocation) q := datastore.NewQuery("workflow").Limit(35) var workflows []shuffle.Workflow _, err = dbclient.GetAll(ctx, q, &workflows) @@ -5918,6 +5920,138 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { resp.Write(respBody) } +func makeWorkflowPublic(resp http.ResponseWriter, request *http.Request) { + cors := shuffle.HandleCors(resp, request) + if cors { + return + } + + user, userErr := shuffle.HandleApiAuthentication(resp, request) + if userErr != nil { + log.Printf("[WARNING] Api authentication failed in make workflow public: %s", userErr) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + location := strings.Split(request.URL.String(), "/") + var fileId string + if location[1] == "api" { + if len(location) <= 4 { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + fileId = location[4] + } + + ctx := context.Background() + if strings.Contains(fileId, "?") { + fileId = strings.Split(fileId, "?")[0] + } + + if len(fileId) != 36 { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Workflow ID when getting workflow is not valid"}`)) + return + } + + workflow, err := shuffle.GetWorkflow(ctx, fileId) + if err != nil { + log.Printf("[WARNING] Workflow %s doesn't exist in app publish. User: %s", fileId, user.Id) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + // CHECK orgs of user, or if user is owner + // FIXME - add org check too, and not just owner + // Check workflow.Sharing == private / public / org too + if user.Id != workflow.Owner || len(user.Id) == 0 { + if workflow.OrgId == user.ActiveOrg.Id && user.Role == "admin" { + log.Printf("[INFO] User %s is accessing workflow %s as admin", user.Username, workflow.ID) + } else { + log.Printf("[WARNING] Wrong user (%s) for workflow %s (get workflow)", user.Username, workflow.ID) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + } + + if !workflow.IsValid || !workflow.PreviouslySaved { + log.Printf("[INFO] Failed uploading workflow because it's invalid or not saved") + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Invalid workflows are not sharable"}`)) + return + } + + // Starting validation of the POST workflow + body, err := ioutil.ReadAll(request.Body) + if err != nil { + log.Printf("[WARNING] Body data error on mail: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + parsedWorkflow := shuffle.Workflow{} + err = json.Unmarshal(body, &parsedWorkflow) + if err != nil { + log.Printf("[WARNING] Unmarshal error on mail: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + // Super basic validation. Doesn't really matter. + if parsedWorkflow.ID != workflow.ID || len(parsedWorkflow.Actions) != len(workflow.Actions) { + log.Printf("[WARNING] Bad ID during publish: %s vs %s", workflow.ID, parsedWorkflow.ID) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if !workflow.IsValid || !workflow.PreviouslySaved { + log.Printf("[INFO] Failed uploading new workflow because it's invalid or not saved") + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Invalid workflows are not sharable"}`)) + return + } + + workflowData, err := json.Marshal(parsedWorkflow) + if err != nil { + log.Printf("[WARNING] Failed marshalling workflow: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + // Sanitization is done in the frontend as well + parsedWorkflow = shuffle.SanitizeWorkflow(parsedWorkflow) + parsedWorkflow.ID = uuid.NewV4().String() + action := shuffle.CloudSyncJob{ + Type: "workflow", + Action: "publish", + OrgId: user.ActiveOrg.Id, + PrimaryItemId: workflow.ID, + SecondaryItem: string(workflowData), + FifthItem: user.Id, + } + + err = executeCloudAction(action, user.ActiveOrg.SyncConfig.Apikey) + if err != nil { + log.Printf("[WARNING] Failed cloud PUBLISH: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + return + } + + log.Printf("[INFO] Successfully published workflow %s (%s) TO CLOUD", workflow.Name, workflow.ID) + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) +} + func initHandlers() { var err error ctx := context.Background() @@ -6041,6 +6175,7 @@ func initHandlers() { r.HandleFunc("/api/v1/get_openapi/{key}", getOpenapi).Methods("GET", "OPTIONS") // NEW for 0.8.0 + r.HandleFunc("/api/v1/workflows/{key}/publish", makeWorkflowPublic).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/cloud/setup", handleCloudSetup).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/orgs", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS") diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 3ff9bdf2..86f27be7 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -981,7 +981,7 @@ func validateNewWorkerExecution(body []byte) error { execution.Status = "FINISHED" } - log.Printf("BASEEXECUTION LENGTH: %d", len(baseExecution.Workflow.Actions)+extra) + log.Printf("[INFO] BASEEXECUTION LENGTH: %d", len(baseExecution.Workflow.Actions)+extra) } // FIXME: Add extra here @@ -4782,6 +4782,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin workflowapp.Sharing = true workflowapp.Downloaded = true workflowapp.Hash = md5 + workflowapp.Public = true err = shuffle.SetWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) if err != nil { diff --git a/docker-compose.yml b/docker-compose.yml index b96b2441..45656376 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,8 @@ version: '3' services: frontend: - #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.64 + build: ./frontend + image: ghcr.io/frikky/shuffle-frontend:0.8.70 container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -16,8 +16,8 @@ services: depends_on: - backend backend: - #build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.64 + build: ./backend + image: ghcr.io/frikky/shuffle-backend:0.8.70 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: @@ -47,7 +47,7 @@ services: - database orborus: #build: ./functions/onprem/orborus - image: ghcr.io/frikky/shuffle-orborus:0.8.63 + image: ghcr.io/frikky/shuffle-orborus:0.8.70 container_name: shuffle-orborus hostname: shuffle-orborus networks: @@ -56,7 +56,7 @@ services: - /var/run/docker.sock:/var/run/docker.sock environment: - SHUFFLE_APP_SDK_VERSION=0.8.60 - - SHUFFLE_WORKER_VERSION=0.8.63 + - SHUFFLE_WORKER_VERSION=0.8.70 - ORG_ID=${ORG_ID} - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT} diff --git a/frontend/Dockerfile b/frontend/Dockerfile index cd11e347..4204651c 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -21,7 +21,7 @@ COPY ./*.json /usr/src/app/ RUN yarn build # Production environment -FROM nginx:latest +FROM nginx:1.19 RUN mkdir -p /usr/share/nginx/html/build RUN mkdir -p /usr/share/nginx/html/css diff --git a/frontend/src/defaultCytoscapeStyle.js b/frontend/src/defaultCytoscapeStyle.js index ff9990a9..f2538a94 100644 --- a/frontend/src/defaultCytoscapeStyle.js +++ b/frontend/src/defaultCytoscapeStyle.js @@ -39,6 +39,23 @@ const data = [{ 'border-color': '#81c784', 'background-width': '100%', 'background-height': '100%', + 'border-radius': '5px', + }, + }, + { + selector: `node[app_name="Shuffle Tools"]`, + css: { + 'width': '30px', + 'height': '30px', + 'font-size': '0px', + }, + }, + { + selector: `node[app_name="Testing"]`, + css: { + 'width': '30px', + 'height': '30px', + 'font-size': '0px', }, }, { @@ -112,7 +129,7 @@ const data = [{ }, }, { - selector: 'node:selected', + selector: ':selected', css: { 'background-color': '#77b0d0', 'border-color': '#77b0d0', @@ -187,6 +204,8 @@ const data = [{ 'border-width': '12px', 'transition-property': 'border-width', 'transition-duration': '0.25s', + 'font-size': '30px', + 'label': 'data(label)', }, }, { diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index dbe016f6..8082415d 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -92,7 +92,6 @@ const AngularWorkflow = (props) => { const theme = useTheme(); const [bodyWidth, bodyHeight] = useWindowSize(); - const appBarSize = 75 var to_be_copied = "" const [cystyle, ] = useState(cytoscapestyle) @@ -190,18 +189,22 @@ const AngularWorkflow = (props) => { const [workflowExecutions, setWorkflowExecutions] = React.useState([]); const [defaultEnvironmentIndex, setDefaultEnvironmentIndex] = React.useState(0) + // This should all be set once, not on every iteration + // Use states and don't update lol const cloudSyncEnabled = props.userdata !== undefined && props.userdata.active_org !== null && props.userdata.active_org !== undefined ? props.userdata.active_org.cloud_sync === true : false //const triggerEnvironments = cloudSyncEnabled ? ["cloud", "onprem"] : environments const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" + const appBarSize = isCloud ? 75 : 60 const triggerEnvironments = isCloud ? ["cloud"] : ["onprem", "cloud"] - const unloadText = 'Are you sure you want to leave without saving (CTRL+S)?' + useBeforeunload(() => { if (!lastSaved) { return unloadText } }) + const [elements, setElements] = useState([]) // No point going as fast, as the nodes aren't realtime anymore, but bulk updated. // Set it from 2500 to 6000 to reduce overall load @@ -1121,7 +1124,7 @@ const AngularWorkflow = (props) => { } if (responseJson.public) { - alert.info("This workflow is public. You'll have to save it to make it your own!") + alert.info("This workflow is public. You will have to save it to make it your own!") setLastSaved(false) } @@ -1215,9 +1218,36 @@ const AngularWorkflow = (props) => { setSelectedTrigger({}) } + // Comparing locations between nodes and setting views + const onNodeDrag = (event, newAppAuth) => { + //console.log("DRAGGING: ", event.target) + //console.log("LEN2: ", event.target.edges.length) + + + /* + event.target.animate({ + style: { + "border-width": "12px", + "border-opacity": ".7", + } + }, { + duration: animationDuration, + }) + event.target.animate({ + style: { + "border-width": "12px", + "border-opacity": ".7", + } + }, { + duration: animationDuration, + }) + */ + } + // Nodeselectbatching: // https://stackoverflow.com/questions/16677856/cy-onselect-callback-only-once const onNodeSelect = (event, newAppAuth) => { + const data = event.target.data() setLastSaved(false) @@ -1619,6 +1649,17 @@ const AngularWorkflow = (props) => { }); } + + if (!firstrequest && graphSetup && established && props.match.params.key !== workflow.id && workflow.id !== undefined && workflow.id !== null && workflow.id.length > 0) { + //console.log(props.match.params.key, workflow.id) + //getWorkflow() + //setCy() + //getWorkflowExecution(props.match.params.key, "") + //setEstablished(false) + //setGraphSetup(false) + window.location.pathname = "/workflows/"+props.match.params.key + } + useEffect(() => { if (firstrequest) { setFirstrequest(false) @@ -1642,14 +1683,14 @@ const AngularWorkflow = (props) => { } // App length necessary cus of cy initialization - if (elements.length === 0 && !graphSetup && Object.getOwnPropertyNames(workflow).length > 0) { + if (elements.length === 0 && workflow.actions !== undefined && !graphSetup && Object.getOwnPropertyNames(workflow).length > 0) { setGraphSetup(true) setupGraph() } else if (!established && cy !== undefined && apps !== null && apps !== undefined && apps.length > 0 && Object.getOwnPropertyNames(workflow).length > 0 && authLoaded){ //This part has to load LAST, as it's kind of not async. //This means we need everything else to happen first. - console.log("AUTH IN HERE: ", appAuthentication) - + // + //console.log("IN THIS PART AGAIN") setEstablished(true) cy.edgehandles({ @@ -1677,6 +1718,9 @@ const AngularWorkflow = (props) => { cy.on('mouseover', 'node', (e) => onNodeHover(e)) cy.on('mouseout', 'node', (e) => onNodeHoverOut(e)) + // Handles dragging + //cy.on('drag', 'node', (e) => onNodeDrag(e)) + //cy.on('mouseover', 'node', () => $(targetElement).addClass('mouseover')); //cy.on('cxttapstart', 'node', (e) => edgeHandler.start(e.target)) @@ -1711,7 +1755,7 @@ const AngularWorkflow = (props) => { const onNodeHover = (event) => { event.target.animate({ style: { - "border-width": "5px", + "border-width": "7px", "border-opacity": ".7", } }, { @@ -1727,17 +1771,23 @@ const AngularWorkflow = (props) => { // This is here to have a proper transition for lines const onEdgeHover = (event) => { + if (event === null || event === undefined) { + return + } + const sourcecolor = cy.getElementById(event.target.data("source")).style("border-color") const targetcolor = cy.getElementById(event.target.data("target")).style("border-color") - event.target.animate({ - style: { - "line-fill": "linear-gradient", - 'target-arrow-color': targetcolor, - "line-gradient-stop-colors": [sourcecolor, targetcolor], - "line-gradient-stop-positions": [0, 1], - }, - duration: 0, - }) + if (sourcecolor !== null && sourcecolor !== undefined && targetcolor !== null && targetcolor !== undefined) { + event.target.animate({ + style: { + "line-fill": "linear-gradient", + 'target-arrow-color': targetcolor, + "line-gradient-stop-colors": [sourcecolor, targetcolor], + "line-gradient-stop-positions": [0, 1], + }, + duration: 0, + }) + } } @@ -1799,6 +1849,31 @@ const AngularWorkflow = (props) => { conditions: conditions, hasErrors: branch.has_errors }; + + // This is an attempt at prettier edges. The numbers are weird to work with. + /* + //http://manual.graphspace.org/projects/graphspace-python/en/latest/demos/edge-types.html + const sourcenode = actions.find(node => node.data._id === branch.source_id) + const destinationnode = actions.find(node => node.data._id === branch.destination_id) + if (sourcenode !== undefined && destinationnode !== undefined && branch.source_id !== branch.destination_id) { + //node.data._id = action["id"] + console.log("SOURCE: ", sourcenode.position) + console.log("DESTINATIONNODE: ", destinationnode.position) + + var opposite = true + if (sourcenode.position.x > destinationnode.position.x) { + opposite = false + } else { + opposite = true + } + + edge.style = { + 'control-point-distance': opposite ? ["25%", "-75%"] : ["-10%", "90%"], + 'control-point-weight': ['0.3', '0.7'], + } + } + */ + return edge; }) @@ -4506,7 +4581,7 @@ const AngularWorkflow = (props) => { : null } onClick={() => { - console.log("CHANGE FIELD") + //console.log("CHANGE FIELD") }} onBlur={(e) => { changeActionVariable(data.action_field, e.target.value) @@ -4803,6 +4878,7 @@ const AngularWorkflow = (props) => { onClick={() => { setSelectedEdge({}) + var data = { condition: conditionValue, source: sourceValue, @@ -4821,6 +4897,18 @@ const AngularWorkflow = (props) => { } } + var label = "" + if (selectedEdge.conditions.length === 1) { + label = selectedEdge.conditions.length+" condition" + } else if (selectedEdge.conditions.length > 1) { + label = selectedEdge.conditions.length+" conditions" + } + + var currentedge = cy.getElementById(selectedEdge.id) + if (currentedge !== undefined && currentedge !== null) { + currentedge.data().label = label + } + setSelectedEdge(selectedEdge) workflow.branches[selectedEdgeIndex] = selectedEdge setWorkflow(workflow) @@ -6494,7 +6582,7 @@ const AngularWorkflow = (props) => { {/* */} - + {workflow.configuration !== null && workflow.configuration !== undefined && workflow.configuration.exit_on_error !== undefined ? : null}
) @@ -7211,6 +7299,7 @@ const AngularWorkflow = (props) => { stylesheet={cystyle} boxSelectionEnabled={true} autounselectify={false} + showGrid={true} cy={(incy) => { // FIXME: There's something specific loading when // you do the first hover of a node. Why is this different? diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 483949be..38f40744 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -1565,9 +1565,9 @@ const AppCreator = (props) => { } // Url verification - if (currentAction.url.length === 0) { - errormessage.push("URL path can't be empty.") - } else if (!currentAction.url.startsWith("/") && baseUrl.length > 0) { + //if (currentAction.url.length === 0) { + // errormessage.push("URL path can't be empty.") + if (!currentAction.url.startsWith("/") && baseUrl.length > 0 && currentAction.url.length > 0) { errormessage.push("URL must start with /") } diff --git a/frontend/src/views/MyView.jsx b/frontend/src/views/MyView.jsx index 6ca5d0b0..b456c6ed 100644 --- a/frontend/src/views/MyView.jsx +++ b/frontend/src/views/MyView.jsx @@ -68,11 +68,9 @@ const flexContainerStyle = { } const flexBoxStyle = { - width: "333px", - height: "125px", - margin: "10px", - borderRadius: "4px", - boxShadow: "0px 1px 2px rgba(0, 0, 0, 0.16), 0px 2px 4px rgba(0, 0, 0, 0.12), 0px 1px 8px rgba(0, 0, 0, 0.1)", + width: 333, + height: 125, + borderRadius: 4, boxSizing: "border-box", letterSpacing: "0.4px", color: "#D6791E", @@ -299,37 +297,37 @@ const MyView = (props) => { } const paperAppContainer = { - cursor: "pointer", display: "flex", flexWrap: 'wrap', alignContent: "space-between", } + + const paperAppStyle = { - minHeight: "148px", - width: "333px", - margin: "10px", + minHeight: 130, + width: "100%", color: "white", backgroundColor: surfaceColor, + padding: "12px 12px 0px 15px", borderRadius: 5, - padding: "10px", - cursor: "pointer", display: "flex", boxSizing: "border-box", + position: "relative", } const gridContainer = { - cursor: "pointer", height: "auto", color: "white", margin: "10px", backgroundColor: surfaceColor, } - const workFlowActionStyle = { + const workflowActionStyle = { flex: "1", display: "flex", - width: "150px", + width: 150, + height: 44, justifyContent: "space-between", overflow: "hidden" } @@ -535,23 +533,28 @@ const MyView = (props) => { }); } - const getWorkFlowMeta = (data) => { + const getWorkflowMeta = (data) => { + let triggers = 0 let schedules = 0 let webhooks = 0 - let webhookImg = "" - let scheduleImg = "" + let subflows = 0 if (data.triggers !== undefined && data.triggers !== null && data.triggers.length > 0) { + triggers = data.triggers.length for (let key in data.triggers) { + if (data.triggers[key].app_name === "Webhook") { webhooks += 1 - webhookImg = data.triggers[key].large_image + //webhookImg = data.triggers[key].large_image } else if (data.triggers[key].app_name === "Schedule") { schedules += 1 - scheduleImg = data.triggers[key].large_image + //scheduleImg = data.triggers[key].large_image + } else if (data.triggers[key].app_name === "Subflow") { + subflows += 1 } } } - return [schedules, webhooks, webhookImg, scheduleImg]; + + return [triggers, schedules, webhooks, subflows] } // dropdown with copy etc I guess @@ -565,9 +568,9 @@ const MyView = (props) => { boxWidth = "4px" } - var boxColor = "orange" + var boxColor = "#FECC00" if (data.is_valid) { - boxColor = "green" + boxColor = "#86c142" } const menuClick = (event) => { @@ -575,45 +578,57 @@ const MyView = (props) => { setAnchorEl(event.currentTarget); } - const actions = data.actions !== null ? data.actions.length : 0 - - const [schedules, webhooks] = getWorkFlowMeta(data); + var parsedName = data.name + console.log("LEN: ", parsedName.length) + if (parsedName !== undefined && parsedName !== null && parsedName.length > 25) { + parsedName = parsedName.slice(0,25)+".." + } + + + const actions = data.actions !== null ? data.actions.length : 0 + const [triggers, schedules, webhooks, subflows] = getWorkflowMeta(data) - const imgSize = 25 return ( - { - }}> -
{ - if (selectedWorkflow.id !== data.id) { - setSelectedWorkflow(data) - //getWorkflowExecution(data.id) - } - }}/> - + + +
- -
{ - if (selectedWorkflow.id !== data.id) { - setSelectedWorkflow(data) - //getWorkflowExecution(data.id) - } - }}> - - {data.name} - -
+ + + {parsedName} + - - - + + + + + + {actions} + + - - executeWorkflow(data.id)} /> + + + + + {triggers} + + + + + + + + + {subflows} + + + + {/* - {webhooks > 0 ? @@ -623,8 +638,9 @@ const MyView = (props) => { : null} + */} - + {data.tags !== undefined ? data.tags.map((tag, index) => { if (index >= 3) { @@ -634,7 +650,7 @@ const MyView = (props) => { return ( { : null} - {data.actions !== undefined && data.actions !== null ? - + - - - - { - setOpen(false) - setAnchorEl(null) - }} - > + + + + { + setOpen(false) + setAnchorEl(null) + }} + > { setModalOpen(true) setEditingWorkflow(data) @@ -685,26 +700,25 @@ const MyView = (props) => { exportWorkflow(data) setOpen(false) }} key={"export"}>{"Export"} - { + { setDeleteModalOpen(true) setSelectedWorkflowId(data.id) setOpen(false) }} key={"delete"}>{"Delete"} - - - - - - - - - + + + + + + + + + - : null} - + : null} + + ) } @@ -772,7 +786,6 @@ borderRadius: "4px", color: "black", height: "30px", "width": "30px", fontSize: marginTop: "5px", color: "white", backgroundColor: surfaceColor, - cursor: "pointer", display: "flex", } @@ -1294,13 +1307,12 @@ borderRadius: "4px", color: "black", height: "30px", "width": "30px", fontSize: let workflowData = ""; if (workflows.length > 0) { const columns = [ - { field: 'id', headerName: 'ID', width: 70, sortable: false, }, { field: 'title', headerName: 'Title', width: 330, }, { field: 'actions', headerName: 'Actions', width: 200, sortable: false, disableClickEventBubbling: true, renderCell: (params) => { const data = params.row.record; - let [schedules, webhooks] = getWorkFlowMeta(data); + let [triggers, schedules, webhooks, subflows] = getWorkflowMeta(data); return @@ -1340,10 +1352,11 @@ borderRadius: "4px", color: "black", height: "30px", "width": "30px", fontSize: if (index >= 3) { return null } + return ( {view === "grid" && ( -
+ {workflows.map((data, index) => { return ( ) })} -
+ )} {view === "list" && ( diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index a955924a..9e477c29 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -447,12 +447,12 @@ const Workflows = (props) => { for (var branchkey in data.branches) { const branch = data.branches[branchkey] if (branch.source_id === data.actions[key].id) { - console.log("CHANGING SOURCE ID IN ACTION") + //console.log("CHANGING SOURCE ID IN ACTION") branch.source_id = newId } if (branch.destination_id === data.actions[key].id) { - console.log("CHANGING DESTINATION ID IN ACTION") + //console.log("CHANGING DESTINATION ID IN ACTION") branch.destination_id = newId } } @@ -505,10 +505,8 @@ const Workflows = (props) => { data = sanitizeWorkflow(data) alert.info("Sanitizing and publishing "+data.name) - const url = isCloud ? globalUrl : "https://shuffler.io" - // This ALWAYS talks to Shuffle cloud - fetch(url+"/api/v1/workflows/"+data.id+"/publish", { + fetch(globalUrl+"/api/v1/workflows/"+data.id+"/publish", { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -521,7 +519,11 @@ const Workflows = (props) => { if (response.status !== 200) { console.log("Status not 200 for workflow publish :O!") } else { - alert.success("Successfully published workflow") + if (isCloud) { + alert.success("Successfully published workflow") + } else { + alert.success("Successfully published workflow to https://shuffler.io") + } } return response.json() @@ -690,12 +692,10 @@ const Workflows = (props) => { setNewWorkflowTags(JSON.parse(JSON.stringify(data.tags))) } }} key={"change"}>{"Change details"} - {isCloud ? - { - console.log("Should publish", data) - publishWorkflow(data) - }} key={"publish"}>{"Publish Workflow"} - : null} + { + console.log("Should publish", data) + publishWorkflow(data) + }} key={"publish"}>{"Publish Workflow"} { copyWorkflow(data) setOpen(false) diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index 2ea4d555..48364d69 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-orborus -VERSION=0.8.64 +VERSION=0.8.70 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index ad508e8a..db337158 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -248,7 +248,7 @@ func initializeImages() { log.Printf("[WARNING] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %s", appSdkVersion) } if workerVersion == "" { - workerVersion = "0.8.64" + workerVersion = "0.8.70" log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %s", workerVersion) } diff --git a/functions/onprem/worker/Dockerfile b/functions/onprem/worker/Dockerfile index 62aa2d0c..826ef4d3 100644 --- a/functions/onprem/worker/Dockerfile +++ b/functions/onprem/worker/Dockerfile @@ -11,7 +11,8 @@ RUN go get github.com/docker/docker/api/types && \ go get github.com/docker/docker/client && \ go get github.com/gorilla/mux && \ go get github.com/patrickmn/go-cache && \ - go get github.com/frikky/shuffle-shared + go get github.com/frikky/shuffle-shared && \ + go get github.com/satori/go.uuid RUN go build RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker . @@ -21,7 +22,7 @@ FROM alpine:3.12 ENV SHUFFLE_BASE_IMAGE_REGISTRY=docker.io ENV SHUFFLE_BASE_IMAGE_NAME=frikky/shuffle -ENV SHUFFLE_BASE_IMAGE_TAG_SUFFIX=0.8.5 +ENV SHUFFLE_BASE_IMAGE_TAG_SUFFIX=0.8.70 RUN apk add --no-cache bash COPY --from=builder /app/ / diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh index 424b6207..ce343930 100644 --- a/functions/onprem/worker/build.sh +++ b/functions/onprem/worker/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-worker -VERSION=0.8.64 +VERSION=0.8.70 echo "Running docker build with $NAME:$VERSION" #CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin . diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index e4de1908..cc981636 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -24,6 +24,7 @@ import ( //"github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/mount" dockerclient "github.com/docker/docker/client" + "github.com/satori/go.uuid" "github.com/gorilla/mux" "github.com/patrickmn/go-cache" @@ -278,7 +279,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] // Waiting to see if it exits.. Stupid, but stable(r) if workflowExecution.ExecutionSource != "default" { - log.Printf("Handling NON-default execution source %s - NOT waiting and validating!", workflowExecution.ExecutionSource) + log.Printf("[INFO] Handling NON-default execution source %s - NOT waiting and validating!", workflowExecution.ExecutionSource) } else if workflowExecution.ExecutionSource == "default" { time.Sleep(2 * time.Second) @@ -850,12 +851,13 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { appname = strings.Replace(appname, ".", "-", -1) appversion = strings.Replace(appversion, ".", "-", -1) - image := fmt.Sprintf("%s:%s_%s", baseimagename, action.AppName, action.AppVersion) + image := fmt.Sprintf("%s:%s_%s", baseimagename, strings.ToLower(action.AppName), action.AppVersion) if strings.Contains(image, " ") { image = strings.ReplaceAll(image, " ", "-") } - identifier := fmt.Sprintf("%s_%s_%s_%s", appname, appversion, action.ID, workflowExecution.ExecutionId) + // Added UUID to identifier just in case + identifier := fmt.Sprintf("%s_%s_%s_%s_%s", appname, appversion, action.ID, workflowExecution.ExecutionId, uuid.NewV4()) if strings.Contains(identifier, " ") { identifier = strings.ReplaceAll(identifier, " ", "-") } @@ -938,15 +940,15 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { } // Uses a few ways of getting / checking if an app is available - // 1. Try original - // 2. Go to lowercase + // 1. Try original with lowercase + // 2. Go to original // 3. Add remote repo location // 4. Actually download last repo images := []string{ - fmt.Sprintf("%s/%s:%s_%s", registryName, baseimagename, strings.ToLower(action.AppName), action.AppVersion), image, - fmt.Sprintf("%s:%s_%s", baseimagename, strings.ToLower(action.AppName), action.AppVersion), + fmt.Sprintf("%s:%s_%s", baseimagename, action.AppName, action.AppVersion), + fmt.Sprintf("%s/%s:%s_%s", registryName, baseimagename, strings.ToLower(action.AppName), action.AppVersion), } // If cleanup is set, it should run for efficiency @@ -959,6 +961,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { } log.Printf("[WARNING] Failed CLEANUP execution. Downloading image remotely.") + image = images[2] reader, err := dockercli.ImagePull(context.Background(), image, pullOptions) if err != nil { log.Printf("[ERROR] Failed getting %s. The couldn't be find locally, AND is missing.", image) @@ -996,7 +999,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { } } else { - err = deployApp(dockercli, image, identifier, env, workflowExecution) + err = deployApp(dockercli, images[0], identifier, env, workflowExecution) if err != nil { if strings.Contains(err.Error(), "exited prematurely") { shutdown(workflowExecution, action.ID, err.Error(), true) @@ -1004,7 +1007,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { // Trying to replace with lowercase to deploy again. This seems to work with Dockerhub well. // FIXME: Should try to remotely download directly if this persists. - image = fmt.Sprintf("%s:%s_%s", baseimagename, strings.ToLower(action.AppName), action.AppVersion) + image = images[1] if strings.Contains(image, " ") { image = strings.ReplaceAll(image, " ", "-") } @@ -1015,7 +1018,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { shutdown(workflowExecution, action.ID, err.Error(), true) } - image = fmt.Sprintf("%s/%s:%s_%s", registryName, baseimagename, strings.ToLower(action.AppName), action.AppVersion) + image = images[2] if strings.Contains(image, " ") { image = strings.ReplaceAll(image, " ", "-") } @@ -1026,7 +1029,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { shutdown(workflowExecution, action.ID, err.Error(), true) } - log.Printf("[WARNING] Failed deploying image THRICE. Attempting to download the latter as last resort.") + log.Printf("[WARNING] Failed deploying image THREE TIMES. Attempting to download the latter as last resort.") reader, err := dockercli.ImagePull(context.Background(), image, pullOptions) if err != nil { log.Printf("[ERROR] Failed getting %s. The couldn't be find locally, AND is missing.", image) From 62d4c0f60276f7c0293caf13ec1f92fd6c2a79d8 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 30 Mar 2021 19:57:21 +0200 Subject: [PATCH 54/57] Downgraded baseimage to 0.8.64 --- backend/go-app/go.mod | 2 +- backend/go-app/go.sum | 6 + docker-compose.yml | 10 +- frontend/src/defaultCytoscapeStyle.js | 1 - frontend/src/views/AngularWorkflow.jsx | 392 ++++++++----- frontend/src/views/AppCreator.jsx | 4 +- frontend/src/views/MyView.jsx | 39 +- frontend/src/views/Workflows.jsx | 758 ++++++++++++++----------- 8 files changed, 698 insertions(+), 514 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index fa5a2e35..4e64fc86 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -18,7 +18,7 @@ require ( github.com/docker/docker v1.13.1 github.com/docker/go-connections v0.4.0 github.com/docker/go-units v0.4.0 // indirect - github.com/frikky/shuffle-shared v0.0.20 + github.com/frikky/shuffle-shared v0.0.23 github.com/ghodss/yaml v1.0.0 github.com/go-git/go-billy/v5 v5.0.0 github.com/go-git/go-git/v5 v5.0.0 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index a20b5e4c..cc0824a0 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -90,6 +90,12 @@ github.com/frikky/shuffle-shared v0.0.12 h1:+0EIfThmK47Po+LogPYZR4XjbS4Ds19WNMFu github.com/frikky/shuffle-shared v0.0.12/go.mod h1:SEY432/xs4oBkOUnGwxiYKCZWZLRaheGInhAoW7N8ww= github.com/frikky/shuffle-shared v0.0.15 h1:508ceeEHfPBMCC8/K4Zve3kwRQqiXNJSw6+BDoq9X4E= github.com/frikky/shuffle-shared v0.0.15/go.mod h1:SEY432/xs4oBkOUnGwxiYKCZWZLRaheGInhAoW7N8ww= +github.com/frikky/shuffle-shared v0.0.20 h1:y6JlPnQDq//elICWvVfUfJyU9gH3fSpQmPy+agqZ5sA= +github.com/frikky/shuffle-shared v0.0.20/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= +github.com/frikky/shuffle-shared v0.0.21 h1:xj/XPsXTa2rx41mm4nUc7+2K9RGkq2/mpjSPtIfpjE4= +github.com/frikky/shuffle-shared v0.0.21/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= +github.com/frikky/shuffle-shared v0.0.22 h1:TFMcJCNmOOSneMMWbg5dNzp2z6m0aZLROuL+bzVToRE= +github.com/frikky/shuffle-shared v0.0.22/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= github.com/getkin/kin-openapi v0.8.0 h1:a6TQjTqwkyscC4/hShJX7WhCVE+4bi9lzw61XHQW5hE= github.com/getkin/kin-openapi v0.8.0/go.mod h1:zZQMFkVgRHCdhgb6ihCTIo9dyDZFvX0k/xAKqw1FhPw= github.com/getkin/kin-openapi v0.52.0 h1:6WqsF5d6PfJ8AscdD+9Rtb2RP2iBWyC7V6GcjssWg7M= diff --git a/docker-compose.yml b/docker-compose.yml index 45656376..1e208eb2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,8 @@ version: '3' services: frontend: - build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.70 + #build: ./frontend + image: ghcr.io/frikky/shuffle-frontend:0.8.64 container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -16,8 +16,8 @@ services: depends_on: - backend backend: - build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.70 + #build: ./backend + image: ghcr.io/frikky/shuffle-backend:0.8.64 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: @@ -47,7 +47,7 @@ services: - database orborus: #build: ./functions/onprem/orborus - image: ghcr.io/frikky/shuffle-orborus:0.8.70 + image: ghcr.io/frikky/shuffle-orborus:0.8.64 container_name: shuffle-orborus hostname: shuffle-orborus networks: diff --git a/frontend/src/defaultCytoscapeStyle.js b/frontend/src/defaultCytoscapeStyle.js index f2538a94..2732e381 100644 --- a/frontend/src/defaultCytoscapeStyle.js +++ b/frontend/src/defaultCytoscapeStyle.js @@ -55,7 +55,6 @@ const data = [{ css: { 'width': '30px', 'height': '30px', - 'font-size': '0px', }, }, { diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 8082415d..dcc792fd 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -10,8 +10,7 @@ import { useBeforeunload } from 'react-beforeunload'; import NestedMenuItem from "material-ui-nested-menu-item"; import {TextField, Drawer, Button, Paper, Grid, Tabs, InputAdornment, Tab, ButtonBase, Tooltip, Select, MenuItem, Divider, Dialog, Modal, DialogActions, DialogTitle, InputLabel, DialogContent, FormControl, IconButton, Menu, Input, FormGroup, FormControlLabel, Typography, Checkbox, Breadcrumbs, CircularProgress, Switch, Fade} from '@material-ui/core'; - -import {ArrowUpward as ArrowUpwardIcon, Visibility as VisibilityIcon, Done as DoneIcon, Close as CloseIcon, Error as ErrorIcon, FindReplace as FindreplaceIcon, ArrowLeft as ArrowLeftIcon, Cached as CachedIcon, DirectionsRun as DirectionsRunIcon, Add as AddIcon, Polymer as PolymerIcon, FormatListNumbered as FormatListNumberedIcon, Create as CreateIcon, PlayArrow as PlayArrowIcon, AspectRatio as AspectRatioIcon, MoreVert as MoreVertIcon, Apps as AppsIcon, Schedule as ScheduleIcon, FavoriteBorder as FavoriteBorderIcon, Pause as PauseIcon, Delete as DeleteIcon, AddCircleOutline as AddCircleOutlineIcon, Save as SaveIcon, KeyboardArrowLeft as KeyboardArrowLeftIcon, KeyboardArrowRight as KeyboardArrowRightIcon, ArrowBack as ArrowBackIcon, Settings as SettingsIcon, LockOpen as LockOpenIcon, ExpandMore as ExpandMoreIcon, VpnKey as VpnKeyIcon} from '@material-ui/icons'; +import {GetApp as GetAppIcon, Search as SearchIcon, ArrowUpward as ArrowUpwardIcon, Visibility as VisibilityIcon, Done as DoneIcon, Close as CloseIcon, Error as ErrorIcon, FindReplace as FindreplaceIcon, ArrowLeft as ArrowLeftIcon, Cached as CachedIcon, DirectionsRun as DirectionsRunIcon, Add as AddIcon, Polymer as PolymerIcon, FormatListNumbered as FormatListNumberedIcon, Create as CreateIcon, PlayArrow as PlayArrowIcon, AspectRatio as AspectRatioIcon, MoreVert as MoreVertIcon, Apps as AppsIcon, Schedule as ScheduleIcon, FavoriteBorder as FavoriteBorderIcon, Pause as PauseIcon, Delete as DeleteIcon, AddCircleOutline as AddCircleOutlineIcon, Save as SaveIcon, KeyboardArrowLeft as KeyboardArrowLeftIcon, KeyboardArrowRight as KeyboardArrowRightIcon, ArrowBack as ArrowBackIcon, Settings as SettingsIcon, LockOpen as LockOpenIcon, ExpandMore as ExpandMoreIcon, VpnKey as VpnKeyIcon} from '@material-ui/icons'; import * as cytoscape from 'cytoscape'; import * as edgehandles from 'cytoscape-edgehandles'; @@ -90,6 +89,8 @@ const AngularWorkflow = (props) => { const alert = useAlert() const borderRadius = 5 const theme = useTheme(); + const green = "#86c142" + const yellow = "#FECC00" const [bodyWidth, bodyHeight] = useWindowSize(); @@ -97,7 +98,6 @@ const AngularWorkflow = (props) => { const [cystyle, ] = useState(cytoscapestyle) const [cy, setCy] = React.useState() - const [appSearch, setAppSearch] = React.useState("") const [currentView, setCurrentView] = React.useState(0) const [triggerAuthentication, setTriggerAuthentication] = React.useState({}) const [triggerFolders, setTriggerFolders] = React.useState([]) @@ -284,14 +284,14 @@ const AngularWorkflow = (props) => { }) .then((response) => { if (response.status !== 200) { - console.log("Status not 200 for WORKFLOW EXECUTION :O!") + console.log("Status not 200 for APIKEY gen :O!") } return response.json() }) .then((responseJson) => { setUserSettings(responseJson) - }) + }) .catch(error => { console.log(error) }); @@ -308,7 +308,7 @@ const AngularWorkflow = (props) => { }) .then((response) => { if (response.status !== 200) { - console.log("Status not 200 for WORKFLOW EXECUTION :O!") + console.log("Status not 200 for get settings :O!") } return response.json() @@ -477,7 +477,7 @@ const AngularWorkflow = (props) => { }) .then((response) => { if (response.status !== 200) { - console.log("Status not 200 for WORKFLOW EXECUTION :O!") + console.log("Status not 200 for ABORT EXECUTION :O!") } else { alert.success("Execution aborted") } @@ -798,6 +798,21 @@ const AngularWorkflow = (props) => { workflow.errors = responseJson.errors if (responseJson.errors.length === 0) { workflow.isValid = true + workflow.is_valid = true + + //console.log("ELEMENTS: ", cy.elements()) + //const setupGraph = () => { + const cyelements = cy.elements() + for (var i = 0; i < cyelements.length; i++) { + cyelements[i].removeStyle() + cyelements[i].data().is_valid = true + cyelements[i].data().errors = [] + } + + for (var key in workflow.actions) { + workflow.actions[key].is_valid = true + workflow.actions[key].errors = [] + } } for (var key in workflow.errors) { @@ -1049,7 +1064,7 @@ const AngularWorkflow = (props) => { setAuthLoaded(true) } else { setAuthLoaded(true) - alert.error("Failed getting authentications") + //alert.error("Failed getting authentications") } }) .catch(error => { @@ -1119,6 +1134,7 @@ const AngularWorkflow = (props) => { if (responseJson.isValid === undefined) { responseJson.isValid = true } + if (responseJson.errors === undefined) { responseJson.errors = [] } @@ -1219,10 +1235,21 @@ const AngularWorkflow = (props) => { } // Comparing locations between nodes and setting views - const onNodeDrag = (event, newAppAuth) => { + const onNodeDrag = (event) => { //console.log("DRAGGING: ", event.target) //console.log("LEN2: ", event.target.edges.length) + const nodedata = event.target.data() + if (nodedata.app_name == "Shuffle Tools" || nodedata.app_name == "Testing") { + //console.log("NODE: ", + //selector: `node[app_name="Shuffle Tools"]`, + console.log(event.target) + + // 1. Find location of node + // 2. Check if it's within view of another node (inside) + // 3. If it is, then hide text + } + /* event.target.animate({ @@ -1258,7 +1285,7 @@ const AngularWorkflow = (props) => { //const branch = workflow.branches.filter(branch => branch.source_id === data.id || branch.destination_id === data.id) console.log("NODE: ", data) - console.log("APPAUTH: ", newAppAuth) + //console.log("APPAUTH: ", newAppAuth) //console.log("BRANCHES: ", branch) if (data.type === "ACTION") { @@ -1682,15 +1709,21 @@ const AngularWorkflow = (props) => { return } + // App length necessary cus of cy initialization + //console.log("PRE ELEMENTS: !", workflow.actions, graphSetup, apps, authLoaded, cy) if (elements.length === 0 && workflow.actions !== undefined && !graphSetup && Object.getOwnPropertyNames(workflow).length > 0) { setGraphSetup(true) setupGraph() + + //console.log("IN ELEMENT CHECK!") } else if (!established && cy !== undefined && apps !== null && apps !== undefined && apps.length > 0 && Object.getOwnPropertyNames(workflow).length > 0 && authLoaded){ //This part has to load LAST, as it's kind of not async. //This means we need everything else to happen first. // //console.log("IN THIS PART AGAIN") + + //console.log("IN ESTABLISHED!") setEstablished(true) cy.edgehandles({ @@ -1719,7 +1752,7 @@ const AngularWorkflow = (props) => { cy.on('mouseout', 'node', (e) => onNodeHoverOut(e)) // Handles dragging - //cy.on('drag', 'node', (e) => onNodeDrag(e)) + cy.on('drag', 'node', (e) => onNodeDrag(e)) //cy.on('mouseover', 'node', () => $(targetElement).addClass('mouseover')); @@ -1801,6 +1834,7 @@ const AngularWorkflow = (props) => { node.data.type = "ACTION" node.isStartNode = action["id"] === workflow.start + var example = "" if (action.example !== undefined && action.example !== null && action.example.length > 0) { example = action.example @@ -1808,6 +1842,8 @@ const AngularWorkflow = (props) => { node.data.example = example + //node.data.is_valid = false + return node; }) @@ -2094,7 +2130,7 @@ const AngularWorkflow = (props) => {
{ }}> -
+
{ setNewVariableName(variable.name) @@ -2161,11 +2197,12 @@ const AngularWorkflow = (props) => {
{ }}> -
+
{ - setNewVariableName(variable.name) - setExecutionVariablesModalOpen(true)}}> + setNewVariableName(variable.name) + setExecutionVariablesModalOpen(true) + }}> Name: {variable.name}
@@ -2228,7 +2265,7 @@ const AngularWorkflow = (props) => { const HandleLeftView = () => { // Defaults to apps. - var thisview = + var thisview = if (currentView === 1) { thisview = } else if (currentView === 2) { @@ -2379,7 +2416,7 @@ const AngularWorkflow = (props) => { : - const color = trigger.is_valid ? "green" : "orange" + const color = trigger.is_valid ? green : yellow return( { const appScrollStyle = { overflow: "scroll", - maxHeight: bodyHeight-appBarSize-55, - minHeight: bodyHeight-appBarSize-55, + maxHeight: bodyHeight-appBarSize-55-50, + minHeight: bodyHeight-appBarSize-55-50, + marginTop: 1, overflowY: "auto", overflowX: "hidden", } - const runAppSearch = (event) => { - setAppSearch(event.target.value) - setFilteredApps(apps.filter(app => app.name.toLowerCase().includes(event.target.value.trim().toLowerCase()))) - } + const AppView = (props) => { + const { allApps, prioritizedApps, filteredApps } = props; + const [visibleApps, setVisibleApps] = React.useState(prioritizedApps.concat(filteredApps.filter(innerapp => !internalIds.includes(innerapp.id)))) - const ParsedAppPaper = (props) => { - const app = props.app - const [hover, setHover] = React.useState(false) + const ParsedAppPaper = (props) => { + const app = props.app + const [hover, setHover] = React.useState(false) - // FIXME - add label to apps, as this might be slow with A LOT of apps - const maxlen = 24 - var newAppname = app.name.replace("_", " ", -1) - newAppname = newAppname.charAt(0).toUpperCase()+newAppname.substring(1) - if (newAppname.length > maxlen) { - newAppname = newAppname.slice(0, maxlen)+".." - } + const maxlen = 24 + var newAppname = app.name.replace("_", " ", -1) + newAppname = newAppname.charAt(0).toUpperCase()+newAppname.substring(1) + if (newAppname.length > maxlen) { + newAppname = newAppname.slice(0, maxlen)+".." + } - //const image = "url("+app.large_image+")" - const image = app.large_image - const newAppStyle = JSON.parse(JSON.stringify(paperAppStyle)) - const pixelSize = !hover ? "2px" : "4px" - newAppStyle.borderLeft = app.is_valid ? `${pixelSize} solid green` : `${pixelSize} solid orange` + //const image = "url("+app.large_image+")" + const image = app.large_image + const newAppStyle = JSON.parse(JSON.stringify(paperAppStyle)) + const pixelSize = !hover ? "2px" : "4px" + newAppStyle.borderLeft = app.is_valid ? `${pixelSize} solid ${green}` : `${pixelSize} solid ${yellow}` - return ( - {handleAppDrag(e, app)}} - onStop={(e) => {handleDragStop(e, app)}} - key={app.id} - dragging={false} - position={{ - x: 0, - y: 0, - }} - > - {setHover(true)}} onMouseOut={() => {setHover(false)}}> - - - {newAppname} - - - -

{newAppname}

+ return ( + {handleAppDrag(e, app)}} + onStop={(e) => {handleDragStop(e, app)}} + key={app.id} + dragging={false} + position={{ + x: 0, + y: 0, + }} + > + {setHover(true)}} onMouseOut={() => {setHover(false)}}> + + + {newAppname} - - Version: {app.app_version} - - - {app.description} + + +

{newAppname}

+ + + Version: {app.app_version} + + + {app.description} + - - ) - } + ) + } + + const runSearch = (event) => { + if (event.target.value.length > 0) { + setVisibleApps(allApps.filter(app => app.name.toLowerCase().includes(event.target.value.trim().toLowerCase()))) + } else { + setVisibleApps(prioritizedApps.concat(filteredApps.filter(innerapp => !internalIds.includes(innerapp.id)))) + } + } - const AppView = () => { return(
+ + + + + + ) + }} + fullWidth + color="primary" + placeholder={"Search Apps"} + id="appsearch" + onBlur={(event) => { + runSearch(event) + }} + />
- {/* - { - runAppSearch(event) - }} - /> - */} - {prioritizedApps.map((app, index) => { - return( - - ) - })} - {filteredApps.filter(innerapp => !internalIds.includes(innerapp.id)).map((app, index) => { - if (app.invalid) { - return null - } + {visibleApps.map((app, index) => { + if (app.invalid) { + return null + } - return( - - ) - })} + return( + + ) + })}
@@ -2979,10 +3022,10 @@ const AngularWorkflow = (props) => { const AppActionArguments = (props) => { const [selectedActionParameters, setSelectedActionParameters] = React.useState([]) const [selectedVariableParameter, setSelectedVariableParameter] = React.useState("") - const [showDropdown, setShowDropdown] = React.useState(false) - const [showDropdownNumber, setShowDropdownNumber] = React.useState(0) const [actionlist, setActionlist] = React.useState([]) const [jsonList, setJsonList] = React.useState([]) + const [showDropdown, setShowDropdown] = React.useState(false) + const [showDropdownNumber, setShowDropdownNumber] = React.useState(0) const [showAutocomplete, setShowAutocomplete] = React.useState(false) const [menuPosition, setMenuPosition] = useState(null) @@ -3865,6 +3908,7 @@ const AngularWorkflow = (props) => { } tmpitem = tmpitem.charAt(0).toUpperCase()+tmpitem.substring(1) + tmpitem = tmpitem.replaceAll("_", " ") const description = data.description === undefined ? "" : data.description return ( @@ -4121,7 +4165,7 @@ const AngularWorkflow = (props) => {
-

{selectedAction.app_name}

+

{selectedAction.app_name.replaceAll("_", " ")}

{ console.log("FIND EXAMPLE RESULTS FOR ", selectedAction) @@ -4969,7 +5013,7 @@ const AngularWorkflow = (props) => { return ( {}}> -
+
{ setSourceValue(condition.source) @@ -5371,6 +5415,11 @@ const AngularWorkflow = (props) => { } const SubflowSidebar = () => { + const [menuPosition, setMenuPosition] = useState(null) + const [showDropdown, setShowDropdown] = React.useState(false) + const [showDropdownNumber, setShowDropdownNumber] = React.useState(0) + const [showAutocomplete, setShowAutocomplete] = React.useState(false) + if (Object.getOwnPropertyNames(selectedTrigger).length > 0) { if (workflow.triggers[selectedTriggerIndex] === undefined) { return null @@ -5485,48 +5534,50 @@ const AngularWorkflow = (props) => { Explore selected workflow } -
-
-
- Select the Startnode -
-
{subworkflow === undefined || subworkflow === null || subworkflow.id === undefined || subworkflow.actions === null || subworkflow.actions === undefined || subworkflow.actions.length === 0 ? null : - { - setSubworkflowStartnode(e.target.value) - - try { - workflow.triggers[selectedTriggerIndex].parameters[3].value = e.target.value.id - } catch { - workflow.triggers[selectedTriggerIndex].parameters[3] = { - "name": "startnode", - "value": e.target.value.id, } - } + }} + fullWidth + onChange={(e) => { + setSubworkflowStartnode(e.target.value) - setWorkflow(workflow) - //setUpdate(Math.random()) - }} - style={{backgroundColor: inputColor, color: "white", height: "50px"}} - > - {subworkflow.actions.map((action, index) => { - //console.log(action) - return ( - parent.id === action.id)} key={index} style={{backgroundColor: inputColor, color: "white"}} value={action}> - {action.label} - - ) - })} - + try { + workflow.triggers[selectedTriggerIndex].parameters[3].value = e.target.value.id + } catch { + workflow.triggers[selectedTriggerIndex].parameters[3] = { + "name": "startnode", + "value": e.target.value.id, + } + } + + setWorkflow(workflow) + //setUpdate(Math.random()) + }} + style={{backgroundColor: inputColor, color: "white", height: "50px"}} + > + {subworkflow.actions.map((action, index) => { + //console.log(action) + return ( + parent.id === action.id)} key={index} style={{backgroundColor: inputColor, color: "white"}} value={action}> + {action.label} + + ) + })} + + }
@@ -5543,6 +5594,21 @@ const AngularWorkflow = (props) => { maxWidth: "95%", fontSize: "1em", }, + endAdornment: ( + + + { + setMenuPosition({ + top: event.pageY+10, + left: event.pageX+10, + }) + //setShowDropdownNumber(count) + setShowDropdown(true) + setShowAutocomplete(true) + }}/> + + + ) }} rows="6" multiline @@ -6321,7 +6387,7 @@ const AngularWorkflow = (props) => { return null } - const cytoscapeViewWidths = 750 + const cytoscapeViewWidths = 800 const bottomBarStyle = { position: "fixed", right: 20, @@ -6551,7 +6617,7 @@ const AngularWorkflow = (props) => { @@ -6562,7 +6628,7 @@ const AngularWorkflow = (props) => { - + + {workflow.public ? + + + + + + : null} {/* */} {workflow.configuration !== null && workflow.configuration !== undefined && workflow.configuration.exit_on_error !== undefined ? : null}
@@ -6830,7 +6918,7 @@ const AngularWorkflow = (props) => { {workflowExecutions.length > 0 ?
{workflowExecutions.map((data, index) => { - const statusColor = data.status === "FINISHED" ? "green" : data.status === "ABORTED" || data.status === "FAILED" ? "red" : "orange" + const statusColor = data.status === "FINISHED" ? green : data.status === "ABORTED" || data.status === "FAILED" ? "red" : yellow const timeElapsed = data.completed_at-data.started_at const resultsLength = data.results !== undefined && data.results !== null ? data.results.length : 0 @@ -7006,7 +7094,7 @@ const AngularWorkflow = (props) => { const curapp = apps.find(a => a.name === data.action.app_name && a.app_version === data.action.app_version) const imgsize = 50 - const statusColor = data.status === "FINISHED" || data.status === "SUCCESS" ? "green" : data.status === "ABORTED" || data.status === "FAILURE" ? "red" : "orange" + const statusColor = data.status === "FINISHED" || data.status === "SUCCESS" ? green : data.status === "ABORTED" || data.status === "FAILURE" ? "red" : yellow var imgSrc = curapp === undefined ? "" : curapp.large_image if (imgSrc.length === 0 && workflow.actions !== undefined && workflow.actions !== null) { @@ -7116,7 +7204,7 @@ const AngularWorkflow = (props) => { // This sucks :) const curapp = !codeModalOpen ? {} : selectedResult.action.app_name === "shuffle-subflow" ? triggers[1] : selectedResult.action.app_name === "User Input" ? triggers[2] : apps.find(a => a.name === selectedResult.action.app_name && a.app_version === selectedResult.action.app_version) const imgsize = 50 - const statusColor = !codeModalOpen ? "red" : selectedResult.status === "FINISHED" || selectedResult.status === "SUCCESS" ? "green" : selectedResult.status === "ABORTED" || selectedResult.status === "FAILURE" ? "red" : "orange" + const statusColor = !codeModalOpen ? "red" : selectedResult.status === "FINISHED" || selectedResult.status === "SUCCESS" ? green : selectedResult.status === "ABORTED" || selectedResult.status === "FAILURE" ? "red" : yellow const validate = !codeModalOpen ? "" : validateJson(selectedResult.result.trim()) if (validate.valid && typeof(validate.result) === "string") { validate.result = JSON.parse(validate.result) @@ -7287,7 +7375,7 @@ const AngularWorkflow = (props) => { - const newView = isLoggedIn ? + const newView = //isLoggedIn ?
{leftView} @@ -7313,10 +7401,13 @@ const AngularWorkflow = (props) => {
+ + /* :
TMP FOR NOT LOGGED IN
+ */ const executionVariableModal = executionVariablesModalOpen ? { : null - const loadedCheck = isLoaded && isLoggedIn && workflowDone ? + //const loadedCheck = isLoaded && isLoggedIn && workflowDone ? + const loadedCheck = isLoaded && workflowDone ?
{newView} {variablesModal} diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 38f40744..bbf85301 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -1340,7 +1340,9 @@ const AppCreator = (props) => { placeholder={'Query name'} helperText={Click required switch} onBlur={(e) => { - urlPathQueries[index].name = e.target.value + console.log("IN BLUR: ", e.target.value) + urlPathQueries[index].name = e.target.value.replaceAll("=", "") + setUrlPathQueries(urlPathQueries) }} InputProps={{ diff --git a/frontend/src/views/MyView.jsx b/frontend/src/views/MyView.jsx index b456c6ed..70927e69 100644 --- a/frontend/src/views/MyView.jsx +++ b/frontend/src/views/MyView.jsx @@ -68,17 +68,21 @@ const flexContainerStyle = { } const flexBoxStyle = { - width: 333, height: 125, borderRadius: 4, boxSizing: "border-box", letterSpacing: "0.4px", color: "#D6791E", + margin: 10, + flex: 1, } -const activeWorkflowStyle = {backgroundColor: "#FFF5EE"} -const availableWorkflowStyle = {backgroundColor: "#F0F3FF"} -const notificationStyle = {backgroundColor: "#E5F9FF"} +//const activeWorkflowStyle = {backgroundColor: "#FFF5EE"} +//const notificationStyle = {backgroundColor: "#E5F9FF"} +//const activeWorkflowStyle = {backgroundColor: "#3d3f43"} +const availableWorkflowStyle = {backgroundColor: "#3d3f43"} +const notificationStyle = {backgroundColor: "#3d3f43"} +const activeWorkflowStyle = {backgroundColor: "#3d3f43"} const flexContentStyle = { display: "flex", @@ -302,8 +306,6 @@ const MyView = (props) => { alignContent: "space-between", } - - const paperAppStyle = { minHeight: 130, width: "100%", @@ -573,13 +575,16 @@ const MyView = (props) => { boxColor = "#86c142" } + if (!data.previously_saved) { + boxColor = "#f85a3e" + } + const menuClick = (event) => { setOpen(!open) setAnchorEl(event.currentTarget); } var parsedName = data.name - console.log("LEN: ", parsedName.length) if (parsedName !== undefined && parsedName !== null && parsedName.length > 25) { parsedName = parsedName.slice(0,25)+".." } @@ -589,7 +594,7 @@ const MyView = (props) => { const [triggers, schedules, webhooks, subflows] = getWorkflowMeta(data) return ( - +
@@ -608,7 +613,7 @@ const MyView = (props) => { - + {triggers} @@ -616,7 +621,7 @@ const MyView = (props) => { - + @@ -1420,10 +1425,6 @@ const MyView = (props) => {

Workflows

-

THIS IS SOME WORKFLOW INFORMATION WHY ISN’T IT GROWING THE CORRECT WAY.

-
-
- {workflowButtons}
@@ -1457,6 +1458,15 @@ const MyView = (props) => {
+
+
+ This is your workflow view. Learn more about Workflows +
+
+ {workflowButtons} +
+
+
{view === "grid" && ( {workflows.map((data, index) => { @@ -1471,6 +1481,7 @@ const MyView = (props) => { )} +
) diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 9e477c29..0b30170e 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -1,8 +1,12 @@ import React, { useEffect} from 'react'; import { useInterval } from 'react-powerhooks'; +import { makeStyles } from '@material-ui/core/styles'; import {Grid, Paper, Tooltip, Divider, Button, TextField, FormControl, IconButton, Menu, MenuItem, FormControlLabel, Chip, Switch, Typography, Zoom, CircularProgress, Dialog, DialogTitle, DialogActions, DialogContent} from '@material-ui/core'; -import {Cached as CachedIcon, GetApp as GetAppIcon, Apps as AppsIcon, Edit as EditIcon, MoreVert as MoreVertIcon, PlayArrow as PlayArrowIcon, Add as AddIcon, Publish as PublishIcon, CloudDownload as CloudDownloadIcon} from '@material-ui/icons'; +import {FileCopy as FileCopyIcon, Delete as DeleteIcon, BubbleChart as BubbleChartIcon, Restore as RestoreIcon, Cached as CachedIcon, GetApp as GetAppIcon, Apps as AppsIcon, Edit as EditIcon, MoreVert as MoreVertIcon, PlayArrow as PlayArrowIcon, Add as AddIcon, Publish as PublishIcon, CloudUpload as CloudUploadIcon, CloudDownload as CloudDownloadIcon} from '@material-ui/icons'; +//import {Search as SearchIcon, ArrowUpward as ArrowUpwardIcon, Visibility as VisibilityIcon, Done as DoneIcon, Close as CloseIcon, Error as ErrorIcon, FindReplace as FindreplaceIcon, ArrowLeft as ArrowLeftIcon, Cached as CachedIcon, DirectionsRun as DirectionsRunIcon, Add as AddIcon, Polymer as PolymerIcon, FormatListNumbered as FormatListNumberedIcon, Create as CreateIcon, PlayArrow as PlayArrowIcon, AspectRatio as AspectRatioIcon, MoreVert as MoreVertIcon, Apps as AppsIcon, Schedule as ScheduleIcon, FavoriteBorder as FavoriteBorderIcon, Pause as PauseIcon, Delete as DeleteIcon, AddCircleOutline as AddCircleOutlineIcon, Save as SaveIcon, KeyboardArrowLeft as KeyboardArrowLeftIcon, KeyboardArrowRight as KeyboardArrowRightIcon, ArrowBack as ArrowBackIcon, Settings as SettingsIcon, LockOpen as LockOpenIcon, ExpandMore as ExpandMoreIcon, VpnKey as VpnKeyIcon} from '@material-ui/icons'; + +import {DataGrid, GridToolbarContainer, GridDensitySelector, GridToolbar} from '@material-ui/data-grid'; //import JSONPretty from 'react-json-pretty'; //import JSONPrettyMon from 'react-json-pretty/dist/monikai' @@ -13,10 +17,86 @@ import {Link} from 'react-router-dom'; import { useAlert } from "react-alert"; import ChipInput from 'material-ui-chip-input' import uuid from "uuid" +import CytoscapeWrapper from '../components/RenderCytoscape' + +//import mobileImage from '../assets/img/mobile.svg'; +//import bagImage from '../assets/img/bag.svg'; +//import bookImage from '../assets/img/book.svg'; const inputColor = "#383B40" const surfaceColor = "#27292D" +const flexContainerStyle = { + display: "flex", + flexDirection: "row", + justifyContent: "left", + alignContent: "space-between", +} + +const flexBoxStyle = { + height: 125, + borderRadius: 4, + boxSizing: "border-box", + letterSpacing: "0.4px", + color: "#D6791E", + margin: 10, + flex: 1, +} + +const useStyles = makeStyles((theme) => ({ + root: { + border: 0, + '& .MuiDataGrid-columnsContainer': { + backgroundColor: theme.palette.type === 'light' ? '#fafafa' : '#1d1d1d', + }, + '& .MuiDataGrid-iconSeparator': { + display: 'none', + }, + '& .MuiDataGrid-colCell, .MuiDataGrid-cell': { + borderRight: `1px solid ${ + theme.palette.type === 'light' ? 'white' : '#303030' + }`, + }, + '& .MuiDataGrid-columnsContainer, .MuiDataGrid-cell': { + borderBottom: `1px solid ${ + theme.palette.type === 'light' ? '#f0f0f0' : '#303030' + }`, + }, + '& .MuiDataGrid-cell': { + color: + theme.palette.type === 'light' + ? 'white' + : 'rgba(255,255,255,0.65)', + }, + '& .MuiPaginationItem-root, .MuiTablePagination-actions, .MuiTablePagination-caption': { + borderRadius: 0, + color: "white", + }, + }, +})); + +//const activeWorkflowStyle = {backgroundColor: "#FFF5EE"} +//const notificationStyle = {backgroundColor: "#E5F9FF"} +//const activeWorkflowStyle = {backgroundColor: "#3d3f43"} +const availableWorkflowStyle = {backgroundColor: "#3d3f43"} +const notificationStyle = {backgroundColor: "#3d3f43"} +const activeWorkflowStyle = {backgroundColor: "#3d3f43"} + +const fontSize_16 = {fontSize: "16px",} +const counterStyle = {fontSize: "36px",fontWeight:"bold"} +const blockRightStyle = {textAlign: "right",padding: "20px 20px 0px 0px",width:"100%"} + +const flexContentStyle = { + display: "flex", + flexDirection: "row" +} + +const iconStyle = { + width: "75px", + height: "75px", + padding: "20px" +} + export const validateJson = (showResult) => { //showResult = showResult.split(" None").join(" \"None\"") showResult = showResult.split(" False").join(" false") @@ -54,6 +134,7 @@ const Workflows = (props) => { document.title = "Shuffle - Workflows" const alert = useAlert() + const classes = useStyles(); var upload = "" const [file, setFile] = React.useState(""); @@ -84,16 +165,9 @@ const Workflows = (props) => { const [executionLoading, setExecutionLoading] = React.useState(false) const [importLoading, setImportLoading] = React.useState(false) const [isDropzone, setIsDropzone] = React.useState(false); + const [view, setView] = React.useState("grid") const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" - const { start, stop } = useInterval({ - duration: 5000, - startImmediate: false, - callback: () => { - //getWorkflowExecution(selectedWorkflow.id) - } - }) - const deleteModal = deleteModalOpen ? { .then((response) => { if (response.status !== 200) { console.log("Status not 200 for workflows :O!: ", response.status) + + if (isCloud) { + window.location.pathname = "/login" + } + alert.info("Failed getting workflows.") setWorkflowDone(true) @@ -244,7 +323,7 @@ const Workflows = (props) => { minWidth: 1024, maxWidth: 1024, margin: "auto", - maxHeight: "90vh", + /*maxHeight: "90vh",*/ } const emptyWorkflowStyle = { @@ -272,78 +351,38 @@ const Workflows = (props) => { overflowY: "auto", } + const paperAppContainer = { + display: "flex", + flexWrap: 'wrap', + alignContent: "space-between", + } + const paperAppStyle = { - minHeight: "100px", - minWidth: "100%", - maxWidth: "100%", - marginTop: "5px", + minHeight: 130, + width: "100%", color: "white", backgroundColor: surfaceColor, + padding: "12px 12px 0px 15px", borderRadius: 5, - padding: 10, - cursor: "pointer", display: "flex", + boxSizing: "border-box", + position: "relative", } - const getWorkflowExecution = (id) => { - setExecutionLoading(true) - fetch(globalUrl+"/api/v1/workflows/"+id+"/executions", { - method: 'GET', - headers: { - 'Content-Type': 'application/json', - 'Accept': 'application/json', - }, - credentials: "include", - }) - .then((response) => { - setExecutionLoading(false) - if (response.status !== 200) { - console.log("Status not 200 for WORKFLOW EXECUTION :O!") - } - - return response.json() - }) - .then((responseJson) => { - if (responseJson.success === false) { - alert.error("Failed getting executions") - } else { - if (responseJson.length > 0) { - setSelectedExecution(responseJson[0]) - setWorkflowExecutions(responseJson) - } else { - //alert.info("Couldn't find executions for the workflow") - setSelectedExecution({}) - setWorkflowExecutions([]) - } - } - }) - .catch(error => { - setExecutionLoading(false) - alert.error(error.toString()) - }); + const gridContainer = { + height: "auto", + color: "white", + margin: "10px", + backgroundColor: surfaceColor, } - const abortExecution = (workflowid, executionid) => { - alert.success("Aborting execution") - fetch(globalUrl+"/api/v1/workflows/"+workflowid+"/executions/"+executionid+"/abort", { - method: 'GET', - headers: { - 'Content-Type': 'application/json', - 'Accept': 'application/json', - }, - credentials: "include", - }) - .then((response) => { - if (response.status !== 200) { - console.log("Status not 200 for WORKFLOW EXECUTION :O!") - } - //getWorkflowExecution(workflowid) - - return response.json() - }) - .catch(error => { - alert.error(error.toString()) - }); + const workflowActionStyle = { + flex: "1", + display: "flex", + width: 150, + height: 44, + justifyContent: "space-between", + overflow: "hidden" } const executeWorkflow = (id) => { @@ -372,13 +411,6 @@ const Workflows = (props) => { .catch(error => { alert.error(error.toString()) }); - - if (id === selectedWorkflow.id) { - sleep(2000).then(() => { - stop() - start() - }) - } } function sleep (time) { @@ -393,7 +425,7 @@ const Workflows = (props) => { const sanitizeWorkflow = (data) => { data["owner"] = "" - console.log(data) + console.log("Sanitize start: ", data) if (data.triggers !== null && data.triggers !== undefined) { for (var key in data.triggers) { const trigger = data.triggers[key] @@ -457,7 +489,8 @@ const Workflows = (props) => { } } - data.actions[key].environment = isCloud ? "cloud" : "Shuffle" + //data.actions[key].environment = isCloud ? "cloud" : "Shuffle" + data.actions[key].environment = "" data.actions[key].id = newId } } @@ -482,16 +515,18 @@ const Workflows = (props) => { // These are backwards.. True = saved before. Very confuse. data["previously_saved"] = false data["first_save"] = false - console.log(data) + console.log("Sanitize end: ", data) return data } const exportWorkflow = (data) => { - console.log("export") let exportFileDefaultName = data.name+'.json'; data = sanitizeWorkflow(data) + //console.log("EXPORT: ", data) + //return + let dataStr = JSON.stringify(data) let dataUri = 'data:application/json;charset=utf-8,'+ encodeURIComponent(dataStr); let linkElement = document.createElement('a'); @@ -600,7 +635,6 @@ const Workflows = (props) => { }); } - // dropdown with copy etc I guess const WorkflowPaper = (props) => { const { data } = props; const [open, setOpen] = React.useState(false); @@ -611,12 +645,11 @@ const Workflows = (props) => { boxWidth = "4px" } - var boxColor = "#f85a3e" + var boxColor = "#FECC00" if (data.is_valid) { - boxColor = "green" + boxColor = "#86c142" } - //console.log(data) if (!data.previously_saved) { boxColor = "#f85a3e" } @@ -626,63 +659,110 @@ const Workflows = (props) => { setAnchorEl(event.currentTarget); } - const actions = data.actions !== null ? data.actions.length : 0 - var schedules = 0 - var webhooks = 0 - var webhookImg = "" - var scheduleImg = "" - if (data.triggers !== undefined && data.triggers !== null && data.triggers.length > 0) { - for (var key in data.triggers) { - if (data.triggers[key].app_name === "Webhook") { - webhooks += 1 - webhookImg = data.triggers[key].large_image - } else if (data.triggers[key].app_name === "Schedule") { - schedules += 1 - scheduleImg = data.triggers[key].large_image - } - } + var parsedName = data.name + if (parsedName !== undefined && parsedName !== null && parsedName.length > 25) { + parsedName = parsedName.slice(0,25)+".." } - const imgSize = 25 - //console.log("TOP INFO: ", data) - return ( - { - }}> -
- - - -
{ - if (selectedWorkflow.id !== data.id) { - //setSelectedWorkflow(data) - //getWorkflowExecution(data.id) - } - }}> - - {data.name} - -
-
- - - - { - setOpen(false) - setAnchorEl(null) - }} - > + const actions = data.actions !== null ? data.actions.length : 0 + const [triggers, schedules, webhooks, subflows] = getWorkflowMeta(data) + + return ( + + +
+ + + + {parsedName} + + + + + + + + {actions} + + + + + + + + {triggers} + + + + + + + + + + {subflows} + + + + {/* + + + + + + + : null} + {schedules > 0 ? + + + + : null} + */} + + + {data.tags !== undefined ? + data.tags.map((tag, index) => { + if (index >= 3) { + return null + } + + return ( + + ) + }) + : null} + + + {data.actions !== undefined && data.actions !== null ? + + + + + + { + setOpen(false) + setAnchorEl(null) + }} + > { setModalOpen(true) setEditingWorkflow(data) @@ -691,146 +771,57 @@ const Workflows = (props) => { if (data.tags !== undefined && data.tags !== null) { setNewWorkflowTags(JSON.parse(JSON.stringify(data.tags))) } - }} key={"change"}>{"Change details"} + }} key={"change"}> + + {"Change details"} + { - console.log("Should publish", data) - publishWorkflow(data) - }} key={"publish"}>{"Publish Workflow"} + publishWorkflow(data) + }} key={"publish"}> + + {"Publish Workflow"} + { copyWorkflow(data) setOpen(false) - }} key={"duplicate"}>{"Duplicate Workflow"} + }} key={"duplicate"}> + + {"Duplicate Workflow"} + { exportWorkflow(data) setOpen(false) - }} key={"export"}>{"Export"} - { + }} key={"export"}> + + {"Export Workflow"} + + { setDeleteModalOpen(true) setSelectedWorkflowId(data.id) setOpen(false) - }} key={"delete"}>{"Delete"} + }} key={"delete"}> + + {"Delete Workflow"} + - -
+ -
{ - if (selectedWorkflow.id !== data.id) { - //setSelectedWorkflow(data) - //getWorkflowExecution(data.id) - } - }}> - - - - - - - - - - + + + + - {data.tags !== undefined ? - data.tags.map((tag, index) => { - if (index >= 3) { - return null - } - - return ( - - ) - }) - : null} - -
- - - {data.actions !== undefined && data.actions !== null ? - - - - - - {webhooks > 0 ? - - {data.title} - - : null} - {schedules > 0 ? - - {data.title} - - : null} - - : null} - - ) - } - - const executionPaper = (data) => { - var boxWidth = "2px" - if (selectedExecution.execution_id === data.execution_id) { - boxWidth = "4px" - } - - var boxColor = "orange" - if (data.status === "ABORTED" || data.status === "UNFINISHED" || data.status === "FAILURE"){ - boxColor = "red" - } else if (data.status === "FINISHED") { - boxColor = "green" - } - - var t = new Date(data.started_at*1000) - if (data.workflow.actions === null || data.workflow.actions === undefined ) { - return null - } - - if (data.workflow.actions === null || data.workflow.actions === undefined) { - return null - } - - var actions = data.workflow.actions.length - if (data.results !== null) { - var results = data.results.length - } - - return ( - { - setSelectedExecution(data) - }}> -
- - - -
-

Status: {data.status}

- Actions: {results}/{actions} -
-
- -
+ -
- - Started: {t.toISOString()} - -
- - + : null} + + ) } + + const dividerColor = "rgb(225, 228, 232)" const resultPaperAppStyle = { @@ -1025,32 +1016,6 @@ const Workflows = (props) => { ) } - const ExecutionsView = () => { - if (workflowExecutions.length > 0) { - const sortedWorkflows = workflowExecutions.sort((a, b) => a.started_at - b.started_at).reverse() - - return ( -
- {sortedWorkflows.map(data => { - return ( - executionPaper(data) - ) - })} -
- ) - } - return ( - executionLoading ? -
- -
- : -

- Executions have been moved to the Workflow itself.
Click here to see them -

- ) - } - // Can create and set workflows const setNewWorkflow = (name, description, tags, editingWorkflow, redirect) => { @@ -1173,6 +1138,111 @@ const Workflows = (props) => { setLoadWorkflowsModalOpen(false) } + const getWorkflowMeta = (data) => { + let triggers = 0 + let schedules = 0 + let webhooks = 0 + let subflows = 0 + if (data.triggers !== undefined && data.triggers !== null && data.triggers.length > 0) { + triggers = data.triggers.length + for (let key in data.triggers) { + + if (data.triggers[key].app_name === "Webhook") { + webhooks += 1 + //webhookImg = data.triggers[key].large_image + } else if (data.triggers[key].app_name === "Schedule") { + schedules += 1 + //scheduleImg = data.triggers[key].large_image + } else if (data.triggers[key].app_name === "Subflow") { + subflows += 1 + } + } + } + + return [triggers, schedules, webhooks, subflows] + } + + const WorkflowGridView = () => { + let workflowData = ""; + if (workflows.length > 0) { + const columns = [ + { field: 'title', headerName: 'Title', width: 330, }, + { field: 'actions', headerName: 'Actions', width: 200, sortable: false, + disableClickEventBubbling: true, + renderCell: (params) => { + const data = params.row.record; + let [triggers, schedules, webhooks, subflows] = getWorkflowMeta(data); + + return + + + + + + + + executeWorkflow(data.id)} /> + + + + + {webhooks > 0 ? + + + + : null} + {schedules > 0 ? + + + + : null} + + } + }, + { field: 'tags', headerName: 'Tags', width: 390, sortable: false, + disableClickEventBubbling: true, + renderCell: (params) => { + const data = params.row.record; + return + {data.tags !== undefined ? + data.tags.map((tag, index) => { + if (index >= 3) { + return null + } + + return ( + + ) + }) + : null} + + } + }, + ]; + let rows = []; + rows = workflows.map((data, index) => { + let obj = {"id":index+1, "title":data.name, "record":data,}; + return obj; + }); + workflowData = + } + return ( +
+ {workflowData} +
+ ); + } + const modalView = modalOpen ? {
-
-

Workflows ({workflows.length})

+
+

Workflows

-
+
+ + {/* +
+
+
+
+
+
{workflows.length}
+
ACTIVE WORKFLOWS
+
+
+
+
+
+
+
+
{workflows.length}
+
AVAILABE WORKFLOWS
+
+
+
+
+
+
+
+
{workflows.length}
+
NOTIFICATIONS
+
+
+
+
+ */} + +
+
+ This is your workflow view. Learn more about Workflows +
+
{workflowButtons}
- - -
- {workflows.map((data, index) => { - return ( +
+ {view === "grid" && ( + + {workflows.map((data, index) => { + return ( - ) - })} -
+ ) + })} + + )} + + {view === "list" && ( + + )} + +
-
-
-
-

Executions: {selectedWorkflow.name}

-
- {/* -
- -
- */} -
- -
- -
-
- {/* -
-
-
-

Execution Timeline

-
-
- Collapse results
} - control={ {setCollapseJson(!collapseJson)}} />} - /> -
-
- -
- -
-
- */}
) } From 52a45268b60463710c3dcb9af383a3185639d05c Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 30 Mar 2021 19:57:43 +0200 Subject: [PATCH 55/57] Added cytoscape renderer --- frontend/src/components/RenderCytoscape.js | 146 +++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 frontend/src/components/RenderCytoscape.js diff --git a/frontend/src/components/RenderCytoscape.js b/frontend/src/components/RenderCytoscape.js new file mode 100644 index 00000000..3a88dcf8 --- /dev/null +++ b/frontend/src/components/RenderCytoscape.js @@ -0,0 +1,146 @@ +import React, {useState, useEffect, useLayoutEffect} from 'react'; +import * as cytoscape from 'cytoscape'; +import CytoscapeComponent from 'react-cytoscapejs'; +import cystyle from '../defaultCytoscapeStyle'; + +const surfaceColor = "#27292D" +const CytoscapeWrapper = (props) => { + const { globalUrl, inworkflow } = props; + + const [elements, setElements] = useState([]) + const [workflow, setWorkflow] = useState(inworkflow) + const [cy, setCy] = React.useState() + const bodyWidth = 200 + const bodyHeight = 150 + + const setupGraph = () => { + const actions = workflow.actions.map(action => { + const node = {} + node.position = action.position + node.data = action + + node.data._id = action["id"] + node.data.type = "ACTION" + node.isStartNode = action["id"] === workflow.start + + + var example = "" + if (action.example !== undefined && action.example !== null && action.example.length > 0) { + example = action.example + } + + node.data.example = example + return node; + }) + + const triggers = workflow.triggers.map(trigger => { + const node = {} + node.position = trigger.position + node.data = trigger + + node.data._id = trigger["id"] + node.data.type = "TRIGGER" + + return node; + }) + + // FIXME - tmp branch update + var insertedNodes = [].concat(actions, triggers) + const edges = workflow.branches.map((branch, index) => { + //workflow.branches[index].conditions = [{ + + const edge = { }; + var conditions = workflow.branches[index].conditions + if (conditions === undefined || conditions === null) { + conditions = [] + } + + var label = "" + if (conditions.length === 1) { + label = conditions.length+" condition" + } else if (conditions.length > 1) { + label = conditions.length+" conditions" + } + + edge.data = { + id: branch.id, + _id: branch.id, + source: branch.source_id, + target: branch.destination_id, + label: label, + conditions: conditions, + hasErrors: branch.has_errors + }; + + // This is an attempt at prettier edges. The numbers are weird to work with. + /* + //http://manual.graphspace.org/projects/graphspace-python/en/latest/demos/edge-types.html + const sourcenode = actions.find(node => node.data._id === branch.source_id) + const destinationnode = actions.find(node => node.data._id === branch.destination_id) + if (sourcenode !== undefined && destinationnode !== undefined && branch.source_id !== branch.destination_id) { + //node.data._id = action["id"] + console.log("SOURCE: ", sourcenode.position) + console.log("DESTINATIONNODE: ", destinationnode.position) + + var opposite = true + if (sourcenode.position.x > destinationnode.position.x) { + opposite = false + } else { + opposite = true + } + + edge.style = { + 'control-point-distance': opposite ? ["25%", "-75%"] : ["-10%", "90%"], + 'control-point-weight': ['0.3', '0.7'], + } + } + */ + + return edge; + }) + + setWorkflow(workflow) + + // Verifies if a branch is valid and skips others + var newedges = [] + for (var key in edges) { + var item = edges[key] + + const sourcecheck = insertedNodes.find(data => data.data.id === item.data.source) + const destcheck = insertedNodes.find(data => data.data.id === item.data.target) + if (sourcecheck === undefined || destcheck === undefined) { + continue + } + + newedges.push(item) + } + + insertedNodes = insertedNodes.concat(newedges) + setElements(insertedNodes) + } + + if (elements.length === 0) { + setupGraph() + } + + return ( + { + // FIXME: There's something specific loading when + // you do the first hover of a node. Why is this different? + //console.log("CY: ", incy) + setCy(incy) + }} + /> + ) +} + +export default CytoscapeWrapper From 92753f0be791f64624c3e459c96347340cd1b70f Mon Sep 17 00:00:00 2001 From: frikky Date: Wed, 31 Mar 2021 09:24:19 +0200 Subject: [PATCH 56/57] Added filtering to workflow view --- frontend/src/views/Workflows.jsx | 123 ++++++++++++++++++++++++++++--- 1 file changed, 114 insertions(+), 9 deletions(-) diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 0b30170e..a2655857 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -140,6 +140,7 @@ const Workflows = (props) => { const [file, setFile] = React.useState(""); const [workflows, setWorkflows] = React.useState([]); + const [filteredWorkflows, setFilteredWorkflows] = React.useState([]); const [selectedWorkflow, setSelectedWorkflow] = React.useState({}); const [selectedExecution, setSelectedExecution] = React.useState({}); const [workflowExecutions, setWorkflowExecutions] = React.useState([]); @@ -166,8 +167,81 @@ const Workflows = (props) => { const [importLoading, setImportLoading] = React.useState(false) const [isDropzone, setIsDropzone] = React.useState(false); const [view, setView] = React.useState("grid") + const [filters, setFilters] = React.useState([]) const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" + const findWorkflow = (filters) => { + if (filters.length === 0) { + setFilteredWorkflows(workflows) + return + } + + var newWorkflows = [] + for (var workflowKey in workflows) { + const curWorkflow = workflows[workflowKey] + + var found = [false] + if (curWorkflow.tags === undefined || curWorkflow.tags === null) { + found = filters.map(filter => curWorkflow.name.toLowerCase().includes(filter)) + } else { + found = filters.map(filter => curWorkflow.name.toLowerCase().includes(filter.toLowerCase()) || curWorkflow.tags.includes(filter)) + } + //console.log("FOUND: ", found) + //if (found) { + if (found.every(v => v === true)) { + newWorkflows.push(curWorkflow) + continue + } + } + + if (newWorkflows.length !== workflows.length) { + setFilteredWorkflows(newWorkflows) + } + } + + const addFilter = (data) => { + if (data === null || data === undefined) { + return + } + + if (data.includes("<") && data.includes(">")) { + return + } + + if (filters.includes(data)) { + return + } + + filters.push(data.toLowerCase()) + setFilters(filters) + + findWorkflow(filters) + } + + const removeFilter = (index) => { + var newfilters = filters + + if (index < 0) { + console.log("Can't handle index: ", index) + return + } + + + //console.log("Removing filter index", index) + newfilters.splice(index, 1) + //console.log("FILTER LENGTH: ", filters.length) + + if (newfilters.length === 0) { + newfilters = [] + setFilters(newfilters) + } else { + setFilters(newfilters) + } + //console.log("FILTERS: ", newfilters) + + findWorkflow(newfilters) + } + const deleteModal = deleteModalOpen ? { if (responseJson !== undefined) { setWorkflows(responseJson) + setFilteredWorkflows(responseJson) setWorkflowDone(true) } else { if (isLoggedIn) { @@ -635,6 +710,11 @@ const Workflows = (props) => { }); } + + const handleChipClick = (e) => { + addFilter(e.target.innerHTML) + } + const WorkflowPaper = (props) => { const { data } = props; const [open, setOpen] = React.useState(false); @@ -674,8 +754,10 @@ const Workflows = (props) => {
- - {parsedName} + + + {parsedName} + @@ -732,6 +814,7 @@ const Workflows = (props) => { key={index} style={{backgroundColor: "#3d3f43", height: 30, marginRight: 5, paddingLeft: 5, paddingRight: 5, height: 28, cursor: "pointer", borderColor: "#3d3f43", color: "white",}} label={tag} + onClick={handleChipClick} variant="outlined" color="primary" /> @@ -809,7 +892,7 @@ const Workflows = (props) => { - + @@ -1180,7 +1263,7 @@ const Workflows = (props) => { borderRadius: "4px", color: "black", height: "20px", "width": "20px", fontSize: "small"}} /> - + executeWorkflow(data.id)} /> @@ -1296,10 +1379,11 @@ const Workflows = (props) => { fullWidth /> {
-
+

Workflows

+
+ { + addFilter(chip) + }} + onDelete={(chip, index) => { + removeFilter(index) + //setUpdate("delete "+chip) + }} + /> +
{/* @@ -1475,7 +1580,7 @@ const Workflows = (props) => {
- This is your workflow view. Learn more about Workflows + Your workflow view. Learn more about Workflows
{workflowButtons} @@ -1484,7 +1589,7 @@ const Workflows = (props) => {
{view === "grid" && ( - {workflows.map((data, index) => { + {filteredWorkflows.map((data, index) => { return ( ) @@ -1600,7 +1705,7 @@ const Workflows = (props) => { onChange={e => setDownloadUrl(e.target.value)} placeholder="https://github.com/frikky/shuffle-apps" fullWidth - /> + /> Branch (default value is "master"):
From ab5c0d82b984a90ce17cc467f9d6cacecfffc86a Mon Sep 17 00:00:00 2001 From: frikky Date: Wed, 31 Mar 2021 20:21:45 +0200 Subject: [PATCH 57/57] Added new colors and workflow view --- backend/app_sdk/app_base.py | 6 +- backend/go-app/go.sum | 2 + docker-compose.yml | 8 +- frontend/src/App.jsx | 27 +--- frontend/src/defaultCytoscapeStyle.js | 3 +- frontend/src/views/AngularWorkflow.jsx | 201 ++++++++++++++++--------- frontend/src/views/Apps.jsx | 90 ++++++----- frontend/src/views/Workflows.jsx | 131 ++++++++++------ functions/onprem/orborus/build.sh | 2 +- functions/onprem/orborus/go.mod | 2 +- functions/onprem/orborus/orborus.go | 4 +- functions/onprem/worker/build.sh | 2 +- functions/onprem/worker/go.mod | 18 +++ 13 files changed, 301 insertions(+), 195 deletions(-) create mode 100644 functions/onprem/worker/go.mod diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 5e2cdd3e..c3bc15c4 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -1543,7 +1543,7 @@ class AppBase: sourcevalue = condition["source"]["value"] check, sourcevalue, is_loop = parse_params(action, fullexecution, condition["source"]) if check: - return False, "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check) + return False, {"success": False, "reason": "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check)} #sourcevalue = sourcevalue.encode("utf-8") @@ -1552,7 +1552,7 @@ class AppBase: check, destinationvalue, is_loop = parse_params(action, fullexecution, condition["destination"]) if check: - return False, "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check) + return False, {"success": False, "reason": "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check)} #destinationvalue = destinationvalue.encode("utf-8") destinationvalue = parse_wrapper_start(destinationvalue) @@ -1593,7 +1593,7 @@ class AppBase: if not validation: self.logger.info("Failed condition check for %s %s %s." % (sourcevalue, condition["condition"]["value"], destinationvalue)) - return False, "Failed condition: %s %s %s" % (sourcevalue, condition["condition"]["value"], destinationvalue) + return False, {"success": False, "reason": "Failed condition: %s %s %s" % (sourcevalue, condition["condition"]["value"], destinationvalue)} # Make a general parser here, at least to get param["name"] = param["value"] in maparameter[string]string diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index cc0824a0..21fd33ff 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -96,6 +96,8 @@ github.com/frikky/shuffle-shared v0.0.21 h1:xj/XPsXTa2rx41mm4nUc7+2K9RGkq2/mpjSP github.com/frikky/shuffle-shared v0.0.21/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= github.com/frikky/shuffle-shared v0.0.22 h1:TFMcJCNmOOSneMMWbg5dNzp2z6m0aZLROuL+bzVToRE= github.com/frikky/shuffle-shared v0.0.22/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= +github.com/frikky/shuffle-shared v0.0.23 h1:Pnlc2M6fHnFRLFd5K1iLTVv4/t4P04Ri1GJ5CMxzq0U= +github.com/frikky/shuffle-shared v0.0.23/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= github.com/getkin/kin-openapi v0.8.0 h1:a6TQjTqwkyscC4/hShJX7WhCVE+4bi9lzw61XHQW5hE= github.com/getkin/kin-openapi v0.8.0/go.mod h1:zZQMFkVgRHCdhgb6ihCTIo9dyDZFvX0k/xAKqw1FhPw= github.com/getkin/kin-openapi v0.52.0 h1:6WqsF5d6PfJ8AscdD+9Rtb2RP2iBWyC7V6GcjssWg7M= diff --git a/docker-compose.yml b/docker-compose.yml index 1e208eb2..56fd9a3c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: frontend: #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.64 + image: ghcr.io/frikky/shuffle-frontend:0.8.71 container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -17,7 +17,7 @@ services: - backend backend: #build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.64 + image: ghcr.io/frikky/shuffle-backend:0.8.71 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: @@ -47,7 +47,7 @@ services: - database orborus: #build: ./functions/onprem/orborus - image: ghcr.io/frikky/shuffle-orborus:0.8.64 + image: ghcr.io/frikky/shuffle-orborus:0.8.71 container_name: shuffle-orborus hostname: shuffle-orborus networks: @@ -56,7 +56,7 @@ services: - /var/run/docker.sock:/var/run/docker.sock environment: - SHUFFLE_APP_SDK_VERSION=0.8.60 - - SHUFFLE_WORKER_VERSION=0.8.70 + - SHUFFLE_WORKER_VERSION=0.8.71 - ORG_ID=${ORG_ID} - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT} diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 011a4c6e..99465936 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -13,6 +13,7 @@ import EditWebhook from "./views/EditWebhook"; import AngularWorkflow from "./views/AngularWorkflow"; import Header from './components/Header'; +import theme from './theme' import Apps from './views/Apps'; import AppCreator from './views/AppCreator'; @@ -44,34 +45,8 @@ if (window.location.protocol == "http:" && window.location.port === "3000") { //globalUrl = "http://localhost:5002" } -const theme = createMuiTheme({ - palette: { - primary: { - main: "#f85a3e" - }, - secondary: { - main: '#e8eaf6', - }, - surfaceColor: "#27292d", - inputColor: "#383B40" - }, - typography: { - useNextVariants: true - }, - overrides: { - MuiMenu: { - list: { - backgroundColor: "#383B40", - }, - }, - }, -}); - - -// FIXME - set client side cookies const App = (message, props) => { const [userdata, setUserData] = useState({}); - //const [homePage, ] = useState(true); const [cookies, setCookie, removeCookie] = useCookies([]); const [isLoggedIn, setIsLoggedIn] = useState(false); const [dataset, setDataset] = useState(false); diff --git a/frontend/src/defaultCytoscapeStyle.js b/frontend/src/defaultCytoscapeStyle.js index 2732e381..8fd5d326 100644 --- a/frontend/src/defaultCytoscapeStyle.js +++ b/frontend/src/defaultCytoscapeStyle.js @@ -47,7 +47,6 @@ const data = [{ css: { 'width': '30px', 'height': '30px', - 'font-size': '0px', }, }, { @@ -118,6 +117,8 @@ const data = [{ css: { 'shape': 'ellipse', 'border-color': '#80deea', + 'width': '80px', + 'height': '80px', }, }, { diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index dcc792fd..bb6e1d40 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -2777,11 +2777,12 @@ const AngularWorkflow = (props) => { const [hover, setHover] = React.useState(false) const maxlen = 24 - var newAppname = app.name.replace("_", " ", -1) + var newAppname = app.name newAppname = newAppname.charAt(0).toUpperCase()+newAppname.substring(1) if (newAppname.length > maxlen) { newAppname = newAppname.slice(0, maxlen)+".." } + app.name.replaceAll("_", " ", -1) //const image = "url("+app.large_image+")" const image = app.large_image @@ -2807,13 +2808,17 @@ const AngularWorkflow = (props) => { -

{newAppname}

+ {newAppname} - Version: {app.app_version} + + Version: {app.app_version} + - {app.description} + + {app.description} + @@ -2833,43 +2838,52 @@ const AngularWorkflow = (props) => { return(
- - - - - - ) - }} - fullWidth - color="primary" - placeholder={"Search Apps"} - id="appsearch" - onBlur={(event) => { - runSearch(event) - }} - /> -
- {visibleApps.map((app, index) => { - if (app.invalid) { - return null - } - - return( - + + + + + ) - })} -
+ }} + fullWidth + color="primary" + placeholder={"Search Active Apps"} + id="appsearch" + onBlur={(event) => { + runSearch(event) + }} + /> + {visibleApps.length > 0 ? +
+ {visibleApps.map((app, index) => { + if (app.invalid) { + return null + } + + return( + + ) + })} +
+ : +
+ + + Loading apps + +
+ }
) @@ -2925,13 +2939,13 @@ const AngularWorkflow = (props) => { // const selectedNameChange = (event) => { //console.log("OLDNAME: ", selectedAction.name) - event.target.value = event.target.value.replace("(", "") - event.target.value = event.target.value.replace(")", "") - event.target.value = event.target.value.replace("$", "") - event.target.value = event.target.value.replace("#", "") - event.target.value = event.target.value.replace(".", "") - event.target.value = event.target.value.replace(",", "") - event.target.value = event.target.value.replace(" ", "_") + event.target.value = event.target.value.replaceAll("(", "") + event.target.value = event.target.value.replaceAll(")", "") + event.target.value = event.target.value.replaceAll("$", "") + event.target.value = event.target.value.replaceAll("#", "") + event.target.value = event.target.value.replaceAll(".", "") + event.target.value = event.target.value.replaceAll(",", "") + event.target.value = event.target.value.replaceAll(" ", "_") selectedAction.label = event.target.value setSelectedAction(selectedAction) @@ -4384,10 +4398,10 @@ const AngularWorkflow = (props) => { newActionname = data.label } // ROFL FIXME - loop - newActionname = newActionname.replace("_", " ") - newActionname = newActionname.replace("_", " ") - newActionname = newActionname.replace("_", " ") - newActionname = newActionname.replace("_", " ") + newActionname = newActionname.replaceAll("_", " ") + newActionname = newActionname.replaceAll("_", " ") + newActionname = newActionname.replaceAll("_", " ") + newActionname = newActionname.replaceAll("_", " ") newActionname = newActionname.charAt(0).toUpperCase()+newActionname.substring(1) return ( @@ -6877,7 +6891,7 @@ const AngularWorkflow = (props) => { // console.log("HANDLE INPUT FIELD FOR COPY!") //} - to_be_copied.replace(" ", "_") + to_be_copied.replaceAll(" ", "_") const elementName = "copy_element_shuffle" var copyText = document.getElementById(elementName); if (copyText !== null && copyText !== undefined) { @@ -7023,37 +7037,59 @@ const AngularWorkflow = (props) => {
{executionData.status !== undefined && executionData.status.length > 0 ? -
- Status:   {executionData.status} +
+ + Status   + + + {executionData.status} +
: null } {executionData.execution_source !== undefined && executionData.execution_source !== null && executionData.execution_source.length > 0 && executionData.execution_source !== "default" ? -
- Source:   {executionData.execution_parent !== null && executionData.execution_parent !== undefined && executionData.execution_parent.length > 0 ? - executionData.execution_source === props.match.params.key ? - { - getWorkflowExecution(props.match.params.key, executionData.execution_parent) - }}> - Parent Execution - +
+ + Source    + + + {executionData.execution_parent !== null && executionData.execution_parent !== undefined && executionData.execution_parent.length > 0 ? + executionData.execution_source === props.match.params.key ? + + { + getWorkflowExecution(props.match.params.key, executionData.execution_parent) + }}> + Parent Execution + + + : + Parent Workflow : - Parent Workflow - : - executionData.execution_source - } + executionData.execution_source + } +
: null } {executionData.started_at !== undefined ? -
- Started:  {new Date(executionData.started_at*1000).toISOString()} +
+ + Started    + + + {new Date(executionData.started_at*1000).toISOString()} +
: null } {executionData.completed_at !== undefined && executionData.completed_at !== null && executionData.completed_at > 0 ? -
- Finished: {new Date(executionData.completed_at*1000).toISOString()} +
+ + Finished   + + + {new Date(executionData.completed_at*1000).toISOString()} +
: null } @@ -7153,10 +7189,21 @@ const AngularWorkflow = (props) => { {actionimg}
{data.action.label}
-
{data.action.name}
+
+ + {data.action.name} + +
-
Status {data.status}
+
+ + Status   + + + {data.status} + +
{validate.valid ? { } : -
- Result  - {data.result} +
+ + Result  + + + {data.result} +
}
diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 642989d5..40d2cb4e 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -16,6 +16,10 @@ import Dropzone from '../components/Dropzone'; const surfaceColor = "#27292D" const inputColor = "#383B40" +const chipStyle = { + backgroundColor: "#3d3f43", height: 30, marginRight: 5, paddingLeft: 5, paddingRight: 5, height: 28, cursor: "pointer", borderColor: "#3d3f43", color: "white", +} + // Parses JSON data into keys that can be used everywhere :) export const GetParsedPaths = (inputdata, basekey) => { const splitkey = " > " @@ -173,7 +177,7 @@ const Apps = (props) => { minHeight: 130, maxHeight: 130, minWidth: "100%", - maxWidth: "100%", + maxWidth: 612.5, marginBottom: 5, borderRadius: 5, color: "white", @@ -341,11 +345,9 @@ const Apps = (props) => { //console.log("IMG LOADED!: ", event.target) }} /> - // FIXME - add label to apps, as this might be slow with A LOT of apps var newAppname = data.name - newAppname = newAppname.replace("_", " ") newAppname = newAppname.charAt(0).toUpperCase()+newAppname.substring(1) - + newAppname = newAppname.replaceAll("_", " ") var sharing = "public" if (!data.sharing) { sharing = "private" @@ -361,7 +363,7 @@ const Apps = (props) => { } var description = data.description - const maxDescLen = 51 + const maxDescLen = 60 if (description.length > maxDescLen) { description = data.description.slice(0, maxDescLen)+"..." } @@ -370,6 +372,7 @@ const Apps = (props) => { return ( { if (selectedApp.id !== data.id) { + data.name = newAppname setSelectedApp(data) console.log(data) @@ -389,17 +392,21 @@ const Apps = (props) => { {imageline}
- - + + -

{newAppname}

+ + {newAppname} + -
- - {description} +
+ + + {description} +
- + {data.tags === null || data.tags === undefined ? null : data.tags.map((tag, index) => { if (index >= 3) { return null @@ -408,7 +415,7 @@ const Apps = (props) => { return ( { // FIXME - add label to apps, as this might be slow with A LOT of apps var newAppname = selectedApp.name if (newAppname !== undefined && newAppname.length > 0) { - newAppname = newAppname.replace("_", " ") + newAppname = newAppname.replaceAll("_", " ") newAppname = newAppname.charAt(0).toUpperCase()+newAppname.substring(1) } else { newAppname = "" @@ -613,9 +620,15 @@ const Apps = (props) => { {imageline}
-

{newAppname}

-

Version {selectedApp.app_version}

-

{description}

+ + {newAppname} + + + Version {selectedApp.app_version} + + + {description} +
{isCloud ? @@ -644,7 +657,7 @@ const Apps = (props) => { return ( { {props.userdata !== undefined && props.userdata.id === selectedApp.owner ?
{/*

ID: {selectedApp.id}

*/} - Sharing: + Sharing