From 1c87838d19e2d062c33ccff9786d2069833db9be Mon Sep 17 00:00:00 2001 From: Harduino Date: Sat, 1 Aug 2020 21:15:40 +0300 Subject: [PATCH 01/19] siemonster :: use Alpine image for backend/orborus as Scratch doesn't work correctly on Minikube --- backend/Dockerfile | 2 +- functions/onprem/orborus/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 14d9137c..d203ed78 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -24,7 +24,7 @@ RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o webapp . FROM alpine:latest as certs RUN apk --update add ca-certificates -from scratch +FROM alpine:3.12 COPY --from=builder /app/ /app COPY --from=builder /app_sdk/ /app_sdk diff --git a/functions/onprem/orborus/Dockerfile b/functions/onprem/orborus/Dockerfile index 1a15a2e6..8e54c8a0 100644 --- a/functions/onprem/orborus/Dockerfile +++ b/functions/onprem/orborus/Dockerfile @@ -8,7 +8,7 @@ RUN go get github.com/docker/docker/api/types github.com/docker/docker/api/types RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o orborus . -from scratch +FROM alpine:3.12 COPY --from=builder /app/ / CMD ["./orborus"] From f7dacd57924b9f22089a05214aa2eac6b4013bc4 Mon Sep 17 00:00:00 2001 From: Harduino Date: Sat, 1 Aug 2020 21:18:29 +0300 Subject: [PATCH 02/19] siemonster :: orborus - optimize rebuilding speed as step of installing GO-packages will be cached --- functions/onprem/orborus/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/functions/onprem/orborus/Dockerfile b/functions/onprem/orborus/Dockerfile index 8e54c8a0..30859959 100644 --- a/functions/onprem/orborus/Dockerfile +++ b/functions/onprem/orborus/Dockerfile @@ -2,10 +2,10 @@ from golang as builder RUN mkdir /app WORKDIR /app -COPY orborus.go /app/orborus.go - RUN go get github.com/docker/docker/api/types github.com/docker/docker/api/types/container github.com/docker/docker/client +COPY orborus.go /app/orborus.go + RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o orborus . FROM alpine:3.12 From 2b773f3c24863deeee0e1ccdd461e00c22ca5e07 Mon Sep 17 00:00:00 2001 From: Harduino Date: Sat, 1 Aug 2020 21:31:30 +0300 Subject: [PATCH 03/19] siemonster :: we don't have to comment build every time in compose file as we can declare build/image simultaneously --- docker-compose.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8faba6f3..df00d6d0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: frontend: - #build: ./frontend + build: ./frontend image: frikky/shuffle:frontend container_name: shuffle-frontend hostname: shuffle-frontend @@ -16,19 +16,19 @@ services: depends_on: - backend backend: - #build: ./backend + build: ./backend image: frikky/shuffle:backend container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: - ports: + ports: - "${BACKEND_PORT}:5001" networks: - shuffle - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - ${APP_HOTLOAD_LOCATION}:/shuffle-apps - environment: + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ${APP_HOTLOAD_LOCATION}:/shuffle-apps + environment: - DATASTORE_EMULATOR_HOST=shuffle-database:8000 - APP_HOTLOAD_FOLDER=/shuffle-apps - ORG_ID=${ORG_ID} @@ -42,14 +42,14 @@ services: depends_on: - database orborus: - #build: ./functions/onprem/orborus + build: ./functions/onprem/orborus image: frikky/shuffle:orborus container_name: shuffle-orborus hostname: shuffle-orborus networks: - shuffle - volumes: - - /var/run/docker.sock:/var/run/docker.sock + volumes: + - /var/run/docker.sock:/var/run/docker.sock environment: - ORG_ID=${ORG_ID} - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} @@ -60,7 +60,7 @@ services: - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY} restart: unless-stopped database: - #build: ./backend/database + build: ./backend/database image: frikky/shuffle:database container_name: shuffle-database hostname: shuffle-database From d4a82713dbf0f273d84f423d708bad89000fe5c5 Mon Sep 17 00:00:00 2001 From: Harduino Date: Sat, 1 Aug 2020 21:42:40 +0300 Subject: [PATCH 04/19] siemonster :: implementation of parent's network --- functions/onprem/orborus/Dockerfile | 1 + functions/onprem/orborus/orborus.go | 91 +++++++++++------------------ functions/onprem/worker/Dockerfile | 34 +++++------ functions/onprem/worker/worker.go | 40 ++++++------- 4 files changed, 69 insertions(+), 97 deletions(-) diff --git a/functions/onprem/orborus/Dockerfile b/functions/onprem/orborus/Dockerfile index 30859959..e5bc379e 100644 --- a/functions/onprem/orborus/Dockerfile +++ b/functions/onprem/orborus/Dockerfile @@ -9,6 +9,7 @@ COPY orborus.go /app/orborus.go RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o orborus . FROM alpine:3.12 +RUN apk add --no-cache bash COPY --from=builder /app/ / CMD ["./orborus"] diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index fbee4052..983b45bd 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -14,12 +14,12 @@ import ( "log" "net/http" "os" + "os/exec" "strings" "time" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" - network "github.com/docker/docker/api/types/network" dockerclient "github.com/docker/docker/client" //network "github.com/docker/docker/api/types/network" //natting "github.com/docker/go-connections/nat" @@ -27,11 +27,11 @@ import ( var baseUrl = os.Getenv("BASE_URL") var baseimagename = "frikky/shuffle" -var shuffleNetwork = "" // Filled in init if found var dockerApiVersion = os.Getenv("DOCKER_API_VERSION") var environment = os.Getenv("ENVIRONMENT_NAME") var orgId = os.Getenv("ORG_ID") +var runningMode = os.Getenv("RUNNING_MODE") // Starts jobs in bulk, so this could be increased var sleepTime = 3 @@ -62,56 +62,50 @@ func init() { if err != nil { panic(fmt.Sprintf("Unable to create docker client: %s", err)) } +} - // FIXME: Move this to global variables? - containerIdentifier := "orborus" - networkIdentifier := "shuffle" +// form id of current running container +func getThisContainerId() string { + containerId := "" + fCol := "" - ctx := context.Background() - containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{ - All: true, - }) - if err != nil { - log.Printf("Failed getting containers during init - running without network check: %s", err) + // some adjusting based on current running mode + switch runningMode { + case "Kubernetes": + // cgroup will be like: + // 11:net_cls,net_prio:/kubepods/besteffort/podf132b44d-cfcf-43f7-9906-79f58e268333/851466f8b5ed5aa0f265b1c95c6d2bafbc51a38dd5c5a1621b6e586572150009 + fCol = "5" + + case "Docker": + // cgroup will be like: + // 12:perf_event:/docker/0f06810364f52a2cd6e80bfba27419cb8a29758a204cd676388f4913bb366f2b + fCol = "3" + + default: + fCol = "3" // for backward-compatibility with production + log.Printf("[WARNING] Running not containerized, so I can't figure out current container id!") } - // Skip random containers. Only handle things related to Shuffle. - for _, container := range containers { - found := false - - // Bad states - it might just be created sometimes, leading to now netowkr - //if container.State == "restarting" || container.State == "paused" || container.State == "exited" || container.State == "dead" { - // continue - //} - - for _, name := range container.Names { - if !strings.Contains(strings.ToLower(name), containerIdentifier) { - found = true - continue - } - } - - if found { - for key, _ := range container.NetworkSettings.Networks { - if strings.Contains(strings.ToLower(key), networkIdentifier) { - shuffleNetwork = key - break - } - } + if fCol != "" { + cmd := fmt.Sprintf("head -1 /proc/self/cgroup | cut -d/ -f%s", fCol) + out, err := exec.Command("bash","-c",cmd).Output() + if err == nil { + containerId = strings.TrimSpace(string(out)) } } - if len(shuffleNetwork) > 0 { - log.Printf("Found shuffle network \"%s\" for container %s", shuffleNetwork, containerIdentifier) - } else { - log.Printf("Running Shuffle without a docker network") - } + return containerId } // Deploys the internal worker whenever something happens func deployWorker(image string, identifier string, env []string) { + // figure out current container id + containerId := getThisContainerId() + // Binds is the actual "-v" volume. hostConfig := &container.HostConfig{ + NetworkMode: container.NetworkMode(fmt.Sprintf("container:%s", containerId)), + IpcMode: container.IpcMode(fmt.Sprintf("container:%s", containerId)), LogConfig: container.LogConfig{ Type: "json-file", Config: map[string]string{}, @@ -121,23 +115,6 @@ func deployWorker(image string, identifier string, env []string) { }, } - // Look for Shuffle network and set it - networkConfig := &network.NetworkingConfig{} - if len(shuffleNetwork) > 0 { - log.Printf("Starting worker with network %s", shuffleNetwork) - networkConfig = &network.NetworkingConfig{ - EndpointsConfig: map[string]*network.EndpointSettings{ - shuffleNetwork: { - NetworkID: shuffleNetwork, - }, - }, - } - - env = append(env, fmt.Sprintf("DOCKER_NETWORK=%s", shuffleNetwork)) - } else { - log.Printf("Starting worker WITHOUT any specified network: %s", shuffleNetwork) - } - // ROFL: https://docker-py.readthedocs.io/en/1.4.0/volumes/ config := &container.Config{ Image: image, @@ -154,7 +131,7 @@ func deployWorker(image string, identifier string, env []string) { context.Background(), config, hostConfig, - networkConfig, + nil, nil, identifier, ) diff --git a/functions/onprem/worker/Dockerfile b/functions/onprem/worker/Dockerfile index 2f8971f8..7f253e1b 100644 --- a/functions/onprem/worker/Dockerfile +++ b/functions/onprem/worker/Dockerfile @@ -1,21 +1,17 @@ -#from golang as builder -# -#RUN mkdir /app -#WORKDIR /app -#COPY worker.go /app/worker.go -# -#RUN go get github.com/docker/docker/api/types -#RUN go get github.com/docker/docker/api/types/container -#RUN go get -u github.com/docker/docker/client -# -#RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker . -# +from golang as builder -# THis is a workaround until I get docker/docker to build in a dockerfile -# PS: This is tricky to google. -# Might not work on some machines. -from scratch -#COPY --from=builder /app/ / -COPY worker.bin /worker.bin +WORKDIR /app -CMD ["./worker.bin"] +RUN go get github.com/docker/docker/api/types && \ + go get github.com/docker/docker/api/types/container && \ + go get -u github.com/docker/docker/client + +COPY worker.go /app/worker.go +RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker . + + +FROM alpine:3.12 +RUN apk add --no-cache bash +COPY --from=builder /app/ / + +CMD ["./worker"] diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 48b11ebc..dce95db1 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -11,12 +11,12 @@ import ( "log" "net/http" "os" + "os/exec" "strings" "time" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" - network "github.com/docker/docker/api/types/network" dockerclient "github.com/docker/docker/client" ) @@ -412,9 +412,26 @@ func shutdown(executionId, workflowId string) { os.Exit(3) } +// form container id of current running container +func getThisContainerId() string { + containerId := "" + cmd := fmt.Sprintf("head -1 /proc/self/cgroup | cut -d/ -f3") + out, err := exec.Command("bash","-c",cmd).Output() + if err == nil { + containerId = strings.TrimSpace(string(out)) + } + + return containerId +} + // Deploys the internal worker whenever something happens func deployApp(cli *dockerclient.Client, image string, identifier string, env []string) error { + // figure out current container id + containerId := getThisContainerId() + hostConfig := &container.HostConfig{ + NetworkMode: container.NetworkMode(fmt.Sprintf("container:%s", containerId)), + IpcMode: container.IpcMode(fmt.Sprintf("container:%s", containerId)), LogConfig: container.LogConfig{ Type: "json-file", Config: map[string]string{}, @@ -426,23 +443,11 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] Env: env, } - networkConfig := &network.NetworkingConfig{} - shuffleNetwork := os.Getenv("DOCKER_NETWORK") - if len(shuffleNetwork) > 0 { - networkConfig = &network.NetworkingConfig{ - EndpointsConfig: map[string]*network.EndpointSettings{ - shuffleNetwork: { - NetworkID: shuffleNetwork, - }, - }, - } - } - cont, err := cli.ContainerCreate( context.Background(), config, hostConfig, - networkConfig, + nil, nil, identifier, ) @@ -1107,13 +1112,6 @@ func main() { } } - shuffleNetwork := os.Getenv("DOCKER_NETWORK") - if len(shuffleNetwork) > 0 { - log.Printf("Running with Docker network %s", shuffleNetwork) - } else { - log.Printf("No docker network specified for Worker.") - } - // WORKER_TESTING_WORKFLOW should be a workflow ID authorization := "" executionId := "" From 4135f339820fca4a716399bc0d4ae8eee13fefee Mon Sep 17 00:00:00 2001 From: Harduino Date: Wed, 5 Aug 2020 12:01:37 +0300 Subject: [PATCH 05/19] siemonster :: improve determining self container id --- functions/onprem/orborus/orborus.go | 2 +- functions/onprem/worker/worker.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 983b45bd..daab7a71 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -87,7 +87,7 @@ func getThisContainerId() string { } if fCol != "" { - cmd := fmt.Sprintf("head -1 /proc/self/cgroup | cut -d/ -f%s", fCol) + cmd := fmt.Sprintf("cat /proc/self/cgroup | grep memory | tail -1 | cut -d/ -f%s", fCol) out, err := exec.Command("bash","-c",cmd).Output() if err == nil { containerId = strings.TrimSpace(string(out)) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index dce95db1..6824434a 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -415,7 +415,7 @@ func shutdown(executionId, workflowId string) { // form container id of current running container func getThisContainerId() string { containerId := "" - cmd := fmt.Sprintf("head -1 /proc/self/cgroup | cut -d/ -f3") + cmd := fmt.Sprintf("cat /proc/self/cgroup | grep memory | tail -1 | cut -d/ -f3") out, err := exec.Command("bash","-c",cmd).Output() if err == nil { containerId = strings.TrimSpace(string(out)) From 03ab5ba0e179b85f0cb648c3376723ab573345ef Mon Sep 17 00:00:00 2001 From: Harduino Date: Wed, 5 Aug 2020 12:03:23 +0300 Subject: [PATCH 06/19] siemonster :: handle empty calculated self container id --- functions/onprem/orborus/orborus.go | 14 +++++++++----- functions/onprem/worker/worker.go | 15 ++++++++++----- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index daab7a71..d37544cd 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -99,13 +99,8 @@ func getThisContainerId() string { // Deploys the internal worker whenever something happens func deployWorker(image string, identifier string, env []string) { - // figure out current container id - containerId := getThisContainerId() - // Binds is the actual "-v" volume. hostConfig := &container.HostConfig{ - NetworkMode: container.NetworkMode(fmt.Sprintf("container:%s", containerId)), - IpcMode: container.IpcMode(fmt.Sprintf("container:%s", containerId)), LogConfig: container.LogConfig{ Type: "json-file", Config: map[string]string{}, @@ -115,6 +110,15 @@ func deployWorker(image string, identifier string, env []string) { }, } + // form container id and use it as network source if it's not empty + containerId := getThisContainerId() + if containerId != "" { + hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) + hostConfig.IpcMode = container.IpcMode(fmt.Sprintf("container:%s", containerId)) + } else { + log.Printf("[WARNING] Empty determined container id, continue without NetworkMode/IpcMode") + } + // ROFL: https://docker-py.readthedocs.io/en/1.4.0/volumes/ config := &container.Config{ Image: image, diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 6824434a..96d1afdf 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -426,18 +426,23 @@ func getThisContainerId() string { // Deploys the internal worker whenever something happens func deployApp(cli *dockerclient.Client, image string, identifier string, env []string) error { - // figure out current container id - containerId := getThisContainerId() - + // form basic hostConfig hostConfig := &container.HostConfig{ - NetworkMode: container.NetworkMode(fmt.Sprintf("container:%s", containerId)), - IpcMode: container.IpcMode(fmt.Sprintf("container:%s", containerId)), LogConfig: container.LogConfig{ Type: "json-file", Config: map[string]string{}, }, } + // form container id and use it as network source if it's not empty + containerId := getThisContainerId() + if containerId != "" { + hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) + hostConfig.IpcMode = container.IpcMode(fmt.Sprintf("container:%s", containerId)) + } else { + log.Printf("[WARNING] Empty determined container id, continue without NetworkMode/IpcMode") + } + config := &container.Config{ Image: image, Env: env, From 6d6e7cdc7ed375d2ae03f8776c4851d65cc478a4 Mon Sep 17 00:00:00 2001 From: Harduino Date: Wed, 5 Aug 2020 12:03:53 +0300 Subject: [PATCH 07/19] siemonster :: remove IpcMode --- functions/onprem/orborus/orborus.go | 3 +-- functions/onprem/worker/worker.go | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index d37544cd..e72d7c7b 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -114,9 +114,8 @@ func deployWorker(image string, identifier string, env []string) { containerId := getThisContainerId() if containerId != "" { hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) - hostConfig.IpcMode = container.IpcMode(fmt.Sprintf("container:%s", containerId)) } else { - log.Printf("[WARNING] Empty determined container id, continue without NetworkMode/IpcMode") + log.Printf("[WARNING] Empty determined container id, continue without NetworkMode") } // ROFL: https://docker-py.readthedocs.io/en/1.4.0/volumes/ diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 96d1afdf..61a84f86 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -438,9 +438,8 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] containerId := getThisContainerId() if containerId != "" { hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) - hostConfig.IpcMode = container.IpcMode(fmt.Sprintf("container:%s", containerId)) } else { - log.Printf("[WARNING] Empty determined container id, continue without NetworkMode/IpcMode") + log.Printf("[WARNING] Empty determined container id, continue without NetworkMode") } config := &container.Config{ From b4ed7f87fc4ad2ff89d2e2dbfc20cd4e5866a8b9 Mon Sep 17 00:00:00 2001 From: Harduino Date: Wed, 5 Aug 2020 14:21:39 +0300 Subject: [PATCH 08/19] siemonster :: verbose found container id --- functions/onprem/orborus/orborus.go | 3 ++- functions/onprem/worker/worker.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index e72d7c7b..8a07d8f9 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -112,10 +112,11 @@ func deployWorker(image string, identifier string, env []string) { // form container id and use it as network source if it's not empty containerId := getThisContainerId() + log.Printf("Found self container id: %s", containerId) if containerId != "" { hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) } else { - log.Printf("[WARNING] Empty determined container id, continue without NetworkMode") + log.Printf("[WARNING] Empty self container id, continue without NetworkMode") } // ROFL: https://docker-py.readthedocs.io/en/1.4.0/volumes/ diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 61a84f86..f773271f 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -436,10 +436,11 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] // form container id and use it as network source if it's not empty containerId := getThisContainerId() + log.Printf("Found self container id: %s", containerId) if containerId != "" { hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) } else { - log.Printf("[WARNING] Empty determined container id, continue without NetworkMode") + log.Printf("[WARNING] Empty self container id, continue without NetworkMode") } config := &container.Config{ From f05077274b63e698211bf8582f6272d67ec09e50 Mon Sep 17 00:00:00 2001 From: Harduino Date: Fri, 7 Aug 2020 10:17:23 +0300 Subject: [PATCH 09/19] siemonster :: force to use NetworkMode=host on building Apps --- backend/go-app/docker.go | 2 ++ functions/stitcher.go | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index 865d5867..2d89127b 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -183,6 +183,7 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin Remove: true, Tags: tags, BuildArgs: map[string]*string{}, + NetworkMode: "host", } httpProxy := os.Getenv("HTTP_PROXY") @@ -245,6 +246,7 @@ func buildImage(tags []string, dockerfileFolder string) error { Remove: true, Tags: tags, BuildArgs: map[string]*string{}, + NetworkMode: "host", } httpProxy := os.Getenv("HTTP_PROXY") diff --git a/functions/stitcher.go b/functions/stitcher.go index 1b752f77..aae060f3 100644 --- a/functions/stitcher.go +++ b/functions/stitcher.go @@ -101,15 +101,15 @@ func getRunner(classname string) string { return fmt.Sprintf(` # Run the actual thing after we've checked params def run(request): - action = request.get_json() + action = request.get_json() print(action) print(type(action)) authorization_key = action.get("authorization") current_execution_id = action.get("execution_id") - + if action and "name" in action and "app_name" in action: asyncio.run(%s.run(action), debug=True) - return f'Attempting to execute function {action["name"]} in app {action["app_name"]}' + return f'Attempting to execute function {action["name"]} in app {action["app_name"]}' else: return f'Invalid action' @@ -610,6 +610,7 @@ func buildImage(client *client.Client, tags []string, dockerBuildCtxDir string) PullParent: true, Remove: true, Tags: tags, + NetworkMode: "host", }, ) From e29c084a53b834f31c9cead419b9d64a336a955b Mon Sep 17 00:00:00 2001 From: frikky Date: Thu, 27 Aug 2020 11:19:03 +0200 Subject: [PATCH 10/19] Added workflow icon trackers --- frontend/src/views/Workflows.jsx | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 10ecae86..314de29d 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -18,6 +18,7 @@ import Switch from '@material-ui/core/Switch'; import CircularProgress from '@material-ui/core/CircularProgress'; import CachedIcon from '@material-ui/icons/Cached'; import GetAppIcon from '@material-ui/icons/GetApp'; +import AppsIcon from '@material-ui/icons/Apps'; import EditIcon from '@material-ui/icons/Edit'; import MoreVertIcon from '@material-ui/icons/MoreVert'; import PlayArrowIcon from '@material-ui/icons/PlayArrow'; @@ -412,12 +413,35 @@ const Workflows = (props) => { setAnchorEl(event.currentTarget); } + const actions = data.actions !== null ? data.actions.length : 0 + var schedules = 0 + var webhooks = 0 + var webhookImg = "" + var scheduleImg = "" + if (data.triggers !== undefined && data.triggers !== null && data.triggers.length > 0) { + console.log("Triggers: ", data.triggers) + for (var key in data.triggers) { + if (data.triggers[key].app_name === "Webhook") { + webhooks += 1 + webhookImg = data.triggers[key].large_image + } else if (data.triggers[key].app_name === "Schedule") { + schedules += 1 + scheduleImg = data.triggers[key].large_image + } + } + } + + const imgSize = 25 return ( { }}> -
-
- + + + {webhooks > 0 ? {data.title} : null} + {schedules > 0 ? {data.title} : null} + +
+
{ From ab7436b5279d4e6bc9871219eaafef6a52d5fe49 Mon Sep 17 00:00:00 2001 From: frikky Date: Thu, 27 Aug 2020 20:23:51 +0200 Subject: [PATCH 11/19] Added sharing and fixed OpenAPI issue --- backend/go-app/codegen.go | 2 +- backend/go-app/main.go | 32 +++--- backend/go-app/walkoff.go | 177 ++++++++++++++++++++++++------- frontend/src/App.jsx | 3 +- frontend/src/views/Admin.jsx | 147 +++++++++++++++++++++++++ frontend/src/views/Apps.jsx | 148 +++++++++++++++++++++++--- frontend/src/views/Workflows.jsx | 53 ++++++--- 7 files changed, 474 insertions(+), 88 deletions(-) diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index 88644465..2b3be803 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -459,7 +459,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, // Jesus what a clusterfuck. // Handles parsing of categories from OpenApi3 custom field if val, ok := swagger.Info.ExtensionProps.Extensions["x-categories"]; ok { - log.Printf("Categories: %#v", val) + //log.Printf("Categories: %#v", val) j, err := json.Marshal(&val) if err == nil { if j[0] == 0x22 && j[len(j)-1] == 0x22 { diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 7e96623f..75fba54f 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -141,7 +141,7 @@ type UserLimits struct { // Saves some data, not sure what to have here lol type UserAuth struct { - Description string `json:"description" datastore:"description" yaml:"description"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` Name string `json:"name" datastore:"name" yaml:"name"` Workflows []string `json:"workflows" datastore:"workflows"` Username string `json:"username" datastore:"username"` @@ -210,7 +210,7 @@ type Translator struct { Src struct { Name string `json:"name" datastore:"name"` Value string `json:"value" datastore:"value"` - Description string `json:"description" datastore:"description"` + Description string `json:"description" datastore:"description,noindex"` Required string `json:"required" datastore:"required"` Type string `json:"type" datastore:"type"` Schema struct { @@ -221,7 +221,7 @@ type Translator struct { Name string `json:"name" datastore:"name"` Value string `json:"value" datastore:"value"` Type string `json:"type" datastore:"type"` - Description string `json:"description" datastore:"description"` + Description string `json:"description" datastore:"description,noindex"` Required string `json:"required" datastore:"required"` Schema struct { Type string `json:"type" datastore:"type"` @@ -284,7 +284,7 @@ type ApiYaml struct { Name string `json:"name" yaml:"name" required:"true datastore:"name"` Foldername string `json:"foldername" yaml:"foldername" required:"true datastore:"foldername"` Id string `json:"id" yaml:"id",required:"true, datastore:"id"` - Description string `json:"description" datastore:"description" yaml:"description"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` AppVersion string `json:"app_version" yaml:"app_version",datastore:"app_version"` ContactInfo struct { Name string `json:"name" datastore:"name" yaml:"name"` @@ -293,10 +293,10 @@ type ApiYaml struct { Types []string `json:"types" datastore:"types" yaml:"types"` Input []struct { Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description" yaml:"description"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` InputParameters []struct { Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description" yaml:"description"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` Required string `json:"required" datastore:"required" yaml:"required"` Schema struct { Type string `json:"type" datastore:"type" yaml:"type"` @@ -304,7 +304,7 @@ type ApiYaml struct { } `json:"inputparameters" datastore:"inputparameters" yaml:"inputparameters"` OutputParameters []struct { Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description" yaml:"description"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` Required string `json:"required" datastore:"required" yaml:"required"` Schema struct { Type string `json:"type" datastore:"type" yaml:"type"` @@ -312,7 +312,7 @@ type ApiYaml struct { } `json:"outputparameters" datastore:"outputparameters" yaml:"outputparameters"` Config []struct { Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description" yaml:"description"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` Required string `json:"required" datastore:"required" yaml:"required"` Schema struct { Type string `json:"type" datastore:"type" yaml:"type"` @@ -321,10 +321,10 @@ type ApiYaml struct { } `json:"input" datastore:"input" yaml:"input"` Output []struct { Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description" yaml:"description"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` Config []struct { Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description" yaml:"description"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` Required string `json:"required" datastore:"required" yaml:"required"` Schema struct { Type string `json:"type" datastore:"type" yaml:"type"` @@ -332,7 +332,7 @@ type ApiYaml struct { } `json:"config" datastore:"config" yaml:"config"` InputParameters []struct { Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description" yaml:"description"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` Required string `json:"required" datastore:"required" yaml:"required"` Schema struct { Type string `json:"type" datastore:"type" yaml:"type"` @@ -340,7 +340,7 @@ type ApiYaml struct { } `json:"inputparameters" datastore:"inputparameters" yaml:"inputparameters"` OutputParameters []struct { Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description" yaml:"description"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` Required string `json:"required" datastore:"required" yaml:"required"` Schema struct { Type string `json:"type" datastore:"type" yaml:"type"` @@ -357,7 +357,7 @@ type Hooks struct { type Info struct { Url string `json:"url" datastore:"url"` Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description"` + Description string `json:"description" datastore:"description,noindex"` } // Actions to be done by webhooks etc @@ -1697,10 +1697,11 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { "success": true, "admin": %s, "tutorials": [], + "id": "%s", "orgs": [{"name": "Shuffle", "id": "123", "role": "admin"}], "selected_org": {"name": "Shuffle", "id": "123", "role": "admin"}, "cookies": [{"key": "session_token", "value": "%s", "expiration": %d}] - }`, parsedAdmin, userInfo.Session, expiration.Unix()) + }`, parsedAdmin, userInfo.Id, userInfo.Session, expiration.Unix()) resp.WriteHeader(200) resp.Write([]byte(returnData)) @@ -2505,7 +2506,7 @@ func handleCors(resp http.ResponseWriter, request *http.Request) bool { resp.Header().Set("Vary", "Origin") resp.Header().Set("Access-Control-Allow-Headers", "Content-Type, Accept, X-Requested-With, remember-me") - resp.Header().Set("Access-Control-Allow-Methods", "POST, GET, PUT, DELETE") + resp.Header().Set("Access-Control-Allow-Methods", "POST, GET, PUT, DELETE, PATCH") resp.Header().Set("Access-Control-Allow-Credentials", "true") resp.Header().Set("Access-Control-Allow-Origin", allowedOrigins) @@ -6497,6 +6498,7 @@ func init() { r.HandleFunc("/api/v1/apps/run_hotload", handleAppHotloadRequest).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/apps/get_existing", loadSpecificApps).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/download_remote", loadSpecificApps).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/apps/{appId}", updateWorkflowAppConfig).Methods("PATCH", "OPTIONS") r.HandleFunc("/api/v1/apps/validate", validateAppInput).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/{appId}", deleteWorkflowApp).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/apps/{appId}/config", getWorkflowAppConfig).Methods("GET", "OPTIONS") diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index a72d6a32..d9fae1ba 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -96,25 +96,26 @@ type AuthenticationUsage struct { // An app inside Shuffle // Source string `json:"source" datastore:"soure" yaml:"source"` - downloadlocation type WorkflowApp struct { - Name string `json:"name" yaml:"name" required:true datastore:"name"` - IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"` - ID string `json:"id" yaml:"id,omitempty" required:false datastore:"id"` - Link string `json:"link" yaml:"link" required:false datastore:"link,noindex"` - AppVersion string `json:"app_version" yaml:"app_version" required:true datastore:"app_version"` - Generated bool `json:"generated" yaml:"generated" required:false datastore:"generated"` - Downloaded bool `json:"downloaded" yaml:"downloaded" required:false datastore:"downloaded"` - Sharing bool `json:"sharing" yaml:"sharing" required:false datastore:"sharing"` - Verified bool `json:"verified" yaml:"verified" required:false datastore:"verified"` - Activated bool `json:"activated" yaml:"activated" required:false datastore:"activated"` - Tested bool `json:"tested" yaml:"tested" required:false datastore:"tested"` - Owner string `json:"owner" datastore:"owner" yaml:"owner"` - Hash string `json:"hash" datastore:"hash" yaml:"hash"` // api.yaml+dockerfile+src/app.py for apps - PrivateID string `json:"private_id" yaml:"private_id" required:false datastore:"private_id"` - Description string `json:"description" datastore:"description,noindex" required:false yaml:"description"` - Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"` - SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"` - LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false` - ContactInfo struct { + Name string `json:"name" yaml:"name" required:true datastore:"name"` + IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"` + ID string `json:"id" yaml:"id,omitempty" required:false datastore:"id"` + Link string `json:"link" yaml:"link" required:false datastore:"link,noindex"` + AppVersion string `json:"app_version" yaml:"app_version" required:true datastore:"app_version"` + SharingConfig string `json:"sharing_config" yaml:"sharing_config" datastore:"sharing_config"` + Generated bool `json:"generated" yaml:"generated" required:false datastore:"generated"` + Downloaded bool `json:"downloaded" yaml:"downloaded" required:false datastore:"downloaded"` + Sharing bool `json:"sharing" yaml:"sharing" required:false datastore:"sharing"` + Verified bool `json:"verified" yaml:"verified" required:false datastore:"verified"` + Activated bool `json:"activated" yaml:"activated" required:false datastore:"activated"` + Tested bool `json:"tested" yaml:"tested" required:false datastore:"tested"` + Owner string `json:"owner" datastore:"owner" yaml:"owner"` + Hash string `json:"hash" datastore:"hash" yaml:"hash"` // api.yaml+dockerfile+src/app.py for apps + PrivateID string `json:"private_id" yaml:"private_id" required:false datastore:"private_id"` + Description string `json:"description" datastore:"description,noindex" required:false yaml:"description"` + Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"` + SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"` + LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false` + ContactInfo struct { Name string `json:"name" datastore:"name" yaml:"name"` Url string `json:"url" datastore:"url" yaml:"url"` } `json:"contact_info" datastore:"contact_info" yaml:"contact_info" required:false` @@ -144,7 +145,7 @@ type SchemaDefinition struct { } type WorkflowAppAction struct { - Description string `json:"description" datastore:"description"` + Description string `json:"description" datastore:"description,noindex"` ID string `json:"id" datastore:"id" yaml:"id,omitempty"` Name string `json:"name" datastore:"name"` Label string `json:"label" datastore:"label"` @@ -157,7 +158,7 @@ type WorkflowAppAction struct { Tested bool `json:"tested" datastore:"tested" yaml:"tested"` Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters"` ExecutionVariable struct { - Description string `json:"description" datastore:"description"` + Description string `json:"description" datastore:"description,noindex"` ID string `json:"id" datastore:"id"` Name string `json:"name" datastore:"name"` Value string `json:"value" datastore:"value"` @@ -192,7 +193,7 @@ type WorkflowExecution struct { Workflow Workflow `json:"workflow" datastore:"workflow,noindex"` Results []ActionResult `json:"results" datastore:"results,noindex"` ExecutionVariables []struct { - Description string `json:"description" datastore:"description"` + Description string `json:"description" datastore:"description,noindex"` ID string `json:"id" datastore:"id"` Name string `json:"name" datastore:"name"` Value string `json:"value" datastore:"value,noindex"` @@ -217,7 +218,7 @@ type Action struct { Name string `json:"name" datastore:"name"` Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"` ExecutionVariable struct { - Description string `json:"description" datastore:"description"` + Description string `json:"description" datastore:"description,noindex"` ID string `json:"id" datastore:"id"` Name string `json:"name" datastore:"name"` Value string `json:"value" datastore:"value,noindex"` @@ -235,7 +236,7 @@ type Action struct { // Added environment for location to execute type Trigger struct { AppName string `json:"app_name" datastore:"app_name"` - Description string `json:"description" datastore:"description"` + Description string `json:"description" datastore:"description,noindex"` LongDescription string `json:"long_description" datastore:"long_description"` Status string `json:"status" datastore:"status"` AppVersion string `json:"app_version" datastore:"app_version"` @@ -294,20 +295,20 @@ type Workflow struct { ID string `json:"id" datastore:"id"` IsValid bool `json:"is_valid" datastore:"is_valid"` Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description"` + Description string `json:"description" datastore:"description,noindex"` Start string `json:"start" datastore:"start"` Owner string `json:"owner" datastore:"owner"` Sharing string `json:"sharing" datastore:"sharing"` Org []Org `json:"org,omitempty" datastore:"org"` ExecutingOrg Org `json:"execution_org,omitempty" datastore:"execution_org"` WorkflowVariables []struct { - Description string `json:"description" datastore:"description"` + Description string `json:"description" datastore:"description,noindex"` ID string `json:"id" datastore:"id"` Name string `json:"name" datastore:"name"` Value string `json:"value" datastore:"value"` } `json:"workflow_variables" datastore:"workflow_variables"` ExecutionVariables []struct { - Description string `json:"description" datastore:"description"` + Description string `json:"description" datastore:"description,noindex"` ID string `json:"id" datastore:"id"` Name string `json:"name" datastore:"name"` Value string `json:"value" datastore:"value,noindex"` @@ -330,7 +331,7 @@ type Authentication struct { } type AuthenticationParams struct { - Description string `json:"description" datastore:"description" yaml:"description"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` ID string `json:"id" datastore:"id" yaml:"id"` Name string `json:"name" datastore:"name" yaml:"name"` Example string `json:"example" datastore:"example" yaml:"example"` @@ -3408,9 +3409,6 @@ func getWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { return } - // log.Printf("%#v", parsedApi) - // log.Printf("API LEN: %d, ID: %s", len(parsedApi.Body), fileId) - //log.Printf("Parsed API: %#v", parsedApi) if len(parsedApi.ID) > 0 { parsedApi.Success = true @@ -3629,6 +3627,90 @@ func getAppAuthentication(resp http.ResponseWriter, request *http.Request) { }` */ } +func updateWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + user, userErr := handleApiAuthentication(resp, request) + if userErr != nil { + log.Printf("Api authentication failed in get all apps: %s", userErr) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + location := strings.Split(request.URL.String(), "/") + var fileId string + if location[1] == "api" { + if len(location) <= 4 { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + fileId = location[4] + } + + ctx := context.Background() + app, err := getApp(ctx, fileId) + if err != nil { + log.Printf("Error getting app: %s (update app)", app.Name) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if user.Id != app.Owner && user.Role != "admin" { + log.Printf("Wrong user (%s) for app %s in update app", user.Username, app.Name) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + body, err := ioutil.ReadAll(request.Body) + if err != nil { + log.Printf("Error with body read in update app: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + type updatefields struct { + Sharing bool `json:"sharing"` + SharingConfig string `json:"sharing_config"` + } + + var tmpfields updatefields + err = json.Unmarshal(body, &tmpfields) + if err != nil { + log.Printf("Error with unmarshal body in update app: %s\n%s", err, string(body)) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if tmpfields.Sharing != app.Sharing { + app.Sharing = tmpfields.Sharing + } + + if tmpfields.SharingConfig != app.SharingConfig { + app.SharingConfig = tmpfields.SharingConfig + } + + err = setWorkflowAppDatastore(ctx, *app, app.ID) + if err != nil { + log.Printf("Failed patching workflowapp: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + log.Printf("Changed workflow app %s", app.ID) + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) +} func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) @@ -3691,12 +3773,12 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { newapps := []WorkflowApp{} baseApps := []WorkflowApp{} - if len(user.PrivateApps) > 0 { - newapps = append(newapps, user.PrivateApps...) - } - for _, workflowapp := range workflowapps { - if !workflowapp.Sharing { + if !workflowapp.Activated && workflowapp.Generated { + continue + } + + if workflowapp.Owner != user.Id && user.Role != "admin" && !workflowapp.Sharing { continue } @@ -3716,6 +3798,25 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { baseApps = append(baseApps, workflowapp) } + if len(user.PrivateApps) > 0 { + found := false + for _, item := range user.PrivateApps { + for _, app := range newapps { + if item.ID == app.ID { + found = true + break + } + } + + if !found { + newapps = append(newapps, item) + } + } + } + + log.Printf("User: %d", len(user.PrivateApps)) + log.Printf("All: %d", len(workflowapps)) + // Double unmarshal because of user apps newbody, err := json.Marshal(newapps) //newbody, err := json.Marshal(workflowapps) @@ -4381,14 +4482,14 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, readFile, err := ioutil.ReadAll(fileReader) if err != nil { - log.Printf("Filereader error yaml: %s", err) + log.Printf("Filereader error yaml for %s: %s", filename, err) continue } // 1. This parses OpenAPI v2 to v3 etc, for use. parsedOpenApi, err := handleSwaggerValidation(readFile) if err != nil { - log.Printf("Validation error: %s", err) + log.Printf("Validation error for %s: %s", filename, err) continue } diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index f7a408d1..ea4224f0 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -129,13 +129,14 @@ const App = (message, props) => { } /> } /> } /> + } /> } /> } /> } /> } /> } /> } /> - } /> + } /> } /> } /> } /> diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index ae1756c3..640c46e8 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -49,6 +49,71 @@ const Admin = (props) => { const [selectedAuthenticationModalOpen, setSelectedAuthenticationModalOpen] = React.useState(false) const [showArchived, setShowArchived] = React.useState(false) + const getApps = () => { + fetch(globalUrl+"/api/v1/workflows/apps", { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for apps :O!") + } + + return response.json() + }) + .then((responseJson) => { + console.log("apps: ", responseJson) + //setApps(responseJson) + //setFilteredApps(responseJson) + //if (responseJson.length > 0) { + // setSelectedApp(responseJson[0]) + // if (responseJson[0].actions !== null && responseJson[0].actions.length > 0) { + // setSelectedAction(responseJson[0].actions[0]) + // } else { + // setSelectedAction({}) + // } + //} + }) + .catch(error => { + alert.error(error.toString()) + }); + } + + const categories = [ + { + "name": "Ticketing", + "apps": [ + "TheHive", + "Service-Now", + "SecureWorks", + ], + "categories": ["tickets", "ticket", "ticketing"] + }, + ] + /* + "SIEM", + "Active Directory", + "Firewalls", + "Proxies web", + "SIEM", + "SOAR", + "Mail", + "EDR", + "AV", + "MDM/MAM", + "DNS", + "Ticketing platform", + "TIP", + "Communication", + "DDOS protection", + "VMS", + ] + */ + const alert = useAlert() const deleteAuthentication = (data) => { @@ -905,6 +970,82 @@ const Admin = (props) => {
: null + const appCategoryView = curTab === 6 ? +
+
+

Categories

+ + Categories are the categories supported by Shuffle, which are mapped to apps and workflows + +
+ + + + + + + + + + {categories.map(data => { + if (data.apps.length === 0) { + return null + } + + return ( + + + + + + + + + + ) + })} + +
+ : null + const authenticationView = curTab === 1 ?
@@ -1182,6 +1323,10 @@ const Admin = (props) => { getSchedules() } + if (newValue === 6) { + console.log("Should get apps for categories.") + } + setModalUser({}) setCurTab(newValue) } @@ -1202,10 +1347,12 @@ const Admin = (props) => { Schedules /> {window.location.protocol == "http:" && window.location.port === "3000" ? Hybrid/> : null} {window.location.protocol == "http:" && window.location.port === "3000" ? Organizations/> : null} + {window.location.protocol === "http:" && window.location.port === "3000" ? Categories/> : null}
{authenticationView} + {appCategoryView} {usersView} {environmentView} {schedulesView} diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 3ae8b7db..1bdbb68c 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -112,6 +112,7 @@ const Apps = (props) => { const [openApiError, setOpenApiError] = React.useState("") const [field1, setField1] = React.useState("") const [field2, setField2] = React.useState("") + const [sharingConfiguration, setSharingConfiguration] = React.useState("you") const { start, stop } = useInterval({ duration: 5000, @@ -134,6 +135,24 @@ const Apps = (props) => { } }) + function sortByKey(array, key) { + return array.sort(function(a, b) { + var x = a[key]; + var y = b[key]; + + if (typeof x == "string") + { + x = (""+x).toLowerCase(); + } + if (typeof y == "string") + { + y = (""+y).toLowerCase(); + } + + return ((x < y) ? 1 : ((x > y) ? -1 : 0)); + }); + } + const appViewStyle = { color: "#ffffff", width: "100%", @@ -169,6 +188,8 @@ const Apps = (props) => { return response.json() }) .then((responseJson) => { + responseJson = sortByKey(responseJson, "large_image") + setApps(responseJson) setFilteredApps(responseJson) if (responseJson.length > 0) { @@ -290,12 +311,18 @@ const Apps = (props) => { { if (selectedApp.id !== data.id) { setSelectedApp(data) + console.log(data) if (data.actions !== undefined && data.actions !== null && data.actions.length > 0) { setSelectedAction(data.actions[0]) } else { setSelectedAction({}) } + + console.log("Sharing: ", data.sharing_config) + if (data.sharing) { + setSharingConfiguration("everyone") + } } }}> @@ -314,9 +341,20 @@ const Apps = (props) => { {description}
- - Sharing: {sharing} - , Valid: {valid} + + {data.tags === null || data.tags === undefined ? null : data.tags.map((tag, index) => { + if (index >= 3) { + return null + } + + return ( + + ) + })} @@ -324,7 +362,7 @@ const Apps = (props) => { {data.activated && data.private_id !== undefined && data.private_id.length > 0 && data.generated ? {downloadApp(data)}}> - + @@ -450,6 +488,7 @@ const Apps = (props) => { ) })} + {/* { displayDataTypes={true} name={"Example return value"} /> + */}
) } return ( -
+
Example return
{selectedAction.returns.example}
) } - //fetch(globalUrl+"/api/v1/get_openapi/"+urlParams.get("id"), { + const userRoles = [ + "you", + "everyone", + ] + + //fetch(globalUrl+"/api/v1/get_openapi/"+urlParams.get("id"), + console.log("User: ", props.userdata) var baseInfo = newAppname.length > 0 ?
@@ -482,15 +528,23 @@ const Apps = (props) => {
{activateButton} - {downloadButton} - {editButton} - {deleteButton} + {props.userdata.role === "admin" || props.userdata.id === selectedApp.owner ? +
+ {downloadButton} + {editButton} + {deleteButton} +
+ : null} {selectedApp.tags !== undefined && selectedApp.tags !== null ?
- {selectedApp.tags.map(tag => { + {selectedApp.tags.map((tag, index) => { + if (index >= 3) { + return null + } + return ( @@ -498,11 +552,41 @@ const Apps = (props) => { })}
: null} - - {selectedApp.link.length > 0 ?

URL: {selectedApp.link}

: null} -

ID: {selectedApp.id}

+ {props.userdata.id === selectedApp.owner ? +
+ {/*

ID: {selectedApp.id}

*/} + Sharing: + +
+ : null} + {/*

Owner: {selectedApp.owner}

*/} {selectedApp.privateId !== undefined && selectedApp.privateId.length > 0 ?

PrivateID: {selectedApp.privateId}

: null} - + + {selectedApp.link.length > 0 ?

URL: {selectedApp.link}

: null}
Actions {selectedApp.actions !== null && selectedApp.actions.length > 0 ? @@ -884,6 +968,36 @@ const Apps = (props) => { }); } + const updateAppField = (app_id, fieldname, fieldvalue) => { + const data = {} + data[fieldname] = fieldvalue + + fetch(globalUrl+"/api/v1/apps/"+app_id, { + method: 'PATCH', + headers: { + 'Accept': 'application/json', + }, + body: JSON.stringify(data), + credentials: "include", + }) + .then((response) => { + //setAppSearchLoading(false) + return response.json() + }) + .then((responseJson) => { + //console.log(responseJson) + //alert.info(responseJson) + if (responseJson.success) { + alert.info("Success") + } else { + alert.error("Error updating app") + } + }) + .catch(error => { + alert.error(error.toString()) + }); + } + const runAppSearch = (searchterm) => { const data = {"search": searchterm} @@ -1165,11 +1279,13 @@ const Apps = (props) => { placeholder="OpenAPI URI" fullWidth /> + {/*
Example:
https://raw.githubusercontent.com/OAI/OpenAPI-Specification/master/examples/v2.0/json/uber.json -

or paste the yaml/JSON directly below

+ */} +

Or paste the yaml/JSON directly below

{ marginTop: "10px", overflow: "scroll", height: "90%", - overflowX: "auto", + overflowX: "hidden", overflowY: "auto", } @@ -212,6 +213,8 @@ const Workflows = (props) => { marginTop: "5px", color: "white", backgroundColor: surfaceColor, + borderRadius: 5, + padding: 10, cursor: "pointer", display: "flex", } @@ -241,6 +244,8 @@ const Workflows = (props) => { setWorkflowExecutions(responseJson) } else { alert.info("Couldn't find executions for the workflow") + setSelectedExecution({}) + setWorkflowExecutions([]) } } }) @@ -393,6 +398,7 @@ const Workflows = (props) => { } // dropdown with copy etc I guess + console.log("Why reset?") const WorkflowPaper = (props) => { const { data } = props; const [open, setOpen] = React.useState(false); @@ -419,7 +425,6 @@ const Workflows = (props) => { var webhookImg = "" var scheduleImg = "" if (data.triggers !== undefined && data.triggers !== null && data.triggers.length > 0) { - console.log("Triggers: ", data.triggers) for (var key in data.triggers) { if (data.triggers[key].app_name === "Webhook") { webhooks += 1 @@ -434,14 +439,9 @@ const Workflows = (props) => { const imgSize = 25 return ( { - }}> - - - {webhooks > 0 ? {data.title} : null} - {schedules > 0 ? {data.title} : null} - -
- + }}> +
+
{ @@ -450,9 +450,11 @@ const Workflows = (props) => { getWorkflowExecution(data.id) } }}> -

{data.name}

+ + {data.name} +
-
+
{ getWorkflowExecution(data.id) } }}> - + + + + {data.tags !== undefined ? data.tags.map(tag => { @@ -533,6 +537,22 @@ const Workflows = (props) => {
+ + + + + + {webhooks > 0 ? + + {data.title} + + : null} + {schedules > 0 ? + + {data.title} + + : null} + ) } @@ -938,7 +958,6 @@ const Workflows = (props) => { setLoadWorkflowsModalOpen(false) } - console.log("WOrkflowtags: ", newWorkflowTags) const modalView = modalOpen ? Date: Wed, 2 Sep 2020 13:39:37 +0200 Subject: [PATCH 12/19] Improved tagging on actions --- .env | 9 +++++++-- backend/go-app/go.mod | 6 ++++-- backend/go-app/go.sum | 14 ++++++++++++++ backend/go-app/main.go | 17 ++++++++++++----- backend/go-app/walkoff.go | 25 +++++++++++++++++-------- docker-compose.yml | 10 +++++----- frontend/src/views/Workflows.jsx | 8 ++++++-- 7 files changed, 65 insertions(+), 24 deletions(-) diff --git a/.env b/.env index 86a2b1f1..dbb4fc87 100644 --- a/.env +++ b/.env @@ -4,9 +4,14 @@ ENVIRONMENT_NAME=Shuffle # Remote github config for first load SHUFFLE_DOWNLOAD_WORKFLOW_LOCATION= +SHUFFLE_DOWNLOAD_WORKFLOW_USERNAME= +SHUFFLE_DOWNLOAD_WORKFLOW_PASSWORD= +SHUFFLE_DOWNLOAD_WORKFLOW_BRANCH= + +SHUFFLE_APP_DOWNLOAD_LOCATION=https://github.com/frikky/shuffle-apps SHUFFLE_DOWNLOAD_AUTH_USERNAME= SHUFFLE_DOWNLOAD_AUTH_PASSWORD= -APP_DOWNLOAD_LOCATION=https://github.com/frikky/shuffle-apps +SHUFFLE_DOWNLOAD_AUTH_BRANCH= # User config for first load. Username & PW: min length 3 SHUFFLE_DEFAULT_USERNAME= @@ -14,7 +19,7 @@ SHUFFLE_DEFAULT_PASSWORD= SHUFFLE_DEFAULT_APIKEY= # Local location of your app directory. Can't use ~/ -APP_HOTLOAD_LOCATION=./shuffle-apps +SHUFFLE_APP_HOTLOAD_LOCATION=./shuffle-apps # Other configs BACKEND_HOSTNAME=shuffle-backend diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 20a8da9f..553b8b3c 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -7,8 +7,9 @@ require ( cloud.google.com/go/datastore v1.1.0 cloud.google.com/go/pubsub v1.3.1 cloud.google.com/go/storage v1.7.0 + github.com/Microsoft/go-winio v0.4.14 // indirect github.com/basgys/goxml2json v1.1.0 - github.com/carlescere/scheduler v0.0.0-20170109141437-ee74d2f83d82 // indirect + github.com/carlescere/scheduler v0.0.0-20170109141437-ee74d2f83d82 github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/docker v1.13.1 github.com/docker/go-connections v0.4.0 @@ -28,7 +29,8 @@ require ( google.golang.org/api v0.23.0 google.golang.org/appengine v1.6.6 google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31 - gopkg.in/src-d/go-git.v4 v4.13.1 // indirect + google.golang.org/grpc v1.29.1 + gopkg.in/src-d/go-git.v4 v4.13.1 gopkg.in/yaml.v2 v2.2.8 gopkg.in/yaml.v3 v3.0.0-20200506231410-2ff61e1afc86 ) diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 8f640a73..4fbd4b69 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -31,8 +31,11 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.7.0 h1:DzdLPI8Em+DEk7IzA2a10ivq3mxIEASC9GeNJ6FFt5Q= cloud.google.com/go/storage v1.7.0/go.mod h1:jGMIBwF+L/tL6WN/W5InNgYYu4HP0DvGB6rQ1mufWfs= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU= +github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= @@ -141,10 +144,12 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= @@ -166,10 +171,13 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -212,6 +220,7 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -219,6 +228,7 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -255,6 +265,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -321,9 +332,11 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200409170454-77362c5149f0/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d h1:lzLdP95xJmMpwQ6LUHwrc5V7js93hTiY7gkznu0BgmY= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -413,6 +426,7 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 75fba54f..1792a9bc 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -41,6 +41,7 @@ import ( "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/storage/memory" // Random @@ -6341,13 +6342,14 @@ func runInit(ctx context.Context) { fs := memfs.New() storer := memory.NewStorage() - url := os.Getenv("APP_DOWNLOAD_LOCATION") + url := os.Getenv("SHUFFLE_APP_DOWNLOAD_LOCATION") if len(url) == 0 { url = "https://github.com/frikky/shuffle-apps" } username := os.Getenv("SHUFFLE_DOWNLOAD_AUTH_USERNAME") password := os.Getenv("SHUFFLE_DOWNLOAD_AUTH_PASSWORD") + cloneOptions := &git.CloneOptions{ URL: url, } @@ -6358,6 +6360,11 @@ func runInit(ctx context.Context) { Password: password, } } + branch := os.Getenv("SHUFFLE_DOWNLOAD_AUTH_BRANCH") + if len(branch) > 0 { + cloneOptions.ReferenceName = plumbing.ReferenceName(branch) + } + log.Printf("Getting apps from %s", url) r, err := git.Clone(storer, fs, cloneOptions) @@ -6377,7 +6384,7 @@ func runInit(ctx context.Context) { iterateAppGithubFolders(fs, dir, "", "", false) // Hotloads locally - location := os.Getenv("APP_HOTLOAD_FOLDER") + location := os.Getenv("SHUFFLE_APP_HOTLOAD_FOLDER") if len(location) != 0 { handleAppHotload(location) } @@ -6417,9 +6424,9 @@ func runInit(ctx context.Context) { log.Printf("Error getting workflows: %s", err) } else { if len(workflows) == 0 { - username := os.Getenv("SHUFFLE_DOWNLOAD_AUTH_USERNAME") - password := os.Getenv("SHUFFLE_DOWNLOAD_AUTH_PASSWORD") - err = loadGithubWorkflows(workflowLocation, username, password, "") + username := os.Getenv("SHUFFLE_DOWNLOAD_WORKFLOW_USERNAME") + password := os.Getenv("SHUFFLE_DOWNLOAD_WORKFLOW_PASSWORD") + err = loadGithubWorkflows(workflowLocation, username, password, "", os.Getenv("SHUFFLE_DOWNLOAD_WORKFLOW_BRANCH")) if err != nil { log.Printf("Failed to upload workflows from github: %s", err) } else { diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index d9fae1ba..86c25561 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -23,14 +23,14 @@ import ( "google.golang.org/api/cloudfunctions/v1" schedulerpb "google.golang.org/genproto/googleapis/cloud/scheduler/v1" + newscheduler "github.com/carlescere/scheduler" + "github.com/getkin/kin-openapi/openapi3" "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-git/v5" - http2 "gopkg.in/src-d/go-git.v4/plumbing/transport/http" - - newscheduler "github.com/carlescere/scheduler" - "github.com/getkin/kin-openapi/openapi3" + "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/storage/memory" + http2 "gopkg.in/src-d/go-git.v4/plumbing/transport/http" //"github.com/gorilla/websocket" //"google.golang.org/appengine" //"google.golang.org/appengine/memcache" @@ -137,6 +137,7 @@ type WorkflowAppActionParameter struct { Variant string `json:"variant" datastore:"variant" yaml:"variant,omitempty"` Required bool `json:"required" datastore:"required" yaml:"required"` Configuration bool `json:"configuration" datastore:"configuration" yaml:"configuration"` + Tags []string `json:"tags" datastore:"tags" yaml:"tags"` Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` } @@ -154,6 +155,7 @@ type WorkflowAppAction struct { Sharing bool `json:"sharing" datastore:"sharing"` PrivateID string `json:"private_id" datastore:"private_id"` AppID string `json:"app_id" datastore:"app_id"` + Tags []string `json:"tags" datastore:"tags" yaml:"tags"` Authentication []AuthenticationStore `json:"authentication" datastore:"authentication,noindex" yaml:"authentication,omitempty"` Tested bool `json:"tested" datastore:"tested" yaml:"tested"` Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters"` @@ -250,6 +252,7 @@ type Trigger struct { Environment string `json:"environment" datastore:"environment"` TriggerType string `json:"trigger_type" datastore:"trigger_type"` Name string `json:"name" datastore:"name"` + Tags []string `json:"tags" datastore:"tags" yaml:"tags"` Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"` Position struct { X float64 `json:"x" datastore:"x"` @@ -4172,7 +4175,7 @@ func deployWebhookFunction(ctx context.Context, name, localization, applocation return nil } -func loadGithubWorkflows(url, username, password, userId string) error { +func loadGithubWorkflows(url, username, password, userId, branch string) error { fs := memfs.New() if strings.Contains(url, "github") || strings.Contains(url, "gitlab") || strings.Contains(url, "bitbucket") { @@ -4189,6 +4192,10 @@ func loadGithubWorkflows(url, username, password, userId string) error { } } + if len(branch) > 0 { + cloneOptions.ReferenceName = plumbing.ReferenceName(branch) + } + storer := memory.NewStorage() r, err := git.Clone(storer, fs, cloneOptions) if err != nil { @@ -4264,6 +4271,7 @@ func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) { URL string `json:"url"` Field1 string `json:"field_1"` Field2 string `json:"field_2"` + Field3 string `json:"field_3"` } //log.Printf("Body: %s", string(body)) @@ -4276,7 +4284,8 @@ func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) { return } - err = loadGithubWorkflows(tmpBody.URL, tmpBody.Field1, tmpBody.Field2, user.Id) + // Field3 = branch + err = loadGithubWorkflows(tmpBody.URL, tmpBody.Field1, tmpBody.Field2, user.Id, tmpBody.Field3) if err != nil { log.Printf("Failed to update workflows: %s", err) resp.WriteHeader(401) @@ -4312,10 +4321,10 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { return } - location := os.Getenv("APP_HOTLOAD_FOLDER") + location := os.Getenv("SHUFFLE_APP_HOTLOAD_FOLDER") if len(location) == 0 { resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "APP_HOTLOAD_FOLDER not specified in .env"}`))) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "SHUFFLE_APP_HOTLOAD_FOLDER not specified in .env"}`))) return } diff --git a/docker-compose.yml b/docker-compose.yml index 8faba6f3..164eeca9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: frontend: - #build: ./frontend + build: ./frontend image: frikky/shuffle:frontend container_name: shuffle-frontend hostname: shuffle-frontend @@ -16,7 +16,7 @@ services: depends_on: - backend backend: - #build: ./backend + build: ./backend image: frikky/shuffle:backend container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} @@ -27,12 +27,12 @@ services: - shuffle volumes: - /var/run/docker.sock:/var/run/docker.sock - - ${APP_HOTLOAD_LOCATION}:/shuffle-apps + - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps environment: - DATASTORE_EMULATOR_HOST=shuffle-database:8000 - - APP_HOTLOAD_FOLDER=/shuffle-apps + - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps - ORG_ID=${ORG_ID} - - APP_DOWNLOAD_LOCATION=${APP_DOWNLOAD_LOCATION} + - SHUFFLE_APP_DOWNLOAD_LOCATION=${SHUFFLE_APP_DOWNLOAD_LOCATION} - SHUFFLE_DEFAULT_USERNAME=${SHUFFLE_DEFAULT_USERNAME} - SHUFFLE_DEFAULT_PASSWORD=${SHUFFLE_DEFAULT_PASSWORD} - SHUFFLE_DEFAULT_APIKEY=${SHUFFLE_DEFAULT_APIKEY} diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 7c4b31ec..66bae799 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -523,10 +523,14 @@ const Workflows = (props) => { {data.tags !== undefined ? - data.tags.map(tag => { + data.tags.map((tag, index) => { + if (index >= 3) { + return null + } + return ( From 9c813006d53f2ecd1b7caf2d5cde5d30aab023d1 Mon Sep 17 00:00:00 2001 From: frikky Date: Wed, 2 Sep 2020 17:58:14 +0200 Subject: [PATCH 13/19] #33: Started docker images migrating to gchr.io --- backend/app_sdk/build.sh | 7 +++++-- docker-compose.yml | 2 +- functions/onprem/orborus/build.sh | 5 +++-- functions/onprem/orborus/orborus.go | 4 +++- functions/onprem/worker/build.sh | 5 +++-- 5 files changed, 15 insertions(+), 8 deletions(-) diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 0f99e856..a8fe1e93 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -3,8 +3,11 @@ NAME=app_sdk VERSION=0.6.0 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force -docker build . -t frikky/shuffle:$NAME -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION +docker build . -t frikky/shuffle:$NAME -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t gchr.io/frikky/app_sdk:0.6.0 #docker push frikky/shuffle:$NAME #docker push frikky/$NAME:$VERSION -docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION +#docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION +#docker push ghcr.io/frikky/$NAME:$VERSION + +docker push gchr.io/frikky/app_sdk diff --git a/docker-compose.yml b/docker-compose.yml index 603deb69..0b6ea2fc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,7 +43,7 @@ services: - database orborus: #build: ./functions/onprem/orborus - image: docker.pkg.github.com/frikky/shuffle/orborus:0.6.0 + image: ghcr.io/frikky/orborus:0.6.0 container_name: shuffle-orborus hostname: shuffle-orborus networks: diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index 285d6772..f34c68f5 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -3,8 +3,9 @@ VERSION=0.6.0 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force -docker build . -t frikky/shuffle:$NAME -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t frikky/$NAME:$VERSION +docker build . -t frikky/shuffle:$NAME -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION #docker push frikky/$NAME:$VERSION #docker push frikky/shuffle:$NAME -docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION +# docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION +docker push ghcr.io/frikky/$NAME:$VERSION diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index fce0608d..61c491cb 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -32,7 +32,9 @@ var sleepTime = 3 var workerTimeout = 300 var appSdkVersion = "0.6.0" var workerVersion = "0.6.0" -var baseimagename = "docker.pkg.github.com/frikky/shuffle" + +//var baseimagename = "docker.pkg.github.com/frikky/shuffle" +var baseimagename = "gchr.io/frikky" var orgId = os.Getenv("ORG_ID") var baseUrl = os.Getenv("BASE_URL") diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh index 786e92c2..c90ce32c 100644 --- a/functions/onprem/worker/build.sh +++ b/functions/onprem/worker/build.sh @@ -3,9 +3,10 @@ VERSION=0.6.0 echo "Running docker build with $NAME:$VERSION" CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin . -docker build . -t frikky/shuffle:$NAME -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t frikky/$NAME:$VERSION +docker build . -t frikky/shuffle:$NAME -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION # Push both for now.. #docker push frikky/$NAME:$VERSION #docker push frikky/shuffle:$NAME -docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION +#docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION +docker push ghcr.io/frikky/$NAME:$VERSION From 991fdfe8949af0a1228c3e409b20b5261358bfbf Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 4 Sep 2020 07:26:35 +0200 Subject: [PATCH 14/19] #130: Added error validation for app creator --- backend/go-app/main.go | 10 +++- backend/go-app/walkoff.go | 3 -- frontend/src/views/Apps.jsx | 78 ++++++++++++++++++++++---------- frontend/src/views/Workflows.jsx | 8 ++-- 4 files changed, 69 insertions(+), 30 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index b50ba5b1..54b29c62 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5458,7 +5458,7 @@ func echoOpenapiData(resp http.ResponseWriter, request *http.Request) { if err != nil { log.Printf("Api authentication failed in validate swagger: %s", err) resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) + resp.Write([]byte(`{"success": false, "reason": "Failed authentication"}`)) return } @@ -5504,6 +5504,12 @@ func echoOpenapiData(resp http.ResponseWriter, request *http.Request) { return } + if newresp.StatusCode >= 400 { + resp.WriteHeader(201) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, urlbody))) + return + } + resp.WriteHeader(200) resp.Write(urlbody) } @@ -5734,6 +5740,8 @@ func validateSwagger(resp http.ResponseWriter, request *http.Request) { err = gyaml.Unmarshal(body, &swagger) if err != nil { log.Printf("Yaml error: %s", err) + } else { + log.Printf("Found valid yaml!") } resp.WriteHeader(422) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 86c25561..bb1d97e8 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -3817,9 +3817,6 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { } } - log.Printf("User: %d", len(user.PrivateApps)) - log.Printf("All: %d", len(workflowapps)) - // Double unmarshal because of user apps newbody, err := json.Marshal(newapps) //newbody, err := json.Marshal(workflowapps) diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 1bdbb68c..105e5dda 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -349,8 +349,10 @@ const Apps = (props) => { return ( ) @@ -515,7 +517,6 @@ const Apps = (props) => { ] //fetch(globalUrl+"/api/v1/get_openapi/"+urlParams.get("id"), - console.log("User: ", props.userdata) var baseInfo = newAppname.length > 0 ?
@@ -544,7 +545,9 @@ const Apps = (props) => { return ( @@ -1030,7 +1033,7 @@ const Apps = (props) => { setValidation(true) fetch(globalUrl+"/api/v1/get_openapi_uri", { - method: 'POST', + method: 'POST', headers: { 'Accept': 'application/json', }, @@ -1038,19 +1041,34 @@ const Apps = (props) => { credentials: "include", }) .then((response) => { + setValidation(false) + if (response.status !== 200) { + return response.json() + } + return response.text() }) - .then((responseText) => { - validateOpenApi(responseText) - setValidation(false) - }) + .then((responseJson) => { + if (typeof(responseJson) !== "string" && !responseJson.success) { + console.log(responseJson.reason) + if (responseJson.reason !== undefined) { + setOpenApiError(responseJson.reason) + } else { + setOpenApiError("Undefined issue with OpenAPI validation") + } + return + } + + validateOpenApi(responseJson) + }) .catch(error => { alert.error(error.toString()) + setOpenApiError(error.toString()) }); } const escapeApiData = (apidata) => { - console.log(apidata) + //console.log(apidata) try { return JSON.stringify(JSON.parse(apidata)) } catch(error) { @@ -1062,7 +1080,7 @@ const Apps = (props) => { return JSON.stringify(YAML.parse(apidata)) } catch(error) { console.log("YAML DECODE ERROR - TRY SOMETHING ELSE?: "+error) - setOpenApiError(error) + setOpenApiError(error.toString()) } return "" @@ -1076,6 +1094,7 @@ const Apps = (props) => { return } + setValidation(true) fetch(globalUrl+"/api/v1/validate_openapi", { method: 'POST', headers: { @@ -1085,10 +1104,10 @@ const Apps = (props) => { credentials: "include", }) .then((response) => { + setValidation(false) return response.json() }) - .then((responseJson) => { - setValidation(false) + .then((responseJson) => { if (responseJson.success) { setAppValidation(responseJson.id) } else { @@ -1099,7 +1118,9 @@ const Apps = (props) => { } }) .catch(error => { + setValidation(false) alert.error(error.toString()) + setOpenApiError(error.toString()) }); } @@ -1245,7 +1266,9 @@ const Apps = (props) => { const modalView = openApiModal ? {setOpenApiModal(false)}} + onClose={() => { + setOpenApiModal(false) + }} PaperProps={{ style: { backgroundColor: surfaceColor, @@ -1269,13 +1292,16 @@ const Apps = (props) => { height: "50px", fontSize: "1em", }, - endAdornment: }} - onChange={e => setOpenApi(e.target.value)} - helperText={
Must point to a version 2 or 3 specification.
} + onChange={e => { + setOpenApi(e.target.value) + }} + helperText={Must point to a version 2 or 3 OpenAPI specification.} placeholder="OpenAPI URI" fullWidth /> @@ -1285,7 +1311,7 @@ const Apps = (props) => {
https://raw.githubusercontent.com/OAI/OpenAPI-Specification/master/examples/v2.0/json/uber.json */} -

Or paste the yaml/JSON directly below

+ Or paste the YAML or JSON specification { color: "white", fontSize: "1em", }, - endAdornment: + }}>Validate OpenAPI }} onChange={e => setOpenApiData(e.target.value)} - helperText={
Must point to a version 2 or 3 specification.
} + helperText={Must point to a version 2 or 3 specification.} placeholder="OpenAPI text" fullWidth /> @@ -1311,13 +1337,19 @@ const Apps = (props) => { {circularLoader} - - + diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 66bae799..a358a376 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -530,8 +530,10 @@ const Workflows = (props) => { return ( ) @@ -542,17 +544,17 @@ const Workflows = (props) => { - + {webhooks > 0 ? - + {data.title} : null} {schedules > 0 ? - + {data.title} : null} From 72566bf3283e0b5eb9dc32c610f7b3debc3fc8e8 Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 4 Sep 2020 18:57:14 +0200 Subject: [PATCH 15/19] Fixed OpenAPI issue by @owentl --- frontend/src/views/AngularWorkflow.jsx | 32 ++++++++++++++++---------- frontend/src/views/AppCreator.jsx | 32 +++++++++++++++++++++++--- frontend/src/views/Apps.jsx | 3 ++- 3 files changed, 51 insertions(+), 16 deletions(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 018d6004..e82d607a 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -639,7 +639,7 @@ const AngularWorkflow = (props) => { } if (executionText.length > 0) { - alert.success("Starting execution with argument "+executionText) + alert.success("Starting execution with an execution argument") } else { alert.success("Starting execution") } @@ -5037,6 +5037,7 @@ const AngularWorkflow = (props) => { @@ -5167,8 +5168,10 @@ const AngularWorkflow = (props) => { return (
-

Execution Argument:

- {executionData.execution_argument} +

Execution Argument

+
+ {executionData.execution_argument} +
) } @@ -5272,14 +5275,7 @@ const AngularWorkflow = (props) => { -

Executing Workflow

- Show failed / skipped actions
} - control={ - {setShowSkippedActions(!showSkippedActions)}} /> - } - /> +

Executing Workflow

{executionData.status !== undefined && executionData.status.length > 0 ?
Status: {executionData.status} @@ -5292,7 +5288,7 @@ const AngularWorkflow = (props) => {
: null } - {executionData.completed_at !== undefined ? + {executionData.completed_at !== undefined && executionData.completed_at !== null && executionData.completed_at > 0 ?
Finished: {new Date(executionData.completed_at*1000).toISOString()}
@@ -5308,7 +5304,19 @@ const AngularWorkflow = (props) => { parsedExecutionArgument() : null } + {executionData.results !== undefined && executionData.results !== null && executionData.results.length > 1 && executionData.results.find(result => result.status === "SKIPPED" || result.status === "FAILURE") ? + Show failed / skipped actions
} + control={ + {setShowSkippedActions(!showSkippedActions)}} /> + } + /> + : + null + }
+ Actions
{executionData.status !== undefined && executionData.status !== "ABORTED" && executionData.status !== "FINISHED" && executionData.status !== "FAILURE" ? : null} diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 9768ada1..030586fd 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -334,6 +334,7 @@ const AppCreator = (props) => { } }) .catch(error => { + console.log("Error: ", error.toString()) alert.error(error.toString()) }); } @@ -405,11 +406,33 @@ const AppCreator = (props) => { securitySchemes = data.components.securitySchemes } + const allowedfunctions = [ + "GET", + "CONNECT", + "HEAD", + "DELETE", + "POST", + "PATCH", + "PUT", + ] + // FIXME - headers? var newActions = [] var wordlist = {} for (let [path, pathvalue] of Object.entries(data.paths)) { for (let [method, methodvalue] of Object.entries(pathvalue)) { + if (methodvalue === null) { + alert.info("Skipped method "+method) + continue + } + + if (!allowedfunctions.includes(method.toUpperCase())) { + console.log(method, path) + continue + } + + console.log("Method: ", method) + console.log("Methodval: ", methodvalue) var newaction = { "name": methodvalue.summary, "description": methodvalue.description, @@ -446,8 +469,11 @@ const AppCreator = (props) => { // https://swagger.io/docs/specification/describing-parameters/ // Need to split the data. } else if (parameter.in === "body") { - console.log("BODY: ", parameter) - newaction.body = parameter.example + // FIXME: Add tracking for components + // E.G: https://raw.githubusercontent.com/owentl/Shuffle/master/gosecure.yaml + if (parameter.example !== undefined) { + newaction.body = parameter.example + } } else if (parameter.in === "header") { newaction.headers += `${parameter.name}=${parameter.example}\n` } @@ -1732,7 +1758,7 @@ const AppCreator = (props) => { margin="normal" variant="outlined" value={baseUrl} - helperText={
Must start with http(s):// and CANT end with /.
} + helperText={Must start with http(s):// and CANT end with /. } placeholder="https://api.example.com" onChange={e => setBaseUrl(e.target.value)} onBlur={(event) => { diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 105e5dda..0aece1c0 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -669,8 +669,9 @@ const Apps = (props) => { How it works  - Security API's  - OpenAPI directory +  - OpenAPI Validator
- Apps interact with eachother in workflows. They are created with the app creator, using OpenAPI specification or manually in python. Use the links above to find potential apps you're looking for using OpenAPI or make one from scratch. There's 1000+ available. + Apps interact with eachother in workflows. They are created with the app creator, using OpenAPI specification or manually in python. The links above are references to OpenAPI tools and other app repositories. There's ten thousands of them.
From b635b6f3e6045f73bab30d00c0c862774d3e99ef Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 5 Sep 2020 06:36:40 +0200 Subject: [PATCH 16/19] Fixed private OpenAPI app deletion --- backend/go-app/walkoff.go | 17 ++++++++--------- frontend/src/views/AppCreator.jsx | 8 ++++---- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index bb1d97e8..f935a212 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -3340,16 +3340,15 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true"}`))) return } - } else { + } - log.Printf("Deleting public app") - err = DeleteKey(ctx, "workflowapp", fileId) - if err != nil { - log.Printf("Failed deleting workflowapp") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed deleting workflow app"}`))) - return - } + log.Printf("Deleting public app") + err = DeleteKey(ctx, "workflowapp", fileId) + if err != nil { + log.Printf("Failed deleting workflowapp") + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed deleting workflow app"}`))) + return } err = increaseStatisticsField(ctx, "total_apps_deleted", fileId, 1) diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 030586fd..d24c2813 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -1290,7 +1290,7 @@ const AppCreator = (props) => {
New action
 - Learn more about actions + Learn more about actions
Name {

Actions

Actions are the tasks performed by an app. Read more about actions and apps - here. + here.
{loopActions} + : null var editButton = selectedApp.activated && selectedApp.private_id !== undefined && selectedApp.private_id.length > 0 && selectedApp.generated ? - : null + + + + : null var activateButton = selectedApp.generated && !selectedApp.activated ? - - : null +
+ + + + + + +
+ : null var deleteButton = ((selectedApp.private_id !== undefined && selectedApp.private_id.length > 0 && selectedApp.generated) || (selectedApp.downloaded != undefined && selectedApp.downloaded == true)) && activateButton === null ? + : null + + + : null var imageline = selectedApp.large_image === undefined || selectedApp.large_image.length === 0 ? {selectedApp.title} @@ -589,74 +614,76 @@ const Apps = (props) => { {/*

Owner: {selectedApp.owner}

*/} {selectedApp.privateId !== undefined && selectedApp.privateId.length > 0 ?

PrivateID: {selectedApp.privateId}

: null} - {selectedApp.link.length > 0 ?

URL: {selectedApp.link}

: null} -
- Actions - {selectedApp.actions !== null && selectedApp.actions.length > 0 ? - { + setSelectedAction(event.target.value) + }} + style={{backgroundColor: inputColor, color: "white", height: "50px"}} + SelectDisplayProps={{ + style: { + marginLeft: 10, + } + }} + > + {selectedApp.actions.map(data => { + var newActionname = data.label !== undefined && data.label.length > 0 ? data.label : data.name - // ROFL FIXME - loop - newActionname = newActionname.replace("_", " ") - newActionname = newActionname.replace("_", " ") - newActionname = newActionname.replace("_", " ") - newActionname = newActionname.replace("_", " ") - newActionname = newActionname.charAt(0).toUpperCase()+newActionname.substring(1) + // ROFL FIXME - loop + newActionname = newActionname.replace("_", " ") + newActionname = newActionname.replace("_", " ") + newActionname = newActionname.replace("_", " ") + newActionname = newActionname.replace("_", " ") + newActionname = newActionname.charAt(0).toUpperCase()+newActionname.substring(1) + return ( + + {newActionname} + + + ) + })} + + : +
+ There are no actions defined for this app. +
+ } +
+ + {selectedAction.parameters !== undefined && selectedAction.parameters !== null ? +
+ Arguments + {selectedAction.parameters.map(data => { + var itemColor = "#f85a3e" + if (!data.required) { + itemColor = "#ffeb3b" + } + + const circleSize = 10 return ( - {newActionname} +
+ {data.name} ) })} - - : -
- There are no actions defined for this app.
- } -
- - {selectedAction.parameters !== undefined && selectedAction.parameters !== null ? -
- Arguments - {selectedAction.parameters.map(data => { - var itemColor = "#f85a3e" - if (!data.required) { - itemColor = "#ffeb3b" - } - - const circleSize = 10 - return ( - -
- {data.name} - - - ) - })} -
- : null} - {selectedAction.description !== undefined && selectedAction.description !== null && selectedAction.description.length > 0 ? -
- Action Description
- {selectedAction.description} -
: null} - + {selectedAction.description !== undefined && selectedAction.description !== null && selectedAction.description.length > 0 ? +
+ Action Description
+ {selectedAction.description} +
+ : null} + +
: null diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index a358a376..8542790d 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -398,7 +398,6 @@ const Workflows = (props) => { } // dropdown with copy etc I guess - console.log("Why reset?") const WorkflowPaper = (props) => { const { data } = props; const [open, setOpen] = React.useState(false); diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index 4902adce..cbea526b 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,5 +1,5 @@ NAME=orborus -VERSION=0.6.1 +VERSION=0.6.2 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 993ad2e7..8e48ce62 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -263,7 +263,7 @@ func main() { // FIXME - during init, BUILD and/or LOAD worker and app_sdk // Build/load app_sdk so it can be loaded as 127.0.0.1:5000/walkoff_app_sdk log.Printf("[INFO] Setting up Docker environment. Downloading worker and App SDK!") - initializeImages() + go initializeImages() //workerName := "worker" //workerVersion := "0.1.0"