diff --git a/.env b/.env index 86a2b1f1..dbb4fc87 100644 --- a/.env +++ b/.env @@ -4,9 +4,14 @@ ENVIRONMENT_NAME=Shuffle # Remote github config for first load SHUFFLE_DOWNLOAD_WORKFLOW_LOCATION= +SHUFFLE_DOWNLOAD_WORKFLOW_USERNAME= +SHUFFLE_DOWNLOAD_WORKFLOW_PASSWORD= +SHUFFLE_DOWNLOAD_WORKFLOW_BRANCH= + +SHUFFLE_APP_DOWNLOAD_LOCATION=https://github.com/frikky/shuffle-apps SHUFFLE_DOWNLOAD_AUTH_USERNAME= SHUFFLE_DOWNLOAD_AUTH_PASSWORD= -APP_DOWNLOAD_LOCATION=https://github.com/frikky/shuffle-apps +SHUFFLE_DOWNLOAD_AUTH_BRANCH= # User config for first load. Username & PW: min length 3 SHUFFLE_DEFAULT_USERNAME= @@ -14,7 +19,7 @@ SHUFFLE_DEFAULT_PASSWORD= SHUFFLE_DEFAULT_APIKEY= # Local location of your app directory. Can't use ~/ -APP_HOTLOAD_LOCATION=./shuffle-apps +SHUFFLE_APP_HOTLOAD_LOCATION=./shuffle-apps # Other configs BACKEND_HOSTNAME=shuffle-backend diff --git a/backend/Dockerfile b/backend/Dockerfile index 14d9137c..d203ed78 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -24,7 +24,7 @@ RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o webapp . FROM alpine:latest as certs RUN apk --update add ca-certificates -from scratch +FROM alpine:3.12 COPY --from=builder /app/ /app COPY --from=builder /app_sdk/ /app_sdk diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index ae57b0d2..29573512 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -804,7 +804,15 @@ class AppBase: #submatch = "([${]{2}([0-9a-zA-Z_-]+)(\[.*\])[}$]{2})" submatch = "([${]{2}([0-9a-zA-Z_-]+)(\[.*\])[}$]{2})" actualitem = re.findall(submatch, value, re.MULTILINE) - print("Multicheck ", actualitem) + try: + if action["skip_multicheck"]: + print("Skipping multicheck") + actualitem = [] + except KeyError: + pass + + actionname = action["name"] + #print("Multicheck ", actualitem) if len(actualitem) > 0: multiexecution = True diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index f9dadcad..0c7af3f7 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,9 +1,13 @@ #!/bin/bash NAME=app_sdk -VERSION=0.2.0 +VERSION=0.6.1 -docker rmi frikky/shuffle:$NAME --force -docker build . -t frikky/shuffle:$NAME -t frikky/$NAME:$VERSION +docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force +docker build . -t frikky/shuffle:$NAME -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/app_sdk:0.6.0 + +#docker push frikky/$NAME:$VERSION +#docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION +#docker push ghcr.io/frikky/$NAME:$VERSION docker push frikky/shuffle:$NAME -docker push frikky/$NAME:$VERSION +docker push ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index 88644465..2b3be803 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -459,7 +459,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, // Jesus what a clusterfuck. // Handles parsing of categories from OpenApi3 custom field if val, ok := swagger.Info.ExtensionProps.Extensions["x-categories"]; ok { - log.Printf("Categories: %#v", val) + //log.Printf("Categories: %#v", val) j, err := json.Marshal(&val) if err == nil { if j[0] == 0x22 && j[len(j)-1] == 0x22 { diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index 865d5867..a68e1643 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -11,11 +11,12 @@ import ( "fmt" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" - network "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" - natting "github.com/docker/go-connections/nat" "github.com/go-git/go-billy/v5" + network "github.com/docker/docker/api/types/network" + natting "github.com/docker/go-connections/nat" + "io" "io/ioutil" "log" @@ -180,9 +181,10 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin // Dockerfile is inside the TAR itself. Not local context // docker build --build-arg http_proxy=http://my.proxy.url buildOptions := types.ImageBuildOptions{ - Remove: true, - Tags: tags, - BuildArgs: map[string]*string{}, + Remove: true, + Tags: tags, + BuildArgs: map[string]*string{}, + NetworkMode: "host", } httpProxy := os.Getenv("HTTP_PROXY") @@ -242,9 +244,10 @@ func buildImage(tags []string, dockerfileFolder string) error { dockerFileTarReader := bytes.NewReader(buf.Bytes()) buildOptions := types.ImageBuildOptions{ - Remove: true, - Tags: tags, - BuildArgs: map[string]*string{}, + Remove: true, + Tags: tags, + BuildArgs: map[string]*string{}, + NetworkMode: "host", } httpProxy := os.Getenv("HTTP_PROXY") @@ -639,6 +642,51 @@ func handleStartHookDocker(resp http.ResponseWriter, request *http.Request) { return } +// Checks if an image exists +func imageCheckBuilder(images []string) error { + log.Printf("[FIXME] ImageNames to check: %#v", images) + return nil + + ctx := context.Background() + client, err := client.NewEnvClient() + if err != nil { + log.Printf("Unable to create docker client: %s", err) + return err + } + + allImages, err := client.ImageList(ctx, types.ImageListOptions{ + All: true, + }) + + if err != nil { + log.Printf("[ERROR] Failed creating imagelist: %s", err) + return err + } + + filteredImages := []types.ImageSummary{} + for _, image := range allImages { + found := false + for _, repoTag := range image.RepoTags { + if strings.Contains(repoTag, baseDockerName) { + found = true + break + } + } + + if found { + filteredImages = append(filteredImages, image) + } + } + + // FIXME: Continue fixing apps here + // https://github.com/frikky/Shuffle/issues/135 + // 1. Find if app exists + // 2. Create app if it doesn't + //log.Printf("Apps: %#v", filteredImages) + + return nil +} + func hookTest() { var hook Hook err := json.Unmarshal([]byte(webhook), &hook) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 20a8da9f..553b8b3c 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -7,8 +7,9 @@ require ( cloud.google.com/go/datastore v1.1.0 cloud.google.com/go/pubsub v1.3.1 cloud.google.com/go/storage v1.7.0 + github.com/Microsoft/go-winio v0.4.14 // indirect github.com/basgys/goxml2json v1.1.0 - github.com/carlescere/scheduler v0.0.0-20170109141437-ee74d2f83d82 // indirect + github.com/carlescere/scheduler v0.0.0-20170109141437-ee74d2f83d82 github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/docker v1.13.1 github.com/docker/go-connections v0.4.0 @@ -28,7 +29,8 @@ require ( google.golang.org/api v0.23.0 google.golang.org/appengine v1.6.6 google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31 - gopkg.in/src-d/go-git.v4 v4.13.1 // indirect + google.golang.org/grpc v1.29.1 + gopkg.in/src-d/go-git.v4 v4.13.1 gopkg.in/yaml.v2 v2.2.8 gopkg.in/yaml.v3 v3.0.0-20200506231410-2ff61e1afc86 ) diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 8f640a73..4fbd4b69 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -31,8 +31,11 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.7.0 h1:DzdLPI8Em+DEk7IzA2a10ivq3mxIEASC9GeNJ6FFt5Q= cloud.google.com/go/storage v1.7.0/go.mod h1:jGMIBwF+L/tL6WN/W5InNgYYu4HP0DvGB6rQ1mufWfs= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU= +github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= @@ -141,10 +144,12 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= @@ -166,10 +171,13 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -212,6 +220,7 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -219,6 +228,7 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -255,6 +265,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -321,9 +332,11 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200409170454-77362c5149f0/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d h1:lzLdP95xJmMpwQ6LUHwrc5V7js93hTiY7gkznu0BgmY= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -413,6 +426,7 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 7e96623f..5f5cf11f 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -41,6 +41,7 @@ import ( "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/storage/memory" // Random @@ -141,7 +142,7 @@ type UserLimits struct { // Saves some data, not sure what to have here lol type UserAuth struct { - Description string `json:"description" datastore:"description" yaml:"description"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` Name string `json:"name" datastore:"name" yaml:"name"` Workflows []string `json:"workflows" datastore:"workflows"` Username string `json:"username" datastore:"username"` @@ -210,7 +211,7 @@ type Translator struct { Src struct { Name string `json:"name" datastore:"name"` Value string `json:"value" datastore:"value"` - Description string `json:"description" datastore:"description"` + Description string `json:"description" datastore:"description,noindex"` Required string `json:"required" datastore:"required"` Type string `json:"type" datastore:"type"` Schema struct { @@ -221,7 +222,7 @@ type Translator struct { Name string `json:"name" datastore:"name"` Value string `json:"value" datastore:"value"` Type string `json:"type" datastore:"type"` - Description string `json:"description" datastore:"description"` + Description string `json:"description" datastore:"description,noindex"` Required string `json:"required" datastore:"required"` Schema struct { Type string `json:"type" datastore:"type"` @@ -284,7 +285,7 @@ type ApiYaml struct { Name string `json:"name" yaml:"name" required:"true datastore:"name"` Foldername string `json:"foldername" yaml:"foldername" required:"true datastore:"foldername"` Id string `json:"id" yaml:"id",required:"true, datastore:"id"` - Description string `json:"description" datastore:"description" yaml:"description"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` AppVersion string `json:"app_version" yaml:"app_version",datastore:"app_version"` ContactInfo struct { Name string `json:"name" datastore:"name" yaml:"name"` @@ -293,10 +294,10 @@ type ApiYaml struct { Types []string `json:"types" datastore:"types" yaml:"types"` Input []struct { Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description" yaml:"description"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` InputParameters []struct { Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description" yaml:"description"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` Required string `json:"required" datastore:"required" yaml:"required"` Schema struct { Type string `json:"type" datastore:"type" yaml:"type"` @@ -304,7 +305,7 @@ type ApiYaml struct { } `json:"inputparameters" datastore:"inputparameters" yaml:"inputparameters"` OutputParameters []struct { Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description" yaml:"description"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` Required string `json:"required" datastore:"required" yaml:"required"` Schema struct { Type string `json:"type" datastore:"type" yaml:"type"` @@ -312,7 +313,7 @@ type ApiYaml struct { } `json:"outputparameters" datastore:"outputparameters" yaml:"outputparameters"` Config []struct { Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description" yaml:"description"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` Required string `json:"required" datastore:"required" yaml:"required"` Schema struct { Type string `json:"type" datastore:"type" yaml:"type"` @@ -321,10 +322,10 @@ type ApiYaml struct { } `json:"input" datastore:"input" yaml:"input"` Output []struct { Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description" yaml:"description"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` Config []struct { Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description" yaml:"description"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` Required string `json:"required" datastore:"required" yaml:"required"` Schema struct { Type string `json:"type" datastore:"type" yaml:"type"` @@ -332,7 +333,7 @@ type ApiYaml struct { } `json:"config" datastore:"config" yaml:"config"` InputParameters []struct { Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description" yaml:"description"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` Required string `json:"required" datastore:"required" yaml:"required"` Schema struct { Type string `json:"type" datastore:"type" yaml:"type"` @@ -340,7 +341,7 @@ type ApiYaml struct { } `json:"inputparameters" datastore:"inputparameters" yaml:"inputparameters"` OutputParameters []struct { Name string `json:"name" datastore:"name" yaml:"name"` - Description string `json:"description" datastore:"description" yaml:"description"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` Required string `json:"required" datastore:"required" yaml:"required"` Schema struct { Type string `json:"type" datastore:"type" yaml:"type"` @@ -357,7 +358,7 @@ type Hooks struct { type Info struct { Url string `json:"url" datastore:"url"` Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description"` + Description string `json:"description" datastore:"description,noindex"` } // Actions to be done by webhooks etc @@ -665,7 +666,7 @@ func handleApiAuthentication(resp http.ResponseWriter, request *http.Request) (U if len(Userdata.Username) > 0 { return Userdata, nil } else { - return Userdata, errors.New(fmt.Sprintf("User is invalid - no username found: %#v", Userdata)) + return Userdata, errors.New(fmt.Sprintf("User is invalid - no username found")) } } @@ -1697,10 +1698,11 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { "success": true, "admin": %s, "tutorials": [], + "id": "%s", "orgs": [{"name": "Shuffle", "id": "123", "role": "admin"}], "selected_org": {"name": "Shuffle", "id": "123", "role": "admin"}, "cookies": [{"key": "session_token", "value": "%s", "expiration": %d}] - }`, parsedAdmin, userInfo.Session, expiration.Unix()) + }`, parsedAdmin, userInfo.Id, userInfo.Session, expiration.Unix()) resp.WriteHeader(200) resp.Write([]byte(returnData)) @@ -2505,7 +2507,7 @@ func handleCors(resp http.ResponseWriter, request *http.Request) bool { resp.Header().Set("Vary", "Origin") resp.Header().Set("Access-Control-Allow-Headers", "Content-Type, Accept, X-Requested-With, remember-me") - resp.Header().Set("Access-Control-Allow-Methods", "POST, GET, PUT, DELETE") + resp.Header().Set("Access-Control-Allow-Methods", "POST, GET, PUT, DELETE, PATCH") resp.Header().Set("Access-Control-Allow-Credentials", "true") resp.Header().Set("Access-Control-Allow-Origin", allowedOrigins) @@ -5456,7 +5458,7 @@ func echoOpenapiData(resp http.ResponseWriter, request *http.Request) { if err != nil { log.Printf("Api authentication failed in validate swagger: %s", err) resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) + resp.Write([]byte(`{"success": false, "reason": "Failed authentication"}`)) return } @@ -5502,6 +5504,12 @@ func echoOpenapiData(resp http.ResponseWriter, request *http.Request) { return } + if newresp.StatusCode >= 400 { + resp.WriteHeader(201) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, urlbody))) + return + } + resp.WriteHeader(200) resp.Write(urlbody) } @@ -5732,6 +5740,8 @@ func validateSwagger(resp http.ResponseWriter, request *http.Request) { err = gyaml.Unmarshal(body, &swagger) if err != nil { log.Printf("Yaml error: %s", err) + } else { + log.Printf("Found valid yaml!") } resp.WriteHeader(422) @@ -6145,7 +6155,7 @@ func createFs(basepath, pathname string) (billy.Filesystem, error) { } // Hotloads new apps from a folder -func handleAppHotload(location string) error { +func handleAppHotload(location string, forceUpdate bool) error { basepath := "base" fs, err := createFs(basepath, location) if err != nil { @@ -6162,7 +6172,7 @@ func handleAppHotload(location string) error { } //log.Printf("Reading app folder: %#v", dir) - err = iterateAppGithubFolders(fs, dir, "", "", false) + err = iterateAppGithubFolders(fs, dir, "", "", forceUpdate) if err != nil { log.Printf("Err: %s", err) return err @@ -6340,13 +6350,14 @@ func runInit(ctx context.Context) { fs := memfs.New() storer := memory.NewStorage() - url := os.Getenv("APP_DOWNLOAD_LOCATION") + url := os.Getenv("SHUFFLE_APP_DOWNLOAD_LOCATION") if len(url) == 0 { url = "https://github.com/frikky/shuffle-apps" } username := os.Getenv("SHUFFLE_DOWNLOAD_AUTH_USERNAME") password := os.Getenv("SHUFFLE_DOWNLOAD_AUTH_PASSWORD") + cloneOptions := &git.CloneOptions{ URL: url, } @@ -6357,6 +6368,11 @@ func runInit(ctx context.Context) { Password: password, } } + branch := os.Getenv("SHUFFLE_DOWNLOAD_AUTH_BRANCH") + if len(branch) > 0 { + cloneOptions.ReferenceName = plumbing.ReferenceName(branch) + } + log.Printf("Getting apps from %s", url) r, err := git.Clone(storer, fs, cloneOptions) @@ -6376,9 +6392,9 @@ func runInit(ctx context.Context) { iterateAppGithubFolders(fs, dir, "", "", false) // Hotloads locally - location := os.Getenv("APP_HOTLOAD_FOLDER") + location := os.Getenv("SHUFFLE_APP_HOTLOAD_FOLDER") if len(location) != 0 { - handleAppHotload(location) + handleAppHotload(location, false) } } @@ -6416,9 +6432,9 @@ func runInit(ctx context.Context) { log.Printf("Error getting workflows: %s", err) } else { if len(workflows) == 0 { - username := os.Getenv("SHUFFLE_DOWNLOAD_AUTH_USERNAME") - password := os.Getenv("SHUFFLE_DOWNLOAD_AUTH_PASSWORD") - err = loadGithubWorkflows(workflowLocation, username, password, "") + username := os.Getenv("SHUFFLE_DOWNLOAD_WORKFLOW_USERNAME") + password := os.Getenv("SHUFFLE_DOWNLOAD_WORKFLOW_PASSWORD") + err = loadGithubWorkflows(workflowLocation, username, password, "", os.Getenv("SHUFFLE_DOWNLOAD_WORKFLOW_BRANCH")) if err != nil { log.Printf("Failed to upload workflows from github: %s", err) } else { @@ -6497,6 +6513,7 @@ func init() { r.HandleFunc("/api/v1/apps/run_hotload", handleAppHotloadRequest).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/apps/get_existing", loadSpecificApps).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/download_remote", loadSpecificApps).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/apps/{appId}", updateWorkflowAppConfig).Methods("PATCH", "OPTIONS") r.HandleFunc("/api/v1/apps/validate", validateAppInput).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/{appId}", deleteWorkflowApp).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/apps/{appId}/config", getWorkflowAppConfig).Methods("GET", "OPTIONS") diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index a72d6a32..0d06dc37 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -23,14 +23,14 @@ import ( "google.golang.org/api/cloudfunctions/v1" schedulerpb "google.golang.org/genproto/googleapis/cloud/scheduler/v1" + newscheduler "github.com/carlescere/scheduler" + "github.com/getkin/kin-openapi/openapi3" "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-git/v5" - http2 "gopkg.in/src-d/go-git.v4/plumbing/transport/http" - - newscheduler "github.com/carlescere/scheduler" - "github.com/getkin/kin-openapi/openapi3" + "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/storage/memory" + http2 "gopkg.in/src-d/go-git.v4/plumbing/transport/http" //"github.com/gorilla/websocket" //"google.golang.org/appengine" //"google.golang.org/appengine/memcache" @@ -42,14 +42,10 @@ var localBase = "http://localhost:5001" var baseEnvironment = "onprem" var cloudname = "cloud" - var defaultLocation = "europe-west2" var scheduledJobs = map[string]*newscheduler.Job{} // To test out firestore before potential merge -var shuffleTestProject = "shuffle-test-258209" -var shuffleTestPath = "./shuffle-test-258209-5a2e8d7e508a.json" - //var upgrader = websocket.Upgrader{ // ReadBufferSize: 1024, // WriteBufferSize: 1024, @@ -96,25 +92,26 @@ type AuthenticationUsage struct { // An app inside Shuffle // Source string `json:"source" datastore:"soure" yaml:"source"` - downloadlocation type WorkflowApp struct { - Name string `json:"name" yaml:"name" required:true datastore:"name"` - IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"` - ID string `json:"id" yaml:"id,omitempty" required:false datastore:"id"` - Link string `json:"link" yaml:"link" required:false datastore:"link,noindex"` - AppVersion string `json:"app_version" yaml:"app_version" required:true datastore:"app_version"` - Generated bool `json:"generated" yaml:"generated" required:false datastore:"generated"` - Downloaded bool `json:"downloaded" yaml:"downloaded" required:false datastore:"downloaded"` - Sharing bool `json:"sharing" yaml:"sharing" required:false datastore:"sharing"` - Verified bool `json:"verified" yaml:"verified" required:false datastore:"verified"` - Activated bool `json:"activated" yaml:"activated" required:false datastore:"activated"` - Tested bool `json:"tested" yaml:"tested" required:false datastore:"tested"` - Owner string `json:"owner" datastore:"owner" yaml:"owner"` - Hash string `json:"hash" datastore:"hash" yaml:"hash"` // api.yaml+dockerfile+src/app.py for apps - PrivateID string `json:"private_id" yaml:"private_id" required:false datastore:"private_id"` - Description string `json:"description" datastore:"description,noindex" required:false yaml:"description"` - Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"` - SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"` - LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false` - ContactInfo struct { + Name string `json:"name" yaml:"name" required:true datastore:"name"` + IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"` + ID string `json:"id" yaml:"id,omitempty" required:false datastore:"id"` + Link string `json:"link" yaml:"link" required:false datastore:"link,noindex"` + AppVersion string `json:"app_version" yaml:"app_version" required:true datastore:"app_version"` + SharingConfig string `json:"sharing_config" yaml:"sharing_config" datastore:"sharing_config"` + Generated bool `json:"generated" yaml:"generated" required:false datastore:"generated"` + Downloaded bool `json:"downloaded" yaml:"downloaded" required:false datastore:"downloaded"` + Sharing bool `json:"sharing" yaml:"sharing" required:false datastore:"sharing"` + Verified bool `json:"verified" yaml:"verified" required:false datastore:"verified"` + Activated bool `json:"activated" yaml:"activated" required:false datastore:"activated"` + Tested bool `json:"tested" yaml:"tested" required:false datastore:"tested"` + Owner string `json:"owner" datastore:"owner" yaml:"owner"` + Hash string `json:"hash" datastore:"hash" yaml:"hash"` // api.yaml+dockerfile+src/app.py for apps + PrivateID string `json:"private_id" yaml:"private_id" required:false datastore:"private_id"` + Description string `json:"description" datastore:"description,noindex" required:false yaml:"description"` + Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"` + SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"` + LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false` + ContactInfo struct { Name string `json:"name" datastore:"name" yaml:"name"` Url string `json:"url" datastore:"url" yaml:"url"` } `json:"contact_info" datastore:"contact_info" yaml:"contact_info" required:false` @@ -125,18 +122,20 @@ type WorkflowApp struct { } type WorkflowAppActionParameter struct { - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - ID string `json:"id" datastore:"id" yaml:"id,omitempty"` - Name string `json:"name" datastore:"name" yaml:"name"` - Example string `json:"example" datastore:"example" yaml:"example"` - Value string `json:"value" datastore:"value" yaml:"value,omitempty"` - Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` - Options []string `json:"options" datastore:"options" yaml:"options"` - ActionField string `json:"action_field" datastore:"action_field" yaml:"actionfield,omitempty"` - Variant string `json:"variant" datastore:"variant" yaml:"variant,omitempty"` - Required bool `json:"required" datastore:"required" yaml:"required"` - Configuration bool `json:"configuration" datastore:"configuration" yaml:"configuration"` - Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` + ID string `json:"id" datastore:"id" yaml:"id,omitempty"` + Name string `json:"name" datastore:"name" yaml:"name"` + Example string `json:"example" datastore:"example" yaml:"example"` + Value string `json:"value" datastore:"value" yaml:"value,omitempty"` + Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` + Options []string `json:"options" datastore:"options" yaml:"options"` + ActionField string `json:"action_field" datastore:"action_field" yaml:"actionfield,omitempty"` + Variant string `json:"variant" datastore:"variant" yaml:"variant,omitempty"` + Required bool `json:"required" datastore:"required" yaml:"required"` + Configuration bool `json:"configuration" datastore:"configuration" yaml:"configuration"` + Tags []string `json:"tags" datastore:"tags" yaml:"tags"` + Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` + SkipMulticheck bool `json:"skip_multicheck" datastore:"skip_multicheck" yaml:"skip_multicheck"` } type SchemaDefinition struct { @@ -144,7 +143,7 @@ type SchemaDefinition struct { } type WorkflowAppAction struct { - Description string `json:"description" datastore:"description"` + Description string `json:"description" datastore:"description,noindex"` ID string `json:"id" datastore:"id" yaml:"id,omitempty"` Name string `json:"name" datastore:"name"` Label string `json:"label" datastore:"label"` @@ -153,11 +152,12 @@ type WorkflowAppAction struct { Sharing bool `json:"sharing" datastore:"sharing"` PrivateID string `json:"private_id" datastore:"private_id"` AppID string `json:"app_id" datastore:"app_id"` + Tags []string `json:"tags" datastore:"tags" yaml:"tags"` Authentication []AuthenticationStore `json:"authentication" datastore:"authentication,noindex" yaml:"authentication,omitempty"` Tested bool `json:"tested" datastore:"tested" yaml:"tested"` Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters"` ExecutionVariable struct { - Description string `json:"description" datastore:"description"` + Description string `json:"description" datastore:"description,noindex"` ID string `json:"id" datastore:"id"` Name string `json:"name" datastore:"name"` Value string `json:"value" datastore:"value"` @@ -192,7 +192,7 @@ type WorkflowExecution struct { Workflow Workflow `json:"workflow" datastore:"workflow,noindex"` Results []ActionResult `json:"results" datastore:"results,noindex"` ExecutionVariables []struct { - Description string `json:"description" datastore:"description"` + Description string `json:"description" datastore:"description,noindex"` ID string `json:"id" datastore:"id"` Name string `json:"name" datastore:"name"` Value string `json:"value" datastore:"value,noindex"` @@ -217,7 +217,7 @@ type Action struct { Name string `json:"name" datastore:"name"` Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"` ExecutionVariable struct { - Description string `json:"description" datastore:"description"` + Description string `json:"description" datastore:"description,noindex"` ID string `json:"id" datastore:"id"` Name string `json:"name" datastore:"name"` Value string `json:"value" datastore:"value,noindex"` @@ -235,7 +235,7 @@ type Action struct { // Added environment for location to execute type Trigger struct { AppName string `json:"app_name" datastore:"app_name"` - Description string `json:"description" datastore:"description"` + Description string `json:"description" datastore:"description,noindex"` LongDescription string `json:"long_description" datastore:"long_description"` Status string `json:"status" datastore:"status"` AppVersion string `json:"app_version" datastore:"app_version"` @@ -249,6 +249,7 @@ type Trigger struct { Environment string `json:"environment" datastore:"environment"` TriggerType string `json:"trigger_type" datastore:"trigger_type"` Name string `json:"name" datastore:"name"` + Tags []string `json:"tags" datastore:"tags" yaml:"tags"` Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"` Position struct { X float64 `json:"x" datastore:"x"` @@ -294,20 +295,20 @@ type Workflow struct { ID string `json:"id" datastore:"id"` IsValid bool `json:"is_valid" datastore:"is_valid"` Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description"` + Description string `json:"description" datastore:"description,noindex"` Start string `json:"start" datastore:"start"` Owner string `json:"owner" datastore:"owner"` Sharing string `json:"sharing" datastore:"sharing"` Org []Org `json:"org,omitempty" datastore:"org"` ExecutingOrg Org `json:"execution_org,omitempty" datastore:"execution_org"` WorkflowVariables []struct { - Description string `json:"description" datastore:"description"` + Description string `json:"description" datastore:"description,noindex"` ID string `json:"id" datastore:"id"` Name string `json:"name" datastore:"name"` Value string `json:"value" datastore:"value"` } `json:"workflow_variables" datastore:"workflow_variables"` ExecutionVariables []struct { - Description string `json:"description" datastore:"description"` + Description string `json:"description" datastore:"description,noindex"` ID string `json:"id" datastore:"id"` Name string `json:"name" datastore:"name"` Value string `json:"value" datastore:"value,noindex"` @@ -330,7 +331,7 @@ type Authentication struct { } type AuthenticationParams struct { - Description string `json:"description" datastore:"description" yaml:"description"` + Description string `json:"description" datastore:"description,noindex" yaml:"description"` ID string `json:"id" datastore:"id" yaml:"id"` Name string `json:"name" datastore:"name" yaml:"name"` Example string `json:"example" datastore:"example" yaml:"example"` @@ -540,6 +541,7 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque return } + // FIXME: Add authentication? id := request.Header.Get("Org-Id") if len(id) == 0 { log.Printf("No Org-Id header set - confirm") @@ -626,7 +628,8 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque resp.Write([]byte("OK")) } -// FIXME: Authenticate this one (especially since we have a default: shuffle) +// FIXME: Authenticate this one? Can org ID be auth enough? +// (especially since we have a default: shuffle) func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -653,6 +656,8 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { if len(executionRequests.Data) == 0 { executionRequests.Data = []ExecutionRequest{} + } else { + log.Printf("[INFO] Executionrequests: %d", len(executionRequests.Data)) } newjson, err := json.Marshal(executionRequests) @@ -2142,7 +2147,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf //log.Printf("Execution data: %#v", execution) if len(execution.Start) == 36 { - log.Printf("SHOULD START ON NODE %s", execution.Start) + log.Printf("[INFO] Should start execution on node %s", execution.Start) workflowExecution.Start = execution.Start found := false @@ -2153,12 +2158,12 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } if !found { - log.Printf("ACTION %s WAS NOT FOUND!", workflow.Start) + log.Printf("[ERROR] ACTION %s WAS NOT FOUND!", workflow.Start) return WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", workflow.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", workflow.Start)) } } else if len(execution.Start) > 0 { - log.Printf("START ACTION %s IS WRONG ID LENGTH %d!", execution.Start, len(execution.Start)) + log.Printf("[ERROR] START ACTION %s IS WRONG ID LENGTH %d!", execution.Start, len(execution.Start)) return WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", execution.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", execution.Start)) } @@ -2288,10 +2293,10 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } if len(workflowExecution.ExecutionSource) == 0 { - log.Printf("No execution source specified. Setting to default") + log.Printf("[INFO] No execution source (trigger) specified. Setting to default") workflowExecution.ExecutionSource = "default" } else { - log.Printf("Execution source is %s for execution ID %s", workflowExecution.ExecutionSource, workflowExecution.ExecutionId) + log.Printf("[INFO] Execution source is %s for execution ID %s", workflowExecution.ExecutionSource, workflowExecution.ExecutionId) } workflowExecution.ExecutionVariables = workflow.ExecutionVariables @@ -2311,7 +2316,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if len(workflowExecution.Start) == 0 { workflowExecution.Start = workflowExecution.Workflow.Start } - log.Printf("STARTNODE: %s", workflowExecution.Start) + log.Printf("[INFO] New startnode: %s", workflowExecution.Start) childNodes := findChildNodes(workflowExecution, workflowExecution.Start) @@ -2419,6 +2424,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf environments := []string{} // Check if the actions are children of the startnode? + imageNames := []string{} for _, action := range workflowExecution.Workflow.Actions { if action.Environment != cloudname { found := false @@ -2429,6 +2435,11 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } } + // Check if the app exists? + newName := action.AppName + newName = strings.ReplaceAll(newName, " ", "-") + imageNames = append(imageNames, fmt.Sprintf("%s:%s_%s", baseDockerName, newName, action.AppVersion)) + if !found { environments = append(environments, action.Environment) } @@ -2437,6 +2448,12 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } } + err = imageCheckBuilder(imageNames) + if err != nil { + log.Printf("[ERROR] Failed building the required images from %#v: %s", imageNames, err) + return WorkflowExecution{}, "Failed building missing Docker images", err + } + err = setWorkflowExecution(ctx, workflowExecution) if err != nil { log.Printf("Error saving workflow execution for updates %s: %s", topic, err) @@ -2448,7 +2465,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if onpremExecution { // FIXME - tmp name based on future companyname-companyId for _, environment := range environments { - log.Printf("EXECUTION: %s should execute onprem with execution environment \"%s\"", workflowExecution.ExecutionId, environment) + log.Printf("[INFO] Execution: %s should execute onprem with execution environment \"%s\"", workflowExecution.ExecutionId, environment) executionRequest := ExecutionRequest{ ExecutionId: workflowExecution.ExecutionId, @@ -2467,12 +2484,13 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } //log.Printf("Execution request: %#v", executionRequest) - err = setWorkflowQueue(ctx, executionRequestWrapper, environment) if err != nil { log.Printf("Failed adding to db: %s", err) } } + } else { + log.Printf("[ERROR] Cloud not implemented yet") } err = increaseStatisticsField(ctx, "workflow_executions", workflow.ID, 1) @@ -2535,7 +2553,7 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("STARTING EXEC OF %s!", fileId) + log.Printf("[INFO] Starting execution of %s!", fileId) workflowExecution, executionResp, err := handleExecution(fileId, *workflow, request) if err == nil { @@ -3047,26 +3065,6 @@ func getSpecificWorkflow(resp http.ResponseWriter, request *http.Request) { resp.Write(body) } -//func setWorkflowExecutionFS(ctx context.Context, reference string, workflowExecution WorkflowExecution) error { -// if len(workflowExecution.ExecutionId) == 0 { -// log.Printf("Workflowexeciton executionId can't be empty.") -// return errors.New("ExecutionId can't be empty.") -// } -// -// firestoreClient, err := firestore.NewClient(ctx, shuffleTestProject, option.WithCredentialsFile(shuffleTestPath)) -// if err != nil { -// return err -// } -// -// executionRef := firestoreClient.Doc(reference) -// _, err = executionRef.Set(ctx, workflowExecution) -// if err != nil { -// return err -// } -// -// return nil -//} - func setWorkflowExecution(ctx context.Context, workflowExecution WorkflowExecution) error { if len(workflowExecution.ExecutionId) == 0 { log.Printf("Workflowexeciton executionId can't be empty.") @@ -3320,7 +3318,6 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { log.Printf("Deleting private app") var privateApps []WorkflowApp for _, item := range user.PrivateApps { - log.Println(item.ID, fileId) if item.ID == fileId { continue } @@ -3336,16 +3333,15 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true"}`))) return } - } else { + } - log.Printf("Deleting public app") - err = DeleteKey(ctx, "workflowapp", fileId) - if err != nil { - log.Printf("Failed deleting workflowapp") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed deleting workflow app"}`))) - return - } + log.Printf("Deleting public app") + err = DeleteKey(ctx, "workflowapp", fileId) + if err != nil { + log.Printf("Failed deleting workflowapp") + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed deleting workflow app"}`))) + return } err = increaseStatisticsField(ctx, "total_apps_deleted", fileId, 1) @@ -3408,9 +3404,6 @@ func getWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { return } - // log.Printf("%#v", parsedApi) - // log.Printf("API LEN: %d, ID: %s", len(parsedApi.Body), fileId) - //log.Printf("Parsed API: %#v", parsedApi) if len(parsedApi.ID) > 0 { parsedApi.Success = true @@ -3629,6 +3622,90 @@ func getAppAuthentication(resp http.ResponseWriter, request *http.Request) { }` */ } +func updateWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + user, userErr := handleApiAuthentication(resp, request) + if userErr != nil { + log.Printf("Api authentication failed in get all apps: %s", userErr) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + location := strings.Split(request.URL.String(), "/") + var fileId string + if location[1] == "api" { + if len(location) <= 4 { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + fileId = location[4] + } + + ctx := context.Background() + app, err := getApp(ctx, fileId) + if err != nil { + log.Printf("Error getting app: %s (update app)", app.Name) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if user.Id != app.Owner && user.Role != "admin" { + log.Printf("Wrong user (%s) for app %s in update app", user.Username, app.Name) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + body, err := ioutil.ReadAll(request.Body) + if err != nil { + log.Printf("Error with body read in update app: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + type updatefields struct { + Sharing bool `json:"sharing"` + SharingConfig string `json:"sharing_config"` + } + + var tmpfields updatefields + err = json.Unmarshal(body, &tmpfields) + if err != nil { + log.Printf("Error with unmarshal body in update app: %s\n%s", err, string(body)) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if tmpfields.Sharing != app.Sharing { + app.Sharing = tmpfields.Sharing + } + + if tmpfields.SharingConfig != app.SharingConfig { + app.SharingConfig = tmpfields.SharingConfig + } + + err = setWorkflowAppDatastore(ctx, *app, app.ID) + if err != nil { + log.Printf("Failed patching workflowapp: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + log.Printf("Changed workflow app %s", app.ID) + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) +} func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) @@ -3691,12 +3768,12 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { newapps := []WorkflowApp{} baseApps := []WorkflowApp{} - if len(user.PrivateApps) > 0 { - newapps = append(newapps, user.PrivateApps...) - } - for _, workflowapp := range workflowapps { - if !workflowapp.Sharing { + if !workflowapp.Activated && workflowapp.Generated { + continue + } + + if workflowapp.Owner != user.Id && user.Role != "admin" && !workflowapp.Sharing { continue } @@ -3716,6 +3793,22 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { baseApps = append(baseApps, workflowapp) } + if len(user.PrivateApps) > 0 { + found := false + for _, item := range user.PrivateApps { + for _, app := range newapps { + if item.ID == app.ID { + found = true + break + } + } + + if !found { + newapps = append(newapps, item) + } + } + } + // Double unmarshal because of user apps newbody, err := json.Marshal(newapps) //newbody, err := json.Marshal(workflowapps) @@ -4071,7 +4164,7 @@ func deployWebhookFunction(ctx context.Context, name, localization, applocation return nil } -func loadGithubWorkflows(url, username, password, userId string) error { +func loadGithubWorkflows(url, username, password, userId, branch string) error { fs := memfs.New() if strings.Contains(url, "github") || strings.Contains(url, "gitlab") || strings.Contains(url, "bitbucket") { @@ -4088,6 +4181,10 @@ func loadGithubWorkflows(url, username, password, userId string) error { } } + if len(branch) > 0 { + cloneOptions.ReferenceName = plumbing.ReferenceName(branch) + } + storer := memory.NewStorage() r, err := git.Clone(storer, fs, cloneOptions) if err != nil { @@ -4163,6 +4260,7 @@ func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) { URL string `json:"url"` Field1 string `json:"field_1"` Field2 string `json:"field_2"` + Field3 string `json:"field_3"` } //log.Printf("Body: %s", string(body)) @@ -4175,7 +4273,8 @@ func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) { return } - err = loadGithubWorkflows(tmpBody.URL, tmpBody.Field1, tmpBody.Field2, user.Id) + // Field3 = branch + err = loadGithubWorkflows(tmpBody.URL, tmpBody.Field1, tmpBody.Field2, user.Id, tmpBody.Field3) if err != nil { log.Printf("Failed to update workflows: %s", err) resp.WriteHeader(401) @@ -4211,15 +4310,15 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { return } - location := os.Getenv("APP_HOTLOAD_FOLDER") + location := os.Getenv("SHUFFLE_APP_HOTLOAD_FOLDER") if len(location) == 0 { resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "APP_HOTLOAD_FOLDER not specified in .env"}`))) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "SHUFFLE_APP_HOTLOAD_FOLDER not specified in .env"}`))) return } log.Printf("Hotloading from %s", location) - err = handleAppHotload(location) + err = handleAppHotload(location, true) if err != nil { resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed loading apps: %s"}`))) @@ -4372,6 +4471,7 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, if strings.Contains(filename, "yaml") || strings.Contains(filename, "yml") { //log.Printf("File: %s", filename) //log.Printf("Found file: %s", filename) + log.Printf("OpenAPI app: %s", filename) tmpExtra := fmt.Sprintf("%s%s/", extra, file.Name()) fileReader, err := fs.Open(tmpExtra) @@ -4381,14 +4481,14 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, readFile, err := ioutil.ReadAll(fileReader) if err != nil { - log.Printf("Filereader error yaml: %s", err) + log.Printf("Filereader error yaml for %s: %s", filename, err) continue } // 1. This parses OpenAPI v2 to v3 etc, for use. parsedOpenApi, err := handleSwaggerValidation(readFile) if err != nil { - log.Printf("Validation error: %s", err) + log.Printf("Validation error for %s: %s", filename, err) continue } diff --git a/docker-compose.yml b/docker-compose.yml index 8faba6f3..dce4ba78 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: frontend: #build: ./frontend - image: frikky/shuffle:frontend + image: ghcr.io/frikky/shuffle-frontend:0.7.1 container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -17,22 +17,22 @@ services: - backend backend: #build: ./backend - image: frikky/shuffle:backend + image: ghcr.io/frikky/shuffle-backend:0.7.1 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: - ports: + ports: - "${BACKEND_PORT}:5001" networks: - shuffle volumes: - /var/run/docker.sock:/var/run/docker.sock - - ${APP_HOTLOAD_LOCATION}:/shuffle-apps - environment: + - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps + environment: - DATASTORE_EMULATOR_HOST=shuffle-database:8000 - - APP_HOTLOAD_FOLDER=/shuffle-apps + - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps - ORG_ID=${ORG_ID} - - APP_DOWNLOAD_LOCATION=${APP_DOWNLOAD_LOCATION} + - SHUFFLE_APP_DOWNLOAD_LOCATION=${SHUFFLE_APP_DOWNLOAD_LOCATION} - SHUFFLE_DEFAULT_USERNAME=${SHUFFLE_DEFAULT_USERNAME} - SHUFFLE_DEFAULT_PASSWORD=${SHUFFLE_DEFAULT_PASSWORD} - SHUFFLE_DEFAULT_APIKEY=${SHUFFLE_DEFAULT_APIKEY} @@ -43,14 +43,16 @@ services: - database orborus: #build: ./functions/onprem/orborus - image: frikky/shuffle:orborus + image: ghcr.io/frikky/orborus:0.6.2 container_name: shuffle-orborus hostname: shuffle-orborus networks: - shuffle - volumes: - - /var/run/docker.sock:/var/run/docker.sock + volumes: + - /var/run/docker.sock:/var/run/docker.sock environment: + - SHUFFLE_APP_SDK_VERSION=0.6.0 + - SHUFFLE_WORKER_VERSION=0.6.0 - ORG_ID=${ORG_ID} - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT} @@ -61,7 +63,7 @@ services: restart: unless-stopped database: #build: ./backend/database - image: frikky/shuffle:database + image: ghcr.io/frikky/shuffle-database:1.0.0 container_name: shuffle-database hostname: shuffle-database ports: diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index f7a408d1..ea4224f0 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -129,13 +129,14 @@ const App = (message, props) => { } /> } /> } /> + } /> } /> } /> } /> } /> } /> } /> - } /> + } /> } /> } /> } /> diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index ae1756c3..640c46e8 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -49,6 +49,71 @@ const Admin = (props) => { const [selectedAuthenticationModalOpen, setSelectedAuthenticationModalOpen] = React.useState(false) const [showArchived, setShowArchived] = React.useState(false) + const getApps = () => { + fetch(globalUrl+"/api/v1/workflows/apps", { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for apps :O!") + } + + return response.json() + }) + .then((responseJson) => { + console.log("apps: ", responseJson) + //setApps(responseJson) + //setFilteredApps(responseJson) + //if (responseJson.length > 0) { + // setSelectedApp(responseJson[0]) + // if (responseJson[0].actions !== null && responseJson[0].actions.length > 0) { + // setSelectedAction(responseJson[0].actions[0]) + // } else { + // setSelectedAction({}) + // } + //} + }) + .catch(error => { + alert.error(error.toString()) + }); + } + + const categories = [ + { + "name": "Ticketing", + "apps": [ + "TheHive", + "Service-Now", + "SecureWorks", + ], + "categories": ["tickets", "ticket", "ticketing"] + }, + ] + /* + "SIEM", + "Active Directory", + "Firewalls", + "Proxies web", + "SIEM", + "SOAR", + "Mail", + "EDR", + "AV", + "MDM/MAM", + "DNS", + "Ticketing platform", + "TIP", + "Communication", + "DDOS protection", + "VMS", + ] + */ + const alert = useAlert() const deleteAuthentication = (data) => { @@ -905,6 +970,82 @@ const Admin = (props) => { : null + const appCategoryView = curTab === 6 ? +
+
+

Categories

+ + Categories are the categories supported by Shuffle, which are mapped to apps and workflows + +
+ + + + + + + + + + {categories.map(data => { + if (data.apps.length === 0) { + return null + } + + return ( + + + + + + + + + + ) + })} + +
+ : null + const authenticationView = curTab === 1 ?
@@ -1182,6 +1323,10 @@ const Admin = (props) => { getSchedules() } + if (newValue === 6) { + console.log("Should get apps for categories.") + } + setModalUser({}) setCurTab(newValue) } @@ -1202,10 +1347,12 @@ const Admin = (props) => { Schedules /> {window.location.protocol == "http:" && window.location.port === "3000" ? Hybrid/> : null} {window.location.protocol == "http:" && window.location.port === "3000" ? Organizations/> : null} + {window.location.protocol === "http:" && window.location.port === "3000" ? Categories/> : null}
{authenticationView} + {appCategoryView} {usersView} {environmentView} {schedulesView} diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 018d6004..b940d7b5 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -293,6 +293,7 @@ const AngularWorkflow = (props) => { .then((response) => { if (response.status !== 200) { console.log("Status not 200 for stream results :O!") + stop() } return response.json() @@ -302,6 +303,7 @@ const AngularWorkflow = (props) => { }) .catch(error => { alert.error(error.toString()) + stop() }); } @@ -639,7 +641,7 @@ const AngularWorkflow = (props) => { } if (executionText.length > 0) { - alert.success("Starting execution with argument "+executionText) + alert.success("Starting execution with an execution argument") } else { alert.success("Starting execution") } @@ -5037,6 +5039,7 @@ const AngularWorkflow = (props) => { @@ -5167,8 +5170,10 @@ const AngularWorkflow = (props) => { return (
-

Execution Argument:

- {executionData.execution_argument} +

Execution Argument

+
+ {executionData.execution_argument} +
) } @@ -5272,14 +5277,7 @@ const AngularWorkflow = (props) => { -

Executing Workflow

- Show failed / skipped actions
} - control={ - {setShowSkippedActions(!showSkippedActions)}} /> - } - /> +

Executing Workflow

{executionData.status !== undefined && executionData.status.length > 0 ?
Status: {executionData.status} @@ -5292,7 +5290,7 @@ const AngularWorkflow = (props) => {
: null } - {executionData.completed_at !== undefined ? + {executionData.completed_at !== undefined && executionData.completed_at !== null && executionData.completed_at > 0 ?
Finished: {new Date(executionData.completed_at*1000).toISOString()}
@@ -5308,7 +5306,19 @@ const AngularWorkflow = (props) => { parsedExecutionArgument() : null } + {executionData.results !== undefined && executionData.results !== null && executionData.results.length > 1 && executionData.results.find(result => result.status === "SKIPPED" || result.status === "FAILURE") ? + Show failed / skipped actions
} + control={ + {setShowSkippedActions(!showSkippedActions)}} /> + } + /> + : + null + }
+ Actions
{executionData.status !== undefined && executionData.status !== "ABORTED" && executionData.status !== "FINISHED" && executionData.status !== "FAILURE" ? : null} diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 9768ada1..2303398f 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -334,6 +334,7 @@ const AppCreator = (props) => { } }) .catch(error => { + console.log("Error: ", error.toString()) alert.error(error.toString()) }); } @@ -405,11 +406,33 @@ const AppCreator = (props) => { securitySchemes = data.components.securitySchemes } + const allowedfunctions = [ + "GET", + "CONNECT", + "HEAD", + "DELETE", + "POST", + "PATCH", + "PUT", + ] + // FIXME - headers? var newActions = [] var wordlist = {} for (let [path, pathvalue] of Object.entries(data.paths)) { for (let [method, methodvalue] of Object.entries(pathvalue)) { + if (methodvalue === null) { + alert.info("Skipped method "+method) + continue + } + + if (!allowedfunctions.includes(method.toUpperCase())) { + console.log(method, path) + continue + } + + console.log("Method: ", method) + console.log("Methodval: ", methodvalue) var newaction = { "name": methodvalue.summary, "description": methodvalue.description, @@ -446,8 +469,11 @@ const AppCreator = (props) => { // https://swagger.io/docs/specification/describing-parameters/ // Need to split the data. } else if (parameter.in === "body") { - console.log("BODY: ", parameter) - newaction.body = parameter.example + // FIXME: Add tracking for components + // E.G: https://raw.githubusercontent.com/owentl/Shuffle/master/gosecure.yaml + if (parameter.example !== undefined) { + newaction.body = parameter.example + } } else if (parameter.in === "header") { newaction.headers += `${parameter.name}=${parameter.example}\n` } @@ -934,7 +960,7 @@ const AppCreator = (props) => { margin="normal" variant="outlined" value={parameterName} - helperText={
Can't be empty. Can't contain any of the following characters: !#$%&'^+-._~|]+$
} + helperText={Can't be empty. Can't contain any of the following characters: !#$%&'^+-._~|]+$} onChange={e => setParameterName(e.target.value)} InputProps={{ classes: { @@ -1005,7 +1031,7 @@ const AppCreator = (props) => { />
-
{deletePathQuery(index)}}> +
{deletePathQuery(index)}}> Delete
@@ -1264,7 +1290,7 @@ const AppCreator = (props) => {
New action
 - Learn more about actions + Learn more about actions
Name {

Actions

Actions are the tasks performed by an app. Read more about actions and apps - here. + here.
{loopActions} + : null var editButton = selectedApp.activated && selectedApp.private_id !== undefined && selectedApp.private_id.length > 0 && selectedApp.generated ? - : null + + + + : null var activateButton = selectedApp.generated && !selectedApp.activated ? - - : null +
+ + + + + + +
+ : null var deleteButton = ((selectedApp.private_id !== undefined && selectedApp.private_id.length > 0 && selectedApp.generated) || (selectedApp.downloaded != undefined && selectedApp.downloaded == true)) && activateButton === null ? + : null + + + : null var imageline = selectedApp.large_image === undefined || selectedApp.large_image.length === 0 ? {selectedApp.title} @@ -450,6 +515,7 @@ const Apps = (props) => { ) })} + {/* { displayDataTypes={true} name={"Example return value"} /> + */}
) } return ( -
+
Example return
{selectedAction.returns.example}
) } - //fetch(globalUrl+"/api/v1/get_openapi/"+urlParams.get("id"), { + const userRoles = [ + "you", + "everyone", + ] + + //fetch(globalUrl+"/api/v1/get_openapi/"+urlParams.get("id"), var baseInfo = newAppname.length > 0 ?
@@ -482,15 +554,25 @@ const Apps = (props) => {
{activateButton} - {downloadButton} - {editButton} - {deleteButton} + {props.userdata.role === "admin" || props.userdata.id === selectedApp.owner ? +
+ {downloadButton} + {editButton} + {deleteButton} +
+ : null} {selectedApp.tags !== undefined && selectedApp.tags !== null ?
- {selectedApp.tags.map(tag => { + {selectedApp.tags.map((tag, index) => { + if (index >= 3) { + return null + } + return ( @@ -498,78 +580,110 @@ const Apps = (props) => { })}
: null} - - {selectedApp.link.length > 0 ?

URL: {selectedApp.link}

: null} -

ID: {selectedApp.id}

- {selectedApp.privateId !== undefined && selectedApp.privateId.length > 0 ?

PrivateID: {selectedApp.privateId}

: null} - -
- Actions - {selectedApp.actions !== null && selectedApp.actions.length > 0 ? + {props.userdata.id === selectedApp.owner ? +
+ {/*

ID: {selectedApp.id}

*/} + Sharing: +
+ : null} + {/*

Owner: {selectedApp.owner}

*/} + {selectedApp.privateId !== undefined && selectedApp.privateId.length > 0 ?

PrivateID: {selectedApp.privateId}

: null} + +
+ {selectedApp.link.length > 0 ?

URL: {selectedApp.link}

: null} +
+ Actions + {selectedApp.actions !== null && selectedApp.actions.length > 0 ? + + : +
+ There are no actions defined for this app. +
+ } +
+ + {selectedAction.parameters !== undefined && selectedAction.parameters !== null ? +
+ Arguments + {selectedAction.parameters.map(data => { + var itemColor = "#f85a3e" + if (!data.required) { + itemColor = "#ffeb3b" + } + + const circleSize = 10 return ( - {newActionname} +
+ {data.name} ) })} - - : -
- There are no actions defined for this app.
- } -
- - {selectedAction.parameters !== undefined && selectedAction.parameters !== null ? -
- Arguments - {selectedAction.parameters.map(data => { - var itemColor = "#f85a3e" - if (!data.required) { - itemColor = "#ffeb3b" - } - - const circleSize = 10 - return ( - -
- {data.name} - - - ) - })} -
- : null} - {selectedAction.description !== undefined && selectedAction.description !== null && selectedAction.description.length > 0 ? -
- Action Description
- {selectedAction.description} -
: null} - + {selectedAction.description !== undefined && selectedAction.description !== null && selectedAction.description.length > 0 ? +
+ Action Description
+ {selectedAction.description} +
+ : null} + +
: null @@ -582,8 +696,9 @@ const Apps = (props) => { How it works  - Security API's  - OpenAPI directory +  - OpenAPI Validator
- Apps interact with eachother in workflows. They are created with the app creator, using OpenAPI specification or manually in python. Use the links above to find potential apps you're looking for using OpenAPI or make one from scratch. There's 1000+ available. + Apps interact with eachother in workflows. They are created with the app creator, using OpenAPI specification or manually in python. The links above are references to OpenAPI tools and other app repositories. There's ten thousands of them.
@@ -884,6 +999,36 @@ const Apps = (props) => { }); } + const updateAppField = (app_id, fieldname, fieldvalue) => { + const data = {} + data[fieldname] = fieldvalue + + fetch(globalUrl+"/api/v1/apps/"+app_id, { + method: 'PATCH', + headers: { + 'Accept': 'application/json', + }, + body: JSON.stringify(data), + credentials: "include", + }) + .then((response) => { + //setAppSearchLoading(false) + return response.json() + }) + .then((responseJson) => { + //console.log(responseJson) + //alert.info(responseJson) + if (responseJson.success) { + alert.info("Success") + } else { + alert.error("Error updating app") + } + }) + .catch(error => { + alert.error(error.toString()) + }); + } + const runAppSearch = (searchterm) => { const data = {"search": searchterm} @@ -916,7 +1061,7 @@ const Apps = (props) => { setValidation(true) fetch(globalUrl+"/api/v1/get_openapi_uri", { - method: 'POST', + method: 'POST', headers: { 'Accept': 'application/json', }, @@ -924,19 +1069,34 @@ const Apps = (props) => { credentials: "include", }) .then((response) => { + setValidation(false) + if (response.status !== 200) { + return response.json() + } + return response.text() }) - .then((responseText) => { - validateOpenApi(responseText) - setValidation(false) - }) + .then((responseJson) => { + if (typeof(responseJson) !== "string" && !responseJson.success) { + console.log(responseJson.reason) + if (responseJson.reason !== undefined) { + setOpenApiError(responseJson.reason) + } else { + setOpenApiError("Undefined issue with OpenAPI validation") + } + return + } + + validateOpenApi(responseJson) + }) .catch(error => { alert.error(error.toString()) + setOpenApiError(error.toString()) }); } const escapeApiData = (apidata) => { - console.log(apidata) + //console.log(apidata) try { return JSON.stringify(JSON.parse(apidata)) } catch(error) { @@ -948,7 +1108,7 @@ const Apps = (props) => { return JSON.stringify(YAML.parse(apidata)) } catch(error) { console.log("YAML DECODE ERROR - TRY SOMETHING ELSE?: "+error) - setOpenApiError(error) + setOpenApiError(error.toString()) } return "" @@ -962,6 +1122,7 @@ const Apps = (props) => { return } + setValidation(true) fetch(globalUrl+"/api/v1/validate_openapi", { method: 'POST', headers: { @@ -971,10 +1132,10 @@ const Apps = (props) => { credentials: "include", }) .then((response) => { + setValidation(false) return response.json() }) - .then((responseJson) => { - setValidation(false) + .then((responseJson) => { if (responseJson.success) { setAppValidation(responseJson.id) } else { @@ -985,7 +1146,9 @@ const Apps = (props) => { } }) .catch(error => { + setValidation(false) alert.error(error.toString()) + setOpenApiError(error.toString()) }); } @@ -1131,7 +1294,9 @@ const Apps = (props) => { const modalView = openApiModal ? {setOpenApiModal(false)}} + onClose={() => { + setOpenApiModal(false) + }} PaperProps={{ style: { backgroundColor: surfaceColor, @@ -1155,21 +1320,26 @@ const Apps = (props) => { height: "50px", fontSize: "1em", }, - endAdornment: }} - onChange={e => setOpenApi(e.target.value)} - helperText={
Must point to a version 2 or 3 specification.
} + onChange={e => { + setOpenApi(e.target.value) + }} + helperText={Must point to a version 2 or 3 OpenAPI specification.} placeholder="OpenAPI URI" fullWidth /> + {/*
Example:
https://raw.githubusercontent.com/OAI/OpenAPI-Specification/master/examples/v2.0/json/uber.json -

or paste the yaml/JSON directly below

+ */} + Or paste the YAML or JSON specification { color: "white", fontSize: "1em", }, - endAdornment: + }}>Validate OpenAPI }} onChange={e => setOpenApiData(e.target.value)} - helperText={
Must point to a version 2 or 3 specification.
} + helperText={Must point to a version 2 or 3 specification.} placeholder="OpenAPI text" fullWidth /> @@ -1195,13 +1365,19 @@ const Apps = (props) => { {circularLoader} - - + diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 10ecae86..8542790d 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -14,10 +14,12 @@ import MenuItem from '@material-ui/core/MenuItem'; import FormControlLabel from '@material-ui/core/FormControlLabel'; import Chip from '@material-ui/core/Chip'; import Switch from '@material-ui/core/Switch'; +import Typography from '@material-ui/core/Typography'; import CircularProgress from '@material-ui/core/CircularProgress'; import CachedIcon from '@material-ui/icons/Cached'; import GetAppIcon from '@material-ui/icons/GetApp'; +import AppsIcon from '@material-ui/icons/Apps'; import EditIcon from '@material-ui/icons/Edit'; import MoreVertIcon from '@material-ui/icons/MoreVert'; import PlayArrowIcon from '@material-ui/icons/PlayArrow'; @@ -200,7 +202,7 @@ const Workflows = (props) => { marginTop: "10px", overflow: "scroll", height: "90%", - overflowX: "auto", + overflowX: "hidden", overflowY: "auto", } @@ -211,6 +213,8 @@ const Workflows = (props) => { marginTop: "5px", color: "white", backgroundColor: surfaceColor, + borderRadius: 5, + padding: 10, cursor: "pointer", display: "flex", } @@ -240,6 +244,8 @@ const Workflows = (props) => { setWorkflowExecutions(responseJson) } else { alert.info("Couldn't find executions for the workflow") + setSelectedExecution({}) + setWorkflowExecutions([]) } } }) @@ -412,12 +418,29 @@ const Workflows = (props) => { setAnchorEl(event.currentTarget); } + const actions = data.actions !== null ? data.actions.length : 0 + var schedules = 0 + var webhooks = 0 + var webhookImg = "" + var scheduleImg = "" + if (data.triggers !== undefined && data.triggers !== null && data.triggers.length > 0) { + for (var key in data.triggers) { + if (data.triggers[key].app_name === "Webhook") { + webhooks += 1 + webhookImg = data.triggers[key].large_image + } else if (data.triggers[key].app_name === "Schedule") { + schedules += 1 + scheduleImg = data.triggers[key].large_image + } + } + } + + const imgSize = 25 return ( { - }}> -
-
- + }}> +
+
{ @@ -426,9 +449,11 @@ const Workflows = (props) => { getWorkflowExecution(data.id) } }}> -

{data.name}

+ + {data.name} +
-
+
{ getWorkflowExecution(data.id) } }}> - + + + + {data.tags !== undefined ? - data.tags.map(tag => { + data.tags.map((tag, index) => { + if (index >= 3) { + return null + } + return ( ) @@ -509,6 +542,22 @@ const Workflows = (props) => {
+ + + + + + {webhooks > 0 ? + + {data.title} + + : null} + {schedules > 0 ? + + {data.title} + + : null} + ) } @@ -914,7 +963,6 @@ const Workflows = (props) => { setLoadWorkflowsModalOpen(false) } - console.log("WOrkflowtags: ", newWorkflowTags) const modalView = modalOpen ? 0 { - log.Printf("Found shuffle network \"%s\" for container %s", shuffleNetwork, containerIdentifier) - } else { - log.Printf("Running Shuffle without a docker network") } } @@ -121,40 +116,24 @@ func deployWorker(image string, identifier string, env []string) { }, } - // Look for Shuffle network and set it - networkConfig := &network.NetworkingConfig{} - if len(shuffleNetwork) > 0 { - log.Printf("Starting worker with network %s", shuffleNetwork) - networkConfig = &network.NetworkingConfig{ - EndpointsConfig: map[string]*network.EndpointSettings{ - shuffleNetwork: { - NetworkID: shuffleNetwork, - }, - }, - } - - env = append(env, fmt.Sprintf("DOCKER_NETWORK=%s", shuffleNetwork)) + // form container id and use it as network source if it's not empty + if containerId != "" { + log.Printf("[INFO] Found container ID %s", containerId) + hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) } else { - log.Printf("Starting worker WITHOUT any specified network: %s", shuffleNetwork) + log.Printf("[INFO] Empty self container id, continue without NetworkMode") } - // ROFL: https://docker-py.readthedocs.io/en/1.4.0/volumes/ config := &container.Config{ Image: image, Env: env, } - //test := &network.EndpointSettings{ - // Gateway: "helo", - //} - //NetworkID - //if connect.EndpointConfig.NetworkID != "NetworkID" { - cont, err := dockercli.ContainerCreate( context.Background(), config, hostConfig, - networkConfig, + nil, nil, identifier, ) @@ -166,7 +145,7 @@ func deployWorker(image string, identifier string, env []string) { err = dockercli.ContainerStart(context.Background(), cont.ID, types.ContainerStartOptions{}) if err != nil { - log.Printf("Failed to start container in environment %s: %s", environment, err) + log.Printf("[ERROR] Failed to start container in environment %s: %s", environment, err) return //stats, err := cli.ContainerInspect(context.Background(), containerName) @@ -191,7 +170,7 @@ func deployWorker(image string, identifier string, env []string) { // } //} } else { - log.Printf("Container %s was created under environment %s", cont.ID, environment) + log.Printf("[INFO] Container %s was created under environment %s", cont.ID, environment) } return @@ -205,7 +184,7 @@ func stopWorker(containername string) error { // }) if err := dockercli.ContainerStop(ctx, containername, nil); err != nil { - log.Printf("Unable to stop container %s - running removal anyway, just in case: %s", containername, err) + log.Printf("[ERROR] Unable to stop container %s - running removal anyway, just in case: %s", containername, err) } removeOptions := types.ContainerRemoveOptions{ @@ -214,7 +193,7 @@ func stopWorker(containername string) error { } if err := dockercli.ContainerRemove(ctx, containername, removeOptions); err != nil { - log.Printf("Unable to remove container: %s", err) + log.Printf("[ERROR] Unable to remove container: %s", err) } return nil @@ -223,32 +202,43 @@ func stopWorker(containername string) error { func initializeImages() { ctx := context.Background() + if appSdkVersion == "" { + appSdkVersion = "0.6.0" + log.Printf("[WARNING] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %s", appSdkVersion) + } + if workerVersion == "" { + workerVersion = "0.6.0" + log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %s", workerVersion) + } + // check whether theyre the same first - //version := "0.1.0" - // fmt.Sprintf("docker.pkg.github.com/frikky/shuffle/orborus:%s", version), - // fmt.Sprintf("docker.pkg.github.com/frikky/shuffle/worker:%s", version), images := []string{ - fmt.Sprintf("docker.io/%s:app_sdk", baseimagename), - fmt.Sprintf("docker.io/%s:worker", baseimagename), + // fmt.Sprintf("docker.io/%s:app_sdk", baseimagename), + // fmt.Sprintf("docker.io/%s:worker", baseimagename), + + fmt.Sprintf("%s/worker:%s", baseimagename, workerVersion), + fmt.Sprintf("%s/app_sdk:%s", baseimagename, appSdkVersion), + fmt.Sprintf("frikky/shuffle:app_sdk"), } pullOptions := types.ImagePullOptions{} for _, image := range images { + log.Printf("[INFO] Pulling image %s", image) reader, err := dockercli.ImagePull(ctx, image, pullOptions) if err != nil { - log.Printf("Failed getting %s: %s", image, err) + log.Printf("[ERROR] Failed getting image %s: %s", image, err) continue } io.Copy(os.Stdout, reader) - log.Printf("Successfully downloaded and built %s", image) + log.Printf("[INFO] Successfully downloaded and built %s", image) } } // Initial loop etc func main() { go zombiecheck() - log.Println("Setting up execution environment") + log.Println("[INFO] Setting up execution environment") //FIXME if baseUrl == "" { @@ -257,30 +247,30 @@ func main() { } if orgId == "" { - log.Printf("Org not defined. Set variable ORG_ID based on your org") + log.Printf("[ERROR] Org not defined. Set variable ORG_ID based on your org") os.Exit(3) } - log.Printf("Running towards %s with Org %s", baseUrl, orgId) + log.Printf("[INFO] Running towards %s with Org %s", baseUrl, orgId) httpProxy := os.Getenv("HTTP_PROXY") httpsProxy := os.Getenv("HTTPS_PROXY") if environment == "" { environment = "onprem" - log.Printf("Defaulting to environment name %s. Set environment variable ENVIRONMENT_NAME to change. This should be the same as in the frontend action.", environment) + log.Printf("[INFO] Defaulting to environment name %s. Set environment variable ENVIRONMENT_NAME to change. This should be the same as in the frontend action.", environment) } // FIXME - during init, BUILD and/or LOAD worker and app_sdk // Build/load app_sdk so it can be loaded as 127.0.0.1:5000/walkoff_app_sdk - log.Printf("--- Setting up Docker environment. Downloading worker and App SDK! ---") - initializeImages() + log.Printf("[INFO] Setting up Docker environment. Downloading worker and App SDK!") + go initializeImages() //workerName := "worker" //workerVersion := "0.1.0" //workerImage := fmt.Sprintf("docker.pkg.github.com/frikky/shuffle/%s:%s", workerName, workerVersion) - workerImage := fmt.Sprintf("%s:worker", baseimagename) + workerImage := fmt.Sprintf("%s/worker:%s", baseimagename, workerVersion) - log.Printf("--- Finished configuring docker environment ---\n") + log.Printf("[INFO] Finished configuring docker environment") // FIXME - time limit client := &http.Client{ @@ -293,10 +283,10 @@ func main() { client = &http.Client{} } else { if len(httpProxy) > 0 { - log.Printf("Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy) + log.Printf("[INFO] Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy) } if len(httpsProxy) > 0 { - log.Printf("Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy) + log.Printf("[INFO] Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy) } } @@ -308,21 +298,21 @@ func main() { ) if err != nil { - log.Printf("Failed making request builder: %s", err) + log.Printf("[ERROR] Failed making request builder: %s", err) os.Exit(3) } zombiecounter := 0 req.Header.Add("Content-Type", "application/json") req.Header.Add("Org-Id", orgId) - log.Printf("Getting data from %s", fullUrl) + log.Printf("[INFO] Waiting for executions at %s", fullUrl) hasStarted := false for { //log.Printf("Prerequest") newresp, err := client.Do(req) //log.Printf("Postrequest") if err != nil { - log.Printf("Failed making request: %s", err) + log.Printf("[WARNING] Failed making request: %s", err) zombiecounter += 1 if zombiecounter*sleepTime > workerTimeout { go zombiecheck() @@ -335,7 +325,7 @@ func main() { // FIXME - add check for StatusCode if newresp.StatusCode != 200 { if hasStarted { - log.Printf("Bad statuscode: %d", newresp.StatusCode) + log.Printf("[WARNING] Bad statuscode: %d", newresp.StatusCode) } } else { hasStarted = true @@ -343,7 +333,7 @@ func main() { body, err := ioutil.ReadAll(newresp.Body) if err != nil { - log.Printf("Failed reading body: %s", err) + log.Printf("[ERROR] Failed reading body: %s", err) zombiecounter += 1 if zombiecounter*sleepTime > workerTimeout { go zombiecheck() @@ -356,7 +346,7 @@ func main() { var executionRequests ExecutionRequestWrapper err = json.Unmarshal(body, &executionRequests) if err != nil { - log.Printf("Failed executionrequest in queue unmarshaling: %s", err) + log.Printf("[WARNING] Failed executionrequest in queue unmarshaling: %s", err) sleepTime = 10 zombiecounter += 1 if zombiecounter*sleepTime > workerTimeout { @@ -368,7 +358,7 @@ func main() { } if hasStarted && len(executionRequests.Data) > 0 { - log.Printf("Body: %s", string(body)) + log.Printf("[INFO] Body: %s", string(body)) // Type string `json:"type"` } @@ -386,16 +376,16 @@ func main() { var toBeRemoved ExecutionRequestWrapper for _, execution := range executionRequests.Data { if len(execution.ExecutionArgument) > 0 { - log.Printf("Argument: %#v", execution.ExecutionArgument) + log.Printf("[INFO] Argument: %#v", execution.ExecutionArgument) } if execution.Type == "schedule" { - log.Printf("SOMETHING ELSE :O: %s", execution.Type) + log.Printf("[INFO] SOMETHING ELSE :O: %s", execution.Type) continue } if execution.Status == "ABORT" || execution.Status == "FAILED" { - log.Printf("Executionstatus issue: ", execution.Status) + log.Printf("[INFO] Executionstatus issue: ", execution.Status) } // Now, how do I execute this one? // FIXME - if error, check the status of the running one. If it's bad, send data back. @@ -418,7 +408,7 @@ func main() { go deployWorker(workerImage, containerName, env) - log.Printf("%s is deployed and to be removed from queue.", execution.ExecutionId) + log.Printf("[INFO] %s is deployed and to be removed from queue.", execution.ExecutionId) zombiecounter += 1 toBeRemoved.Data = append(toBeRemoved.Data, execution) } @@ -429,7 +419,7 @@ func main() { data, err := json.Marshal(toBeRemoved) if err != nil { - log.Printf("Failed removal marshalling: %s", err) + log.Printf("[WARNING] Failed removal marshalling: %s", err) time.Sleep(time.Duration(sleepTime) * time.Second) continue } @@ -441,7 +431,7 @@ func main() { ) if err != nil { - log.Printf("Failed building confirm request: %s", err) + log.Printf("[ERROR] Failed building confirm request: %s", err) time.Sleep(time.Duration(sleepTime) * time.Second) continue } @@ -451,14 +441,14 @@ func main() { resultResp, err := client.Do(result) if err != nil { - log.Printf("Failed making confirm request: %s", err) + log.Printf("[ERROR] Failed making confirm request: %s", err) time.Sleep(time.Duration(sleepTime) * time.Second) continue } body, err := ioutil.ReadAll(resultResp.Body) if err != nil { - log.Printf("Failed reading confirm body: %s", err) + log.Printf("[ERROR] Failed reading confirm body: %s", err) time.Sleep(time.Duration(sleepTime) * time.Second) continue } @@ -472,7 +462,7 @@ func main() { if len(toBeRemoved.Data) == len(executionRequests.Data) { //log.Println("Should remove ALL!") } else { - log.Printf("NOT IMPLEMENTED: Should remove %d workflows from backend because they're executed!", len(toBeRemoved.Data)) + log.Printf("[INFO] NOT IMPLEMENTED: Should remove %d workflows from backend because they're executed!", len(toBeRemoved.Data)) } } @@ -483,7 +473,7 @@ func main() { // FIXME - add this to remove exited workers // Should it check what happened to the execution? idk func zombiecheck() error { - log.Println("Looking for old containers") + log.Println("[INFO] Looking for old containers") ctx := context.Background() containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{ @@ -491,7 +481,7 @@ func zombiecheck() error { }) if err != nil { - log.Printf("Failed creating Containerlist: %s", err) + log.Printf("[ERROR] Failed creating Containerlist: %s", err) return err } @@ -524,7 +514,7 @@ func zombiecheck() error { continue } - log.Printf("NAME: %s", name) + log.Printf("[INFO] NAME: %s", name) // Need to check time here too because a container can be removed the same instant as its created currenttime := time.Now().Unix() @@ -544,7 +534,7 @@ func zombiecheck() error { // FIXME - add killing of apps with same execution ID too for _, containername := range stopContainers { - log.Printf("Stopping and removing container %s", containerNames[containername]) + log.Printf("[INFO] Stopping and removing container %s", containerNames[containername]) go dockercli.ContainerStop(ctx, containername, nil) removeContainers = append(removeContainers, containername) } diff --git a/functions/onprem/orborus/run.sh b/functions/onprem/orborus/run.sh index 9c6d8db5..48e5b701 100644 --- a/functions/onprem/orborus/run.sh +++ b/functions/onprem/orborus/run.sh @@ -3,6 +3,7 @@ docker run \ --env ENVIRONMENT_NAME="Shuffle" \ --env BASE_URL=http://shuffle-backend:5001 \ --env DOCKER_API_VERSION=1.42 \ + --env RUNNING_MODE="Docker" \ --network "shuffle_shuffle" \ -v /var/run/docker.sock:/var/run/docker.sock \ frikky/shuffle:orborus diff --git a/functions/onprem/worker/Dockerfile b/functions/onprem/worker/Dockerfile index 2f8971f8..685b426b 100644 --- a/functions/onprem/worker/Dockerfile +++ b/functions/onprem/worker/Dockerfile @@ -1,21 +1,16 @@ -#from golang as builder -# -#RUN mkdir /app -#WORKDIR /app -#COPY worker.go /app/worker.go -# -#RUN go get github.com/docker/docker/api/types -#RUN go get github.com/docker/docker/api/types/container -#RUN go get -u github.com/docker/docker/client -# -#RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker . -# +from golang as builder -# THis is a workaround until I get docker/docker to build in a dockerfile -# PS: This is tricky to google. -# Might not work on some machines. -from scratch -#COPY --from=builder /app/ / -COPY worker.bin /worker.bin +WORKDIR /app -CMD ["./worker.bin"] +RUN go get -u github.com/docker/docker/api/types +RUN go get -u github.com/docker/docker/api/types/container +RUN go get -u github.com/docker/docker/client + +COPY worker.go /app/worker.go +RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker . + +FROM alpine:3.12 +RUN apk add --no-cache bash +COPY --from=builder /app/ / + +CMD ["./worker"] diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh index 171308aa..c90ce32c 100644 --- a/functions/onprem/worker/build.sh +++ b/functions/onprem/worker/build.sh @@ -1,11 +1,12 @@ NAME=worker -VERSION=0.1.0 +VERSION=0.6.0 echo "Running docker build with $NAME:$VERSION" CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin . -docker build . -t frikky/shuffle:$NAME -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t frikky/$NAME:$VERSION +docker build . -t frikky/shuffle:$NAME -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION # Push both for now.. -docker push frikky/$NAME:$VERSION -docker push frikky/shuffle:$NAME +#docker push frikky/$NAME:$VERSION +#docker push frikky/shuffle:$NAME #docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION +docker push ghcr.io/frikky/$NAME:$VERSION diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 48b11ebc..247a2369 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -11,12 +11,12 @@ import ( "log" "net/http" "os" + "os/exec" "strings" "time" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" - network "github.com/docker/docker/api/types/network" dockerclient "github.com/docker/docker/client" ) @@ -25,6 +25,29 @@ var baseUrl = os.Getenv("BASE_URL") var baseimagename = "frikky/shuffle" var sleepTime = 2 +var containerId string + +// form container id of current running container +func getThisContainerId() string { + id := "" + cmd := fmt.Sprintf("cat /proc/self/cgroup | grep memory | tail -1 | cut -d/ -f3") + out, err := exec.Command("bash", "-c", cmd).Output() + if err == nil { + id = strings.TrimSpace(string(out)) + } + + return id +} + +func init() { + containerId = getThisContainerId() + if len(containerId) == 0 { + log.Printf("[ERROR] No container ID found.") + } else { + log.Printf("[INFO] Found container ID: %s", containerId) + } +} + type User struct { Username string `datastore:"Username" json:"username"` Password string `datastore:"password,noindex" password:"password,omitempty"` @@ -336,7 +359,7 @@ type ExecutionRequestWrapper struct { func shutdown(executionId, workflowId string) { dockercli, err := dockerclient.NewEnvClient() if err != nil { - log.Printf("Unable to create docker client: %s", err) + log.Printf("[ERROR] Unable to create docker client: %s", err) os.Exit(3) } @@ -373,18 +396,18 @@ func shutdown(executionId, workflowId string) { ) if err != nil { - log.Println("Failed building request: %s", err) + log.Println("[INFO] Failed building request: %s", err) } // FIXME: Add an API call to the backend authorization := os.Getenv("AUTHORIZATION") if len(authorization) > 0 { req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", authorization)) + } else { + log.Printf("[ERROR] No authorization specified for abort") } req.Header.Add("Content-Type", "application/json") - //req.Header.Add("Authorization", authorization) - client := &http.Client{ Transport: &http.Transport{ Proxy: nil, @@ -397,23 +420,24 @@ func shutdown(executionId, workflowId string) { client = &http.Client{} } else { if len(httpProxy) > 0 { - log.Printf("Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy) + log.Printf("[INFO] Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy) } if len(httpsProxy) > 0 { - log.Printf("Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy) + log.Printf("[INFO] Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy) } } _, err = client.Do(req) if err != nil { - log.Printf("Failed abort request: %s", err) + log.Printf("[INFO] Failed abort request: %s", err) } - log.Printf("Finished shutdown.") + log.Printf("[INFO] Finished shutdown.") os.Exit(3) } // Deploys the internal worker whenever something happens func deployApp(cli *dockerclient.Client, image string, identifier string, env []string) error { + // form basic hostConfig hostConfig := &container.HostConfig{ LogConfig: container.LogConfig{ Type: "json-file", @@ -421,40 +445,34 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] }, } + // form container id and use it as network source if it's not empty + if containerId != "" { + hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) + } else { + log.Printf("[WARNING] Empty self container id, continue without NetworkMode") + } + config := &container.Config{ Image: image, Env: env, } - networkConfig := &network.NetworkingConfig{} - shuffleNetwork := os.Getenv("DOCKER_NETWORK") - if len(shuffleNetwork) > 0 { - networkConfig = &network.NetworkingConfig{ - EndpointsConfig: map[string]*network.EndpointSettings{ - shuffleNetwork: { - NetworkID: shuffleNetwork, - }, - }, - } - } - cont, err := cli.ContainerCreate( context.Background(), config, hostConfig, - networkConfig, + nil, nil, identifier, ) if err != nil { - log.Println(err) + log.Printf("Container error: %s", err) return err } cli.ContainerStart(context.Background(), cont.ID, types.ContainerStartOptions{}) - fmt.Printf("\n") - log.Printf("Container %s is created", cont.ID) + log.Printf("[INFO] Container %s is created", cont.ID) return nil } @@ -463,7 +481,7 @@ func removeContainer(containername string) error { cli, err := dockerclient.NewEnvClient() if err != nil { - log.Printf("Unable to create docker client: %s", err) + log.Printf("[INFO] Unable to create docker client: %s", err) return err } @@ -882,9 +900,11 @@ func handleExecution(client *http.Client, req *http.Request, workflowExecution W err = deployApp(dockercli, image, identifier, env) if err != nil { - log.Printf("Failed deploying %s from image %s: %s", identifier, image, err) - log.Printf("Should send status and exit the entire thing?") - //shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + log.Printf("[ERROR] Failed deploying %s from image %s: %s", identifier, image, err) + if strings.Contains(err.Error(), "No such image") { + log.Printf("[ERROR] Image doesn't exist. Shutting down") + shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + } } log.Printf("Adding visited (3): %s", action.Label) @@ -1072,7 +1092,7 @@ func runTestExecution(client *http.Client, workflowId, apikey string) (string, s return "", "" } - log.Printf("Body: %s", string(body)) + log.Printf("[INFO] Body: %s", string(body)) var workflowExecution WorkflowExecution err = json.Unmarshal(body, &workflowExecution) if err != nil { @@ -1085,7 +1105,7 @@ func runTestExecution(client *http.Client, workflowId, apikey string) (string, s // Initial loop etc func main() { - log.Printf("Setting up worker environment") + log.Printf("[INFO] Setting up worker environment") sleepTime := 5 client := &http.Client{ @@ -1107,13 +1127,6 @@ func main() { } } - shuffleNetwork := os.Getenv("DOCKER_NETWORK") - if len(shuffleNetwork) > 0 { - log.Printf("Running with Docker network %s", shuffleNetwork) - } else { - log.Printf("No docker network specified for Worker.") - } - // WORKER_TESTING_WORKFLOW should be a workflow ID authorization := "" executionId := "" @@ -1121,7 +1134,7 @@ func main() { shuffle_apikey := os.Getenv("WORKER_TESTING_APIKEY") if len(testing) > 0 && len(shuffle_apikey) > 0 { // Execute a workflow and use that info - log.Printf("!! Running test environment for worker by executing workflow %s", testing) + log.Printf("[WARNING] Running test environment for worker by executing workflow %s", testing) authorization, executionId = runTestExecution(client, testing, shuffle_apikey) //os.Exit(3) @@ -1132,12 +1145,12 @@ func main() { } if len(authorization) == 0 { - log.Println("No AUTHORIZATION key set in env") + log.Println("[INFO] No AUTHORIZATION key set in env") shutdown(executionId, "") } if len(executionId) == 0 { - log.Println("No EXECUTIONID key set in env") + log.Println("[INFO] No EXECUTIONID key set in env") shutdown(executionId, "") } @@ -1151,7 +1164,7 @@ func main() { ) if err != nil { - log.Println("Failed making request builder") + log.Println("[ERROR] Failed making request builder for backend") shutdown(executionId, "") } @@ -1160,20 +1173,20 @@ func main() { // Removed request requirement from app_sdk newresp, err := client.Do(req) if err != nil { - log.Printf("Failed request: %s", err) + log.Printf("[ERROR] Failed request: %s", err) time.Sleep(time.Duration(sleepTime) * time.Second) continue } body, err := ioutil.ReadAll(newresp.Body) if err != nil { - log.Printf("Failed reading body: %s", err) + log.Printf("[ERROR] Failed reading body: %s", err) time.Sleep(time.Duration(sleepTime) * time.Second) continue } if newresp.StatusCode != 200 { - log.Printf("Err: %s\nStatusCode: %d", string(body), newresp.StatusCode) + log.Printf("[ERROR] %s\nStatusCode: %d", string(body), newresp.StatusCode) time.Sleep(time.Duration(sleepTime) * time.Second) continue } @@ -1181,13 +1194,13 @@ func main() { var workflowExecution WorkflowExecution err = json.Unmarshal(body, &workflowExecution) if err != nil { - log.Printf("Failed workflowExecution unmarshal: %s", err) + log.Printf("[ERROR] Failed workflowExecution unmarshal: %s", err) time.Sleep(time.Duration(sleepTime) * time.Second) continue } if workflowExecution.Status == "FINISHED" || workflowExecution.Status == "SUCCESS" { - log.Printf("Workflow %s is finished. Exiting worker.", workflowExecution.ExecutionId) + log.Printf("[INFO] Workflow %s is finished. Exiting worker.", workflowExecution.ExecutionId) shutdown(executionId, workflowExecution.Workflow.ID) } @@ -1195,15 +1208,14 @@ func main() { //log.Printf("Status: %s", workflowExecution.Status) err = handleExecution(client, req, workflowExecution) if err != nil { - log.Printf("Workflow %s is finished: %s", workflowExecution.ExecutionId, err) + log.Printf("[INFO] Workflow %s is finished: %s", workflowExecution.ExecutionId, err) shutdown(executionId, workflowExecution.Workflow.ID) } } else { - log.Printf("Workflow %s has status %s. Exiting worker.", workflowExecution.ExecutionId, workflowExecution.Status) + log.Printf("[INFO] Workflow %s has status %s. Exiting worker.", workflowExecution.ExecutionId, workflowExecution.Status) shutdown(executionId, workflowExecution.Workflow.ID) } - //log.Println(string(body)) time.Sleep(time.Duration(sleepTime) * time.Second) } } diff --git a/functions/stitcher.go b/functions/stitcher.go index 1b752f77..aae060f3 100644 --- a/functions/stitcher.go +++ b/functions/stitcher.go @@ -101,15 +101,15 @@ func getRunner(classname string) string { return fmt.Sprintf(` # Run the actual thing after we've checked params def run(request): - action = request.get_json() + action = request.get_json() print(action) print(type(action)) authorization_key = action.get("authorization") current_execution_id = action.get("execution_id") - + if action and "name" in action and "app_name" in action: asyncio.run(%s.run(action), debug=True) - return f'Attempting to execute function {action["name"]} in app {action["app_name"]}' + return f'Attempting to execute function {action["name"]} in app {action["app_name"]}' else: return f'Invalid action' @@ -610,6 +610,7 @@ func buildImage(client *client.Client, tags []string, dockerBuildCtxDir string) PullParent: true, Remove: true, Tags: tags, + NetworkMode: "host", }, )