diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..08fca979 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,15 @@ +**/.git* +**/.gitlab-ci.yml +**/.env +**/.dockerignore +**/Dockerfile* +**/node_modules +buildSrc/libs +.gradle/ +build/ +**build/ +.idea/ +!.idea/codeStyles/codeStyleConfig.xml +.DS_Store +*.log +out/ diff --git a/.env b/.env index dbb4fc87..4fd9eced 100644 --- a/.env +++ b/.env @@ -34,3 +34,7 @@ DB_LOCATION=./shuffle-database SHUFFLE_HTTP_PROXY= SHUFFLE_HTTPS_PROXY= SHUFFLE_PASS_WORKER_PROXY=TRUE + +SHUFFLE_BASE_IMAGE_REGISTRY=docker.io +SHUFFLE_BASE_IMAGE_NAME=frikky/shuffle +SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-0.6.0" \ No newline at end of file diff --git a/backend/app_gen/openapi/test.go b/backend/app_gen/openapi/test.go index b7cde4f4..ec818b25 100644 --- a/backend/app_gen/openapi/test.go +++ b/backend/app_gen/openapi/test.go @@ -180,7 +180,7 @@ func generateYaml(swagger *openapi3.Swagger) (WorkflowApp, []string, error) { api.Name = swagger.Info.Title api.Description = swagger.Info.Description api.IsValid = true - api.Link = swagger.Servers[0].URL // host doesnt exist lol + api.Link = swagger.Servers[0].URL // host does not exist lol api.AppVersion = "1.0.0" api.Environment = "cloud" api.ID = "" diff --git a/backend/app_gen/openapi/testGCP.go b/backend/app_gen/openapi/testGCP.go index da5510b8..a792ff3c 100644 --- a/backend/app_gen/openapi/testGCP.go +++ b/backend/app_gen/openapi/testGCP.go @@ -216,7 +216,7 @@ func generateYaml(swagger *openapi3.Swagger) (WorkflowApp, []string, error) { api.Name = swagger.Info.Title api.Description = swagger.Info.Description api.IsValid = true - api.Link = swagger.Servers[0].URL // host doesnt exist lol + api.Link = swagger.Servers[0].URL // host does not exist lol api.AppVersion = "1.0.0" api.Environment = "cloud" api.ID = "" diff --git a/backend/app_gen/python-lib/generator.py b/backend/app_gen/python-lib/generator.py index 2af39cdc..e982439c 100644 --- a/backend/app_gen/python-lib/generator.py +++ b/backend/app_gen/python-lib/generator.py @@ -246,7 +246,7 @@ def loop_modules(modules, data): param["position"] = cnt if not found: - # FIXME - waht here? + # FIXME - what here? pass #print("HANDLE NOT FOUND") #print(param) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 97026b67..e896f67c 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -150,7 +150,7 @@ class AppBase: self.logger.info("AFTER FULLEXEC stream result") - # Gets the value at the paranthesis level you want + # Gets the value at the parenthesis level you want def parse_nested_param(string, level): """ Generate strings contained in nested (), indexing i = level @@ -173,7 +173,7 @@ class AppBase: except IndexError: return [string[LeftRightIndex[level+1][0]:LeftRightIndex[level+1][1]]] - # Finds the deepest level paranthesis in a string + # Finds the deepest level parenthesis in a string def maxDepth(S): current_max = 0 max = 0 diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index ef7a12f2..d10f16c2 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -426,7 +426,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, //uuid.NewV4().String() api.IsValid = true - api.Link = swagger.Servers[0].URL // host doesnt exist lol + api.Link = swagger.Servers[0].URL // host does not exist lol if strings.HasSuffix(api.Link, "/") { api.Link = api.Link[:len(api.Link)-1] } diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 4969950a..4a78d089 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -6838,7 +6838,7 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { } } -func init() { +func initHandlers() { var err error ctx := context.Background() @@ -6964,7 +6964,7 @@ func init() { // Had to move away from mux, which means Method is fucked up right now. func main() { - //init() + initHandlers() hostname, err := os.Hostname() if err != nil { hostname = "MISSING" diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 9d45971b..95f54058 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -28,6 +28,7 @@ import ( "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/storage/memory" http2 "gopkg.in/src-d/go-git.v4/plumbing/transport/http" @@ -1551,6 +1552,12 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { log.Printf("Failed to increase total workflows: %s", err) } + // recalculate authenticators stats + err = recalculateAppAuthentications() + if err != nil { + log.Printf("Authentications recalculation failed: %s", err) + } + //memcacheName := fmt.Sprintf("%s_%s", user.Username, fileId) //memcache.Delete(ctx, memcacheName) //memcacheName = fmt.Sprintf("%s_workflows", user.Username) @@ -4396,6 +4403,7 @@ func deployWebhookFunction(ctx context.Context, name, localization, applocation func loadGithubWorkflows(url, username, password, userId, branch string) error { fs := memfs.New() + // FIXME: add more git options lol if strings.Contains(url, "github") || strings.Contains(url, "gitlab") || strings.Contains(url, "bitbucket") { cloneOptions := &git.CloneOptions{ URL: url, @@ -4404,16 +4412,11 @@ func loadGithubWorkflows(url, username, password, userId, branch string) error { // FIXME: Better auth. if len(username) > 0 && len(password) > 0 { cloneOptions.Auth = &http2.BasicAuth{ - Username: username, Password: password, } } - if len(branch) > 0 { - cloneOptions.ReferenceName = plumbing.ReferenceName(branch) - } - storer := memory.NewStorage() r, err := git.Clone(storer, fs, cloneOptions) if err != nil { @@ -4421,9 +4424,31 @@ func loadGithubWorkflows(url, username, password, userId, branch string) error { return err } + if len(branch) > 0 { + log.Printf("Checkout to branch: %s", branch) + + w, _ := r.Worktree() + + err := r.Fetch(&git.FetchOptions{ + RefSpecs: []config.RefSpec{"refs/*:refs/*", "HEAD:refs/heads/HEAD"}, + }) + if err != nil { + log.Printf("Failed fetch for git repo: %s", err) + } + + err = w.Checkout(&git.CheckoutOptions{ + Branch: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branch)), + Force: true, + }) + if err != nil { + log.Printf("Failed checkout for git repo: %s", err) + return errors.New(fmt.Sprintf("Failed checking out to branch %s - does it exist?", branch)) + } + } + dir, err := fs.ReadDir("/") if err != nil { - log.Printf("FAiled reading folder: %s", err) + log.Printf("Failed reading folder: %s", err) } _ = r @@ -4498,7 +4523,7 @@ func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) { if err != nil { log.Printf("Error with unmarshal tmpBody: %s", err) resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) + resp.Write([]byte(`{"success": false, "reason": "json decode error"}`)) return } @@ -4507,14 +4532,82 @@ func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) { if err != nil { log.Printf("Failed to update workflows: %s", err) resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } + // recalculate authenticators stats + err = recalculateAppAuthentications() + if err != nil { + log.Printf("Authentications recalculation failed: %s", err) + } + resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } +func recalculateAppAuthentications() error { + // create context + ctx := context.Background() + + // form workflows list + workflows, err := getAllWorkflows(ctx) + if err != nil { + log.Printf("Error: Failed getting workflows: %s", err) + return err + } + + // form authenticators list + auths, err := getAllWorkflowAppAuth(ctx) + if err != nil { + log.Printf("Error: Failed getting auths: %s", err) + return err + } + + // iterate through auths + for _, auth := range auths { + // reset calculated values + auth.WorkflowCount = 0 + auth.NodeCount = 0 + auth.Usage = []AuthenticationUsage{} + + // iterate through workflows to find which uses this auth + for _, workflow := range workflows { + hasCurrentAuth := false + usageItem := AuthenticationUsage{ + WorkflowId: workflow.ID, + Nodes: []string{}, + } + + // iterate through actions + for _, action := range workflow.Actions { + if action.AuthenticationId == auth.Id { + // this workflow should be added to "usage" field + hasCurrentAuth = true + + // add this action to list + usageItem.Nodes = append(usageItem.Nodes, action.ID) + } + } + + // update current auth with found workflow + if hasCurrentAuth { + auth.WorkflowCount += 1 + auth.NodeCount += int64(len(usageItem.Nodes)) + auth.Usage = append(auth.Usage, usageItem) + } + } + + // update record in database + err := setWorkflowAppAuthDatastore(ctx, auth, auth.Id) + if err != nil { + log.Printf("Failed setting up app auth %s: %s", auth.Id, err) + } + } + + return nil +} + func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { diff --git a/docker-compose.yml b/docker-compose.yml index 9c9438cb..d5fa531d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -61,6 +61,9 @@ services: - HTTPS_PROXY=${SHUFFLE_HTTPS_PROXY} - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY} - SHUFFLE_ORBORUS_EXECUTION_TIMEOUT=600 + - SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME} + - SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY} + - SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX} restart: unless-stopped database: #build: ./backend/database diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index 77f2f2c2..cc4cb29a 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -1159,7 +1159,7 @@ const Admin = (props) => { console.log("Show apps with this category") }} > - Find app ({data.apps.length}) + Find app ({data.apps === null ? 0 : data.apps.length}) @@ -1222,7 +1222,7 @@ const Admin = (props) => { style={{minWidth: 150, maxWidth: 150}} /> { : null - // FIXME - needs refresh everytime you add a new workflow + // FIXME - needs refresh every time you add a new workflow const workflowdata = Object.getOwnPropertyNames(webhookData).length > 0 && selectedWorkflows.length > 0 ? : null diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index f256b72e..ad27220d 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -65,6 +65,7 @@ const Workflows = (props) => { const [field1, setField1] = React.useState("") const [field2, setField2] = React.useState("") const [downloadUrl, setDownloadUrl] = React.useState("https://github.com/frikky/shuffle-workflows") + const [downloadBranch, setDownloadBranch] = React.useState("master") const [loadWorkflowsModalOpen, setLoadWorkflowsModalOpen] = React.useState(false) const [modalOpen, setModalOpen] = React.useState(false); @@ -1186,6 +1187,7 @@ const Workflows = (props) => { const parsedData = { "url": url, + "field_3": downloadBranch || 'master' } if (field1.length > 0) { @@ -1280,6 +1282,26 @@ const Workflows = (props) => { fullWidth /> + Branch (default value is "master"): +
+ setDownloadBranch(e.target.value)} + placeholder="master" + fullWidth + /> +
+ Authentication (optional - private repos etc):