From 63db013438c55d42f1c8f446b8e9cf4e3e5110b6 Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 2 Apr 2021 21:57:02 +0200 Subject: [PATCH 01/96] Fixed issues with string and json parsing in SDK --- backend/app_sdk/app_base.py | 61 ++++++++++++------ backend/app_sdk/build.sh | 2 +- backend/app_sdk/requirements.txt | 4 +- backend/go-app/go.mod | 2 +- backend/go-app/main.go | 4 +- backend/go-app/walkoff.go | 2 +- frontend/src/views/Admin.jsx | 102 +++++++++++++++++++++--------- frontend/src/views/AppCreator.jsx | 3 +- frontend/src/views/Workflows.jsx | 50 ++++++++++++++- functions/onprem/orborus/go.sum | 5 ++ 10 files changed, 177 insertions(+), 58 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index c3bc15c4..7aecd2bf 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -859,7 +859,6 @@ class AppBase: return parse_nested_param(string + ')', level) elif len(re.findall("\(", string)) < len(re.findall("\)", string)): return parse_nested_param('(' + string, level) - else: return 'Failed to parse params' @@ -950,17 +949,30 @@ class AppBase: print(f"JSON ERROR in join(): {e}") if "len" in thistype or "length" in thistype or "lenght" in thistype: + print(f"Trying to length-parse: {data}") + tmp = "" try: - tmp = json.loads(tmpdata) - except: + tmp = json.loads(data) + except (NameError, KeyError, TypeError, json.decoder.JSONDecodeError) as e: try: - tmpdata = data.replace("\'", "\"") - tmp = json.loads(tmpdata) - except: - print("[ERROR] Parsing bug for length in app sdk") + print(f"[WARNING] INITIAL Parsing bug for length in app sdk: {e}") + #data = data.replace("\'", "\"") + data = data.replace("True", "true") + data = data.replace("False", "false") + data = data.replace("None", "null") + data = data.replace("\"", "\\\"") + data = data.replace("'", "\"") + + tmp = json.loads(data) + except (NameError, KeyError, TypeError, json.decoder.JSONDecodeError) as e: + print(f"[ERROR] Parsing bug for length in app sdk: {e}") pass + if tmp == "": + print("[WARNING] Length parsing: item wasn't parsed") + tmp = data + if isinstance(tmp, list): return str(len(tmp)) elif isinstance(tmp, object): @@ -1028,20 +1040,29 @@ class AppBase: print("INNER: ", innervalue) print("OUTER: ", outervalue) + # FIXME: There is a known bug here with nested paranthesis + #if outervalue != innervalue: + # # FIXME: This line right here WILL break things when we + # # Do recursive paranthesis in the future + # innervalue = outervalue + + # #print("Outer: ", outervalue, " inner: ", innervalue) + # for key in range(len(innervalue)): + # # Replace OUTERVALUE[key] with INNERVALUE[key] in data. + # print("\nReplace %s\nwith\n%s\nin\n%s" % (outervalue[key], innervalue[key], data)) + # data = data.replace(outervalue[key], innervalue[key]) + #else: if outervalue != innervalue: - #print("Outer: ", outervalue, " inner: ", innervalue) - for key in range(len(innervalue)): - # Replace OUTERVALUE[key] with INNERVALUE[key] in data. - print("Replace %s with %s in %s" % (outervalue[key], innervalue[key], data)) - data = data.replace(outervalue[key], innervalue[key]) - else: - for thistype in wrappers: - if thistype.lower() not in data.lower(): - continue - - parsed_value = parse_type(innervalue[0], thistype.lower()) - print("Parsed value from %s: %s" % (thistype, parsed_value)) - return (parsed_value, True) + print("Setting inner to outer") + innervalue = outervalue + + for thistype in wrappers: + if thistype.lower() not in data.lower(): + continue + + parsed_value = parse_type(innervalue[0], thistype.lower()) + print("Parsed value from %s: %s" % (thistype, parsed_value)) + return (parsed_value, True) #print("DATA: %s\n" % data) return (parse_wrapper(data)[0], True) diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index eaee980c..695cacd6 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,6 +1,6 @@ #!/bin/bash NAME=shuffle-app_sdk -VERSION=0.8.64 +VERSION=0.8.71 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/app_sdk/requirements.txt b/backend/app_sdk/requirements.txt index acde3f06..60b59271 100644 --- a/backend/app_sdk/requirements.txt +++ b/backend/app_sdk/requirements.txt @@ -1,2 +1,2 @@ -urllib3=1.25.9 -requests=2.25.1 +urllib3==1.25.9 +requests==2.25.1 diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 4e64fc86..02af7a49 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -2,7 +2,7 @@ module shuffle go 1.13 -//replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared +replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi require ( diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 097bc190..d2110f56 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5140,12 +5140,13 @@ func runInit(ctx context.Context) { // Gets environments and inits if it doesn't exist count, err := getEnvironmentCount() if count == 0 && err == nil && len(activeOrgs) == 1 { - log.Printf("Setting up environment with org %s", activeOrgs[0].Id) + log.Printf("[INFO] Setting up environment with org %s", activeOrgs[0].Id) item := shuffle.Environment{ Name: "Shuffle", Type: "onprem", OrgId: activeOrgs[0].Id, Default: true, + Id: uuid.NewV4().String(), } err = setEnvironment(ctx, &item) @@ -5889,6 +5890,7 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { Registered: true, Default: false, OrgId: org.Id, + Id: uuid.NewV4().String(), } err = setEnvironment(ctx, &newEnv) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 86f27be7..c694fd82 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -1058,7 +1058,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { } if workflowExecution.Status == "FINISHED" { - log.Printf("Workflowexecution is already FINISHED. No further action can be taken") + log.Printf("[INFO] Workflowexecution is already FINISHED. No further action can be taken.") resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Workflowexecution is already finished because of %s with status %s"}`, workflowExecution.LastNode, workflowExecution.Status))) return diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index dfefcd5f..6201fdf8 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -474,6 +474,39 @@ const Admin = (props) => { }); } + const inviteUser = (data) => { + console.log("INPUT: ", data) + setLoginInfo("") + + // Just use this one? + var data = { "username": data.Username, "type": "invite", "org_id": selectedOrganization.id} + var baseurl = globalUrl + const url = baseurl + '/api/v1/users/register_org'; + + fetch(url, { + method: 'POST', + credentials: "include", + body: JSON.stringify(data), + headers: { + 'Content-Type': 'application/json', + }, + }) + .then(response => + response.json().then(responseJson => { + if (responseJson["success"] === false) { + setLoginInfo("Error: " + responseJson.reason) + } else { + setLoginInfo("") + setModalOpen(false) + getUsers() + } + }), + ) + .catch(error => { + console.log("Error in userdata: ", error) + }); + } + const submitUser = (data) => { console.log("INPUT: ", data) setLoginInfo("") @@ -1689,6 +1722,11 @@ const Admin = (props) => { {curTab === 1 ? "Add user" : "Add environment"} + {curTab === 1 && isCloud ? + + We'll send an email to invite them to your organization. + + : null} {curTab === 1 ?
Username @@ -1712,32 +1750,36 @@ const Admin = (props) => { variant="outlined" onChange={(event) => changeModalData("Username", event.target.value)} /> - Password - changeModalData("Password", event.target.value)} - /> + {isCloud ? null : + + Password + changeModalData("Password", event.target.value)} + /> + + }
- : curTab === 3 ? + : curTab === 5 ?
Environment Name - { + + + + + : null + const deleteModal = deleteModalOpen ? { >
- Are you sure?
Other workflows relying on this one may stop working + Are you sure you want to delete this workflow?
Other workflows relying on this one may stop working
@@ -888,7 +932,8 @@ const Workflows = (props) => { {"Change details"} { - publishWorkflow(data) + setSelectedWorkflow(data) + setPublishModalOpen(true) }} key={"publish"}> {"Publish Workflow"} @@ -1830,6 +1875,7 @@ const Workflows = (props) => { {modalView} {deleteModal} + {publishModal} {workflowDownloadModalOpen}
: diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index d3ea7e29..ada1bd1f 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -71,6 +71,8 @@ github.com/frikky/kin-openapi v0.38.0 h1:V7ttwIJS8Vks4KL+mZVj1ZSqhIcQtgaG8akeqXE github.com/frikky/kin-openapi v0.38.0/go.mod h1:Fr28TtCHL4K0kIqtqui8HWxN1LG5uAh3z/tDfFyiA1s= github.com/frikky/shuffle-shared v0.0.12 h1:+0EIfThmK47Po+LogPYZR4XjbS4Ds19WNMFu2YUSjhw= github.com/frikky/shuffle-shared v0.0.12/go.mod h1:SEY432/xs4oBkOUnGwxiYKCZWZLRaheGInhAoW7N8ww= +github.com/frikky/shuffle-shared v0.0.23 h1:Pnlc2M6fHnFRLFd5K1iLTVv4/t4P04Ri1GJ5CMxzq0U= +github.com/frikky/shuffle-shared v0.0.23/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -162,6 +164,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= +github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -194,6 +198,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= From 18a3d696230de521aec2447747078a35be5c3a31 Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 3 Apr 2021 20:06:56 +0200 Subject: [PATCH 02/96] Started migration according to #324 --- backend/go-app/main.go | 103 ++++++++++++++++++++----- backend/go-app/walkoff.go | 16 ++-- frontend/src/views/Admin.jsx | 6 +- frontend/src/views/AngularWorkflow.jsx | 76 +++++++++++++++++- functions/onprem/worker/worker.go | 17 +++- 5 files changed, 180 insertions(+), 38 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index d2110f56..0b2b47cd 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -842,7 +842,7 @@ func handleSetEnvironments(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(`{"success": true}`)) } -func createNewUser(username, password, role, apikey string, org shuffle.Org) error { +func createNewUser(username, password, role, apikey string, org shuffle.OrgMini) error { // Returns false if there is an issue // Use this for register err := shuffle.CheckPasswordStrength(password) @@ -893,7 +893,10 @@ func createNewUser(username, password, role, apikey string, org shuffle.Org) err newUser.Roles = []string{"user"} } - newUser.ActiveOrg = org + newUser.ActiveOrg = shuffle.OrgMini{ + Id: org.Id, + Name: org.Name, + } if len(apikey) > 0 { newUser.ApiKey = apikey @@ -1000,7 +1003,10 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) { _, err = dbclient.GetAll(ctx, q, &orgs) if err == nil && len(orgs) == 1 { log.Printf("No org exists in auth. Setting to default (first one)") - currentOrg = orgs[0] + currentOrg = shuffle.OrgMini{ + Id: orgs[0].Id, + Name: orgs[0].Name, + } } } @@ -1274,7 +1280,11 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { } if len(newOrgs) > 0 { - userInfo.ActiveOrg = newOrgs[0] + userInfo.ActiveOrg = shuffle.OrgMini{ + Id: newOrgs[0].Id, + Name: newOrgs[0].Name, + } + userInfo.Orgs = newStringOrgs err = shuffle.SetUser(ctx, &userInfo) @@ -1291,7 +1301,7 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { } else { // 1. Check if the org exists by ID // 2. if it does, overwrite user - userInfo.ActiveOrg = shuffle.Org{ + userInfo.ActiveOrg = shuffle.OrgMini{ Id: userInfo.Orgs[0], } err = shuffle.SetUser(ctx, &userInfo) @@ -1304,12 +1314,15 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { // FIXME: Remove this dependency by updating users' orgs when org itself is updated org, err := shuffle.GetOrg(ctx, userInfo.ActiveOrg.Id) if err == nil { - userInfo.ActiveOrg = *org - userInfo.ActiveOrg.Users = []shuffle.User{} + userInfo.ActiveOrg = shuffle.OrgMini{ + Id: org.Id, + Name: org.Name, + } + + userInfo.ActiveOrg.Users = []shuffle.UserMini{} } - userInfo.ActiveOrg.Users = []shuffle.User{} - userInfo.ActiveOrg.SyncConfig = shuffle.SyncConfig{} + userInfo.ActiveOrg.Users = []shuffle.UserMini{} currentOrg, err := json.Marshal(userInfo.ActiveOrg) if err != nil { currentOrg = []byte("{}") @@ -4988,6 +5001,39 @@ func runInit(ctx context.Context) { } } else { log.Printf("There are %d org(s).", len(activeOrgs)) + + if len(activeOrgs) == 1 { + if len(activeOrgs[0].Users) == 0 { + log.Printf("ORG doesn't have any users??") + + q := datastore.NewQuery("Users") + var users []shuffle.User + _, err = dbclient.GetAll(ctx, q, &users) + if err != nil && len(users) == 0 { + log.Printf("Failed getting users in org fix") + } else { + // Remapping everyone to admin. This should never happen. + + for _, user := range users { + user.ActiveOrg = shuffle.OrgMini{ + Id: activeOrgs[0].Id, + Name: activeOrgs[0].Name, + Role: "admin", + } + + activeOrgs[0].Users = append(activeOrgs[0].Users, user) + } + + err = shuffle.SetOrg(ctx, activeOrgs[0], activeOrgs[0].Id) + if err != nil { + log.Printf("Failed setting org: %s", err) + } else { + log.Printf("Successfully updated org to have users!") + } + } + + } + } } } @@ -5004,7 +5050,7 @@ func runInit(ctx context.Context) { newUser := shuffle.User{ Username: user.Username, Id: user.Id, - ActiveOrg: shuffle.Org{ + ActiveOrg: shuffle.OrgMini{ Id: activeOrg.Id, }, Orgs: []string{activeOrg.Id}, @@ -5050,7 +5096,7 @@ func runInit(ctx context.Context) { q := datastore.NewQuery("Users").Filter("active =", true) var activeusers []shuffle.User _, err = dbclient.GetAll(ctx, q, &activeusers) - if err != nil { + if err != nil && len(activeusers) == 0 { log.Printf("Error getting users during init: %s", err) } else { q := datastore.NewQuery("Users") @@ -5074,9 +5120,9 @@ func runInit(ctx context.Context) { if len(user.Orgs) == 0 { defaultName := "default" user.Orgs = []string{defaultName} - user.ActiveOrg = shuffle.Org{ + user.ActiveOrg = shuffle.OrgMini{ Name: defaultName, - Role: "user", + Role: "admin", } } @@ -5101,9 +5147,10 @@ func runInit(ctx context.Context) { } else { apikey := os.Getenv("SHUFFLE_DEFAULT_APIKEY") - tmpOrg := shuffle.Org{ + tmpOrg := shuffle.OrgMini{ Name: "default", } + err = createNewUser(username, password, "admin", apikey, tmpOrg) if err != nil { log.Printf("Failed to create default user %s: %s", username, err) @@ -5123,7 +5170,11 @@ func runInit(ctx context.Context) { if len(activeOrgs) == 1 && len(users) > 0 { for _, user := range users { if user.ActiveOrg.Id == "" && len(user.Username) > 0 { - user.ActiveOrg = activeOrgs[0] + user.ActiveOrg = shuffle.OrgMini{ + Id: activeOrgs[0].Id, + Name: activeOrgs[0].Name, + } + err = shuffle.SetUser(ctx, &user) if err != nil { log.Printf("Failed updating user %s with org", user.Username) @@ -5178,7 +5229,7 @@ func runInit(ctx context.Context) { q := datastore.NewQuery("workflow").Limit(35) var workflows []shuffle.Workflow _, err = dbclient.GetAll(ctx, q, &workflows) - if err != nil { + if err != nil && len(workflows) == 0 { log.Printf("Error getting workflows in runinit: %s", err) } else { updated := 0 @@ -5187,7 +5238,11 @@ func runInit(ctx context.Context) { setLocal := false if workflow.ExecutingOrg.Id == "" || len(workflow.OrgId) == 0 { workflow.OrgId = activeOrgs[0].Id - workflow.ExecutingOrg = activeOrgs[0] + workflow.ExecutingOrg = shuffle.OrgMini{ + Id: activeOrgs[0].Id, + Name: activeOrgs[0].Name, + } + setLocal = true } else if workflow.Edited == 0 { workflow.Edited = timeNow @@ -5475,7 +5530,7 @@ func runInit(ctx context.Context) { q := datastore.NewQuery("workflow").Limit(35) var workflows []shuffle.Workflow _, err = dbclient.GetAll(ctx, q, &workflows) - if err != nil { + if err != nil && len(workflows) == 0 { log.Printf("Error getting workflows: %s", err) } else { if len(workflows) == 0 { @@ -6041,7 +6096,15 @@ func makeWorkflowPublic(resp http.ResponseWriter, request *http.Request) { FifthItem: user.Id, } - err = executeCloudAction(action, user.ActiveOrg.SyncConfig.Apikey) + org, err := shuffle.GetOrg(ctx, user.ActiveOrg.Id) + if err != nil { + log.Printf("[WARNING] Failed setting getting org during cloud job setting: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + return + } + + err = executeCloudAction(action, org.SyncConfig.Apikey) if err != nil { log.Printf("[WARNING] Failed cloud PUBLISH: %s", err) resp.WriteHeader(401) @@ -6160,7 +6223,7 @@ func initHandlers() { r.HandleFunc("/api/v1/workflows/{key}/schedule/{schedule}", stopSchedule).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/outlook", createOutlookSub).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/outlook/{triggerId}", handleDeleteOutlookSub).Methods("DELETE", "OPTIONS") - r.HandleFunc("/api/v1/workflows/{key}/executions", getWorkflowExecutions).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/workflows/{key}/executions", shuffle.GetWorkflowExecutions).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/executions/{key}/abort", shuffle.AbortExecution).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}", deleteWorkflow).Methods("DELETE", "OPTIONS") diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index c694fd82..935e989e 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -642,7 +642,7 @@ func createSchedule(ctx context.Context, scheduleId, workflowId, name, startNode Body: ioutil.NopCloser(strings.NewReader(bodyWrapper)), } - _, _, err := handleExecution(workflowId, shuffle.Workflow{ExecutingOrg: shuffle.Org{Id: orgId}}, request) + _, _, err := handleExecution(workflowId, shuffle.Workflow{ExecutingOrg: shuffle.OrgMini{Id: orgId}}, request) if err != nil { log.Printf("Failed to execute %s: %s", workflowId, err) } @@ -2513,10 +2513,10 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request // //workflowExecution.ExecutionOrg.SyncFeatures = Org{} //} - workflowExecution.Workflow.ExecutingOrg = shuffle.Org{ + workflowExecution.Workflow.ExecutingOrg = shuffle.OrgMini{ Id: workflowExecution.Workflow.ExecutingOrg.Id, } - workflowExecution.Workflow.Org = []shuffle.Org{ + workflowExecution.Workflow.Org = []shuffle.OrgMini{ workflowExecution.Workflow.ExecutingOrg, } @@ -2698,8 +2698,8 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { //memcacheName := fmt.Sprintf("%s_%s", user.Username, fileId) ctx := context.Background() workflow, err := shuffle.GetWorkflow(ctx, fileId) - if err != nil { - log.Printf("Failed getting the workflow locally (execute workflow): %s", err) + if err != nil && workflow.ID == "" { + log.Printf("[WARNING] Failed getting the workflow locally (execute workflow): %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -2716,7 +2716,7 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { log.Printf("[INFO] Starting execution of %s!", fileId) - user.ActiveOrg.Users = []shuffle.User{} + user.ActiveOrg.Users = []shuffle.UserMini{} workflow.ExecutingOrg = user.ActiveOrg workflowExecution, executionResp, err := handleExecution(fileId, *workflow, request) @@ -4502,11 +4502,11 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra workflow.ID = uuid.NewV4().String() workflow.OrgId = orgId - workflow.ExecutingOrg = shuffle.Org{ + workflow.ExecutingOrg = shuffle.OrgMini{ Id: orgId, } - workflow.Org = append(workflow.Org, shuffle.Org{ + workflow.Org = append(workflow.Org, shuffle.OrgMini{ Id: orgId, }) workflow.IsValid = false diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index 6201fdf8..b73af2fb 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -2043,7 +2043,7 @@ const Admin = (props) => { /> { return ( { if (nodedata.app_name == "Shuffle Tools" || nodedata.app_name == "Testing") { //console.log("NODE: ", //selector: `node[app_name="Shuffle Tools"]`, - console.log(event.target) + //console.log(event.target) // 1. Find location of node // 2. Check if it's within view of another node (inside) @@ -1387,6 +1387,42 @@ const AngularWorkflow = (props) => { const onEdgeAdded = (event) => { setLastSaved(false) const edge = event.target.data() + var targetnode = workflow.triggers.findIndex(data => data.id === edge.target) + if (targetnode !== -1) { + console.log("TARGETNODE: ", targetnode) + if (workflow.triggers[targetnode].app_name === "User Input" || workflow.triggers[targetnode].app_name === "Shuffle Workflow") { + } else { + alert.error("Can't have triggers as target of branch") + event.target.remove() + } + } + + targetnode = -1 + var sourcenode = workflow.triggers.findIndex(data => data.id === edge.source) + //console.log("SOURCENODE: ", sourcenode) + if (sourcenode !== -1) { + if (workflow.triggers[sourcenode].app_name === "User Input" || workflow.triggers[sourcenode].app_name === "Shuffle Workflow") { + //console.log("NORMAL TRIGGER") + } else { + var currentnode = cy.getElementById(workflow.triggers[sourcenode].id) + if (currentnode !== null && currentnode !== undefined) { + console.log("SHOULD CHECK IF TRIGGER HAS MULTIPLE EDGES: ", currentnode) + // https://js.cytoscape.org/#edges.connectedNodes + //console.log("CURRENTNODE: ", currentnode) + //console.log("EDGES: ", currentnode.connectedEdges(`node[id=${workflow.triggers[sourcenode].id}]`)) + //console.log("EDGES2: ", currentnode.connectedEdges()) + //currentnode.connectedEdges().animate({style: {lineColor: "red"}}) + //console.log("OUTGOERS: ", currentnode.outgoers()) + + //console.log("LEN2: ", currentnode.edges().length) + //if (currentnode.connectedNodes().length > 0) { + // alert.error("Can't have multiple branches from this trigger") + // event.target.remove() + //} + } + } + } + //console.log(workflow.branches) // Check if: @@ -1407,7 +1443,7 @@ const AngularWorkflow = (props) => { found = true break } else if (edge.target === workflow.start) { - var targetnode = workflow.triggers.findIndex(data => data.id === edge.source) + targetnode = workflow.triggers.findIndex(data => data.id === edge.source) if (targetnode === -1) { alert.error("Can't make arrow to starting node") event.target.remove() @@ -1428,10 +1464,14 @@ const AngularWorkflow = (props) => { // break // } } else { + console.log("INSIDE LAST CHECK: ", edge) + // Find the targetnode and check if its a trigger // FIXME - do this for both actions and other types? + /* targetnode = workflow.triggers.findIndex(data => data.id === edge.target) if (targetnode !== -1) { + console.log("TARGETNODE: ", targetnode) if (workflow.triggers[targetnode].app_name === "User Input" || workflow.triggers[targetnode].app_name === "Shuffle Workflow") { } else { alert.error("Can't have triggers as target of branch") @@ -1440,6 +1480,9 @@ const AngularWorkflow = (props) => { break } } + */ + + } } @@ -1491,6 +1534,7 @@ const AngularWorkflow = (props) => { workflow.branches = workflow.branches.filter(a => a.id !== edge.data().id) setWorkflow(workflow) + event.target.remove() // trigger as source check const indexcheck = workflow.triggers.findIndex(data => edge.data()["source"] === data.id) @@ -6811,6 +6855,9 @@ const AngularWorkflow = (props) => { theme="solarized" collapsed={true} displayDataTypes={false} + enableClipboard={(copy) => { + handleReactJsonClipboard(copy) + }} onSelect={(select) => { HandleJsonCopy(showResult, select, "exec") console.log("SELECTED!: ", select) @@ -6858,6 +6905,22 @@ const AngularWorkflow = (props) => { ) } + const handleReactJsonClipboard = (copy) => { + console.log("COPY: ", copy) + + const elementName = "copy_element_shuffle" + var copyText = document.getElementById(elementName); + if (copyText !== null && copyText !== undefined) { + navigator.clipboard.writeText(JSON.stringify(copy)) + copyText.select(); + copyText.setSelectionRange(0, 99999); /* For mobile devices */ + + /* Copy the text inside the text field */ + document.execCommand("copy"); + alert.success("Copied data") + } + } + const HandleJsonCopy = (base, copy, base_node_name) => { console.log("COPY: ", copy) var newitem = JSON.parse(base) @@ -7204,10 +7267,14 @@ const AngularWorkflow = (props) => { {data.status}
- {validate.valid ? + { + handleReactJsonClipboard(copy) + }} displayDataTypes={false} onSelect={(select) => { HandleJsonCopy(showResult, select, data.action.label) @@ -7394,6 +7461,9 @@ const AngularWorkflow = (props) => { theme="solarized" collapsed={false} displayDataTypes={false} + enableClipboard={(copy) => { + handleReactJsonClipboard(copy) + }} onSelect={(select) => { HandleJsonCopy(JSON.stringify(validate.result), select, selectedResult.action.label) }} diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index cc981636..642af1bc 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -24,7 +24,7 @@ import ( //"github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/mount" dockerclient "github.com/docker/docker/client" - "github.com/satori/go.uuid" + //"github.com/satori/go.uuid" "github.com/gorilla/mux" "github.com/patrickmn/go-cache" @@ -857,17 +857,25 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { } // Added UUID to identifier just in case - identifier := fmt.Sprintf("%s_%s_%s_%s_%s", appname, appversion, action.ID, workflowExecution.ExecutionId, uuid.NewV4()) + //identifier := fmt.Sprintf("%s_%s_%s_%s_%s", appname, appversion, action.ID, workflowExecution.ExecutionId, uuid.NewV4()) + identifier := fmt.Sprintf("%s_%s_%s_%s", appname, appversion, action.ID, workflowExecution.ExecutionId) if strings.Contains(identifier, " ") { identifier = strings.ReplaceAll(identifier, " ", "-") } + //if arrayContains(executed, action.ID) || arrayContains(visited, action.ID) { + // log.Printf("[WARNING] Action %s is already executed") + // continue + //} + //visited = append(visited, action.ID) + //executed = append(executed, action.ID) + // FIXME - check whether it's running locally yet too dockercli, err := dockerclient.NewEnvClient() if err != nil { log.Printf("[ERROR] Unable to create docker client (2): %s", err) //return err - return + continue } stats, err := dockercli.ContainerInspect(context.Background(), identifier) @@ -1309,6 +1317,7 @@ func arrayContains(visited []string, id string) bool { for _, item := range visited { if item == id { found = true + break } } @@ -2094,7 +2103,7 @@ func sendResult(workflowExecution shuffle.WorkflowExecution, data []byte) { } body, err := ioutil.ReadAll(newresp.Body) - log.Printf("[INFO] BACKEND STATUS: %d", newresp.StatusCode) + //log.Printf("[INFO] BACKEND STATUS: %d", newresp.StatusCode) if err != nil { log.Printf("[ERROR] Failed reading body: %s", err) } else { From e9a94fa145db53dadac000d870afe6cb60635c1c Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 3 Apr 2021 21:17:29 +0200 Subject: [PATCH 03/96] #313: Reactivated execution variables --- frontend/src/views/AngularWorkflow.jsx | 20 ++-- functions/onprem/orborus/build.sh | 2 +- functions/onprem/orborus/orborus.go | 2 +- functions/onprem/worker/build.sh | 2 +- functions/onprem/worker/go.mod | 2 +- functions/onprem/worker/worker.go | 127 +++++++++---------------- 6 files changed, 62 insertions(+), 93 deletions(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 8b90537d..156e476c 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1464,7 +1464,7 @@ const AngularWorkflow = (props) => { // break // } } else { - console.log("INSIDE LAST CHECK: ", edge) + //console.log("INSIDE LAST CHECK: ", edge) // Find the targetnode and check if its a trigger // FIXME - do this for both actions and other types? @@ -2871,9 +2871,9 @@ const AngularWorkflow = (props) => { ) } - const runSearch = (event) => { - if (event.target.value.length > 0) { - setVisibleApps(allApps.filter(app => app.name.toLowerCase().includes(event.target.value.trim().toLowerCase()))) + const runSearch = (value) => { + if (value.length > 0) { + setVisibleApps(allApps.filter(app => app.name.toLowerCase().includes(value.trim().toLowerCase()))) } else { setVisibleApps(prioritizedApps.concat(filteredApps.filter(innerapp => !internalIds.includes(innerapp.id)))) } @@ -2904,8 +2904,14 @@ const AngularWorkflow = (props) => { color="primary" placeholder={"Search Active Apps"} id="appsearch" + onKeyPress={(event) => { + if (event.key === "Enter") { + console.log("ENTER!") + runSearch(event.target.value) + } + }} onBlur={(event) => { - runSearch(event) + runSearch(event.target.value) }} /> {visibleApps.length > 0 ? @@ -7253,7 +7259,7 @@ const AngularWorkflow = (props) => {
{data.action.label}
- + {data.action.name}
@@ -7261,7 +7267,7 @@ const AngularWorkflow = (props) => {
- Status   + Status  {data.status} diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index c6df2439..08b121c2 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-orborus -VERSION=0.8.71 +VERSION=0.8.72 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index a3fd36dd..c2cafa0f 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -248,7 +248,7 @@ func initializeImages() { log.Printf("[WARNING] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %s", appSdkVersion) } if workerVersion == "" { - workerVersion = "0.8.70" + workerVersion = "0.8.72" log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %s", workerVersion) } diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh index 10f2c70c..0c4d435e 100644 --- a/functions/onprem/worker/build.sh +++ b/functions/onprem/worker/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-worker -VERSION=0.8.71 +VERSION=0.8.72 echo "Running docker build with $NAME:$VERSION" #CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin . diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 44d51077..0e54ab5a 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -8,7 +8,7 @@ require ( github.com/docker/docker v20.10.5+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect - github.com/frikky/shuffle-shared v0.0.20 // indirect + github.com/frikky/shuffle-shared v0.0.24 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 642af1bc..91e5b975 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -53,6 +53,7 @@ var nextActions []string var containerIds []string var extra int var startAction string +var results []shuffle.ActionResult var containerId string @@ -264,7 +265,10 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] ) if err != nil { - log.Printf("[WARNING] Container CREATE error: %s", err) + if !strings.Contains(err.Error(), "Conflict. The container name") { + log.Printf("[ERROR] Container CREATE error: %s", err) + } + return err } @@ -963,7 +967,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { pullOptions := types.ImagePullOptions{} if cleanupEnv == "true" { err = deployApp(dockercli, images[0], identifier, env, workflowExecution) - if err != nil { + if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { if strings.Contains(err.Error(), "exited prematurely") { shutdown(workflowExecution, action.ID, err.Error(), true) } @@ -978,7 +982,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { buildBuf := new(strings.Builder) _, err = io.Copy(buildBuf, reader) - if err != nil { + if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { log.Printf("[ERROR] Error in IO copy: %s", err) shutdown(workflowExecution, action.ID, err.Error(), true) } else { @@ -991,7 +995,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { } err = deployApp(dockercli, image, identifier, env, workflowExecution) - if err != nil { + if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { log.Printf("[ERROR] Failed deploying image for the FOURTH time. Aborting if the image doesn't exist") if strings.Contains(err.Error(), "exited prematurely") { @@ -1008,7 +1012,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { } else { err = deployApp(dockercli, images[0], identifier, env, workflowExecution) - if err != nil { + if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { if strings.Contains(err.Error(), "exited prematurely") { shutdown(workflowExecution, action.ID, err.Error(), true) } @@ -1021,7 +1025,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { } err = deployApp(dockercli, image, identifier, env, workflowExecution) - if err != nil { + if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { if strings.Contains(err.Error(), "exited prematurely") { shutdown(workflowExecution, action.ID, err.Error(), true) } @@ -1032,14 +1036,14 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { } err = deployApp(dockercli, image, identifier, env, workflowExecution) - if err != nil { + if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { if strings.Contains(err.Error(), "exited prematurely") { shutdown(workflowExecution, action.ID, err.Error(), true) } log.Printf("[WARNING] Failed deploying image THREE TIMES. Attempting to download the latter as last resort.") reader, err := dockercli.ImagePull(context.Background(), image, pullOptions) - if err != nil { + if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { log.Printf("[ERROR] Failed getting %s. The couldn't be find locally, AND is missing.", image) shutdown(workflowExecution, action.ID, err.Error(), true) } @@ -1059,7 +1063,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { } err = deployApp(dockercli, image, identifier, env, workflowExecution) - if err != nil { + if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { log.Printf("[ERROR] Failed deploying image for the FOURTH time. Aborting if the image doesn't exist") if strings.Contains(err.Error(), "exited prematurely") { shutdown(workflowExecution, action.ID, err.Error(), true) @@ -1120,6 +1124,8 @@ func executionInit(workflowExecution shuffle.WorkflowExecution) error { parents = map[string][]string{} children = map[string][]string{} + results = workflowExecution.Results + startAction = workflowExecution.Start log.Printf("[INFO] STARTACTION: %s", startAction) if len(startAction) == 0 { @@ -1501,57 +1507,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { } } - //if actionResult.Status == "WAITING" && actionResult.Action.AppName == "User Input" { - // log.Printf("SHOULD WAIT A BIT AND RUN CLOUD STUFF WITH USER INPUT! WAITING!") - - // var trigger shuffle.Trigger - // err = json.Unmarshal([]byte(actionResult.Result), &trigger) - // if err != nil { - // log.Printf("Failed unmarshaling actionresult for user input: %s", err) - // resp.WriteHeader(401) - // resp.Write([]byte(`{"success": false}`)) - // return - // } - - // orgId := workflowExecution.ExecutionOrg - // if len(workflowExecution.OrgId) == 0 && len(workflowExecution.Workflow.OrgId) > 0 { - // orgId = workflowExecution.Workflow.OrgId - // } - - // err := handleUserInput(trigger, orgId, workflowExecution.Workflow.ID, workflowExecution.ExecutionId) - // if err != nil { - // log.Printf("Failed userinput handler: %s", err) - // actionResult.Result = fmt.Sprintf("Cloud error: %s", err) - // workflowExecution.Results = append(workflowExecution.Results, actionResult) - // workflowExecution.Status = "ABORTED" - // err = setshuffle.WorkflowExecution(ctx, *workflowExecution, true) - // if err != nil { - // log.Printf("Failed ") - // } else { - // log.Printf("Successfully set the execution to waiting.") - // } - - // resp.WriteHeader(401) - // resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error: %s"}`, err))) - // } else { - // log.Printf("Successful userinput handler") - // resp.WriteHeader(200) - // resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "CLOUD IS DONE"}`))) - - // actionResult.Result = "Waiting for user feedback based on configuration" - - // workflowExecution.Results = append(workflowExecution.Results, actionResult) - // workflowExecution.Status = actionResult.Status - // err = setshuffle.WorkflowExecution(ctx, *workflowExecution, true) - // if err != nil { - // log.Printf("Failed ") - // } else { - // log.Printf("Successfully set the execution to waiting.") - // } - // } - - // return - //} + results = append(results, actionResult) resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) @@ -1622,30 +1578,33 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl resultLength := len(workflowExecution.Results) dbSave := false setExecution := true - //tx, err := dbclient.NewTransaction(ctx) - //if err != nil { - // log.Printf("client.NewTransaction: %v", err) - // resp.WriteHeader(401) - // resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed creating transaction"}`))) - // return - //} - //key := datastore.NameKey("workflowexecution", workflowExecutionId, nil) - //workflowExecution := &shuffle.WorkflowExecution{} - //if err := tx.Get(key, workflowExecution); err != nil { - // log.Printf("[ERROR] tx.Get bug: %v", err) - // tx.Rollback() - // resp.WriteHeader(401) - // resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting the workflow key"}`))) - // return - //} + if len(actionResult.Action.ExecutionVariable.Name) > 0 { + actionResult.Action.ExecutionVariable.Value = actionResult.Result + + foundIndex := -1 + for i, executionVariable := range workflowExecution.ExecutionVariables { + if executionVariable.Name == actionResult.Action.ExecutionVariable.Name { + foundIndex = i + break + } + } + + if foundIndex >= 0 { + workflowExecution.ExecutionVariables[foundIndex] = actionResult.Action.ExecutionVariable + } else { + workflowExecution.ExecutionVariables = append(workflowExecution.ExecutionVariables, actionResult.Action.ExecutionVariable) + } + } + actionResult.Action = shuffle.Action{ - AppName: actionResult.Action.AppName, - AppVersion: actionResult.Action.AppVersion, - Label: actionResult.Action.Label, - Name: actionResult.Action.Name, - ID: actionResult.Action.ID, - Parameters: actionResult.Action.Parameters, + AppName: actionResult.Action.AppName, + AppVersion: actionResult.Action.AppVersion, + Label: actionResult.Action.Label, + Name: actionResult.Action.Name, + ID: actionResult.Action.ID, + Parameters: actionResult.Action.Parameters, + ExecutionVariable: actionResult.Action.ExecutionVariable, } if actionResult.Status == "ABORTED" || actionResult.Status == "FAILURE" { @@ -2026,6 +1985,10 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } } + if len(results) != len(workflowExecution.Results) { + log.Printf("\n\n[WARNING] There may have been an issue in transaction queue. Result lengths: %d vs %d. Should check which exists the base results, but not in entire execution, then append.\n\n", len(results), len(workflowExecution.Results)) + } + // Validating that action results hasn't changed // Handled using cachhing, so actually pretty fast cacheKey := fmt.Sprintf("workflowexecution-%s", workflowExecution.ExecutionId) From d33ffe8a838734ab0ace758c27f26f5c60af90a9 Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 3 Apr 2021 21:52:34 +0200 Subject: [PATCH 04/96] #331: Made admins able to delete apps without a owner --- backend/go-app/main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 0b2b47cd..5a055d15 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5424,10 +5424,10 @@ func runInit(ctx context.Context) { } // Getting apps to see if we should initialize a test - log.Printf("Getting remote workflow apps") + log.Printf("[INFO] Getting and validating workflowapps") workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500) - if err != nil { - log.Printf("Failed getting apps (runInit): %s", err) + if err != nil && len(workflowapps) == 0 { + log.Printf("[WARNING] Failed getting apps (runInit): %s", err) } else if err == nil && len(workflowapps) > 0 { var allworkflowapps []shuffle.WorkflowApp q := datastore.NewQuery("workflowapp") From 0aa5d95580c8f21073ed9390edf8769ac283ab54 Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 3 Apr 2021 22:09:09 +0200 Subject: [PATCH 05/96] #330: Fixed issue where user can't execute other users' workflow --- backend/go-app/main.go | 4 ++-- backend/go-app/walkoff.go | 19 ++++++++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 5a055d15..56b4ec22 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -861,8 +861,8 @@ func createNewUser(username, password, role, apikey string, org shuffle.OrgMini) q := datastore.NewQuery("Users").Filter("Username =", username) var users []shuffle.User _, err = dbclient.GetAll(ctx, q, &users) - if err != nil { - log.Printf("Failed getting user for registration: %s", err) + if err != nil && len(users) == 0 { + log.Printf("[WARNING] Failed getting user for registration: %s", err) return err } diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 935e989e..49b550d2 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -2076,19 +2076,20 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request } //log.Printf("Execution data: %#v", execution) - if len(execution.Start) == 36 { + if len(execution.Start) == 36 && len(workflow.Actions) > 0 { log.Printf("[INFO] Should start execution on node %s", execution.Start) workflowExecution.Start = execution.Start found := false for _, action := range workflow.Actions { - if action.ID == workflow.Start { + if action.ID == execution.Start { found = true + break } } if !found { - log.Printf("[ERROR] ACTION %s WAS NOT FOUND!", workflow.Start) + log.Printf("[ERROR] ACTION %s WAS NOT FOUND!", execution.Start) return shuffle.WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", workflow.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", workflow.Start)) } } else if len(execution.Start) > 0 { @@ -2708,10 +2709,14 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { // FIXME - have a check for org etc too.. // FIXME - admin check like this? idk if user.Id != workflow.Owner && user.Role != "scheduler" && user.Role != fmt.Sprintf("workflow_%s", fileId) { - log.Printf("Wrong user (%s) for workflow %s (execute)", user.Username, workflow.ID) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return + if workflow.OrgId == user.ActiveOrg.Id && user.Role == "admin" { + log.Printf("[INFO] Letting user %s execute %s because they're admin of the same org", user.Username, workflow.ID) + } else { + log.Printf("[WARNING] Wrong user (%s) for workflow %s (execute)", user.Username, workflow.ID) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } } log.Printf("[INFO] Starting execution of %s!", fileId) From 9d3b4b733a08ede7bbccf2395af408918987f627 Mon Sep 17 00:00:00 2001 From: Felipe Rocha Date: Sat, 3 Apr 2021 18:59:25 -0300 Subject: [PATCH 06/96] fix recursive casting --- backend/app_sdk/app_base.py | 124 +++++++++++++++++------------------- 1 file changed, 60 insertions(+), 64 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 7aecd2bf..9d397aeb 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -950,35 +950,24 @@ class AppBase: if "len" in thistype or "length" in thistype or "lenght" in thistype: print(f"Trying to length-parse: {data}") - - tmp = "" try: - tmp = json.loads(data) + tmp_len = json.loads(data, parse_float=str, parse_int=str, parse_constant=str) except (NameError, KeyError, TypeError, json.decoder.JSONDecodeError) as e: try: print(f"[WARNING] INITIAL Parsing bug for length in app sdk: {e}") - #data = data.replace("\'", "\"") + # data = data.replace("\'", "\"") data = data.replace("True", "true") data = data.replace("False", "false") data = data.replace("None", "null") data = data.replace("\"", "\\\"") data = data.replace("'", "\"") - tmp = json.loads(data) + tmp_len = json.loads(data, parse_float=str, parse_int=str, parse_constant=str) except (NameError, KeyError, TypeError, json.decoder.JSONDecodeError) as e: - print(f"[ERROR] Parsing bug for length in app sdk: {e}") - pass + tmp_len = str(data) - if tmp == "": - print("[WARNING] Length parsing: item wasn't parsed") - tmp = data + return str(len(tmp_len)) - if isinstance(tmp, list): - return str(len(tmp)) - elif isinstance(tmp, object): - return str(len(tmp)) - - return str(len(data)) if "parse" in thistype: splitvalues = [] default_error = """Error. Expected syntax: parse(["hello","test1"],0:1)""" @@ -1015,58 +1004,65 @@ class AppBase: def parse_wrapper(data): try: if "(" not in data or ")" not in data: - return (data, False) + return data, False except TypeError: - return (data, False) - - #print("Running %s" % data) - - # Look for the INNER wrapper first, then move out - wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length", "lenght", "join"] - found = False - for wrapper in wrappers: - if wrapper not in data.lower(): - continue - - found = True - break - - if not found: - return (data, False) - + return data, False + + wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length", "lenght", + "join"] + + if not any(wrapper in data for wrapper in wrappers): + return data, False + # Do stuff here. - innervalue = parse_nested_param(data, maxDepth(data)-0) - outervalue = parse_nested_param(data, maxDepth(data)-1) - print("INNER: ", innervalue) - print("OUTER: ", outervalue) - - # FIXME: There is a known bug here with nested paranthesis - #if outervalue != innervalue: - # # FIXME: This line right here WILL break things when we - # # Do recursive paranthesis in the future - # innervalue = outervalue + inner_value = parse_nested_param(data, maxDepth(data) - 0) + outer_value = parse_nested_param(data, maxDepth(data) - 1) - # #print("Outer: ", outervalue, " inner: ", innervalue) - # for key in range(len(innervalue)): - # # Replace OUTERVALUE[key] with INNERVALUE[key] in data. - # print("\nReplace %s\nwith\n%s\nin\n%s" % (outervalue[key], innervalue[key], data)) - # data = data.replace(outervalue[key], innervalue[key]) - #else: - if outervalue != innervalue: - print("Setting inner to outer") - innervalue = outervalue + print("INNER: ", inner_value) + print("OUTER: ", outer_value) - for thistype in wrappers: - if thistype.lower() not in data.lower(): - continue - - parsed_value = parse_type(innervalue[0], thistype.lower()) - print("Parsed value from %s: %s" % (thistype, parsed_value)) - return (parsed_value, True) - - #print("DATA: %s\n" % data) - return (parse_wrapper(data)[0], True) - + wrapper_group = "|".join(wrappers) + parse_string = data + max_depth = maxDepth(parse_string) + + if outer_value != inner_value: + for casting_items in reversed(range(max_depth + 1)): + c_parentheses = parse_nested_param(parse_string, casting_items)[0] + match_string = re.escape(c_parentheses) + custom_casting = re.findall(fr"({wrapper_group})\({match_string}", parse_string) + + # no matching ; go next group + if len(custom_casting) == 0: + continue + + inner_result = parse_type(c_parentheses, custom_casting[0]) + + # if result is a string then parse else return + if isinstance(inner_result, str): + parse_string = parse_string.replace(f"{custom_casting[0]}({c_parentheses})", inner_result) + elif isinstance(inner_result, list): + parse_string = parse_string.replace(f"{custom_casting[0]}({c_parentheses})", + json.dumps(inner_result)) + else: + parse_string = inner_result + break + else: + c_parentheses = parse_nested_param(parse_string, 0)[0] + match_string = re.escape(c_parentheses) + custom_casting = re.findall(fr"({wrapper_group})\({match_string}", parse_string) + + # check if a wrapper was found + if len(custom_casting) != 0: + inner_result = parse_type(c_parentheses, custom_casting[0]) + if isinstance(inner_result, str): + parse_string = parse_string.replace(f"{custom_casting[0]}({c_parentheses})", inner_result) + elif isinstance(inner_result, list): + parse_string = parse_string.replace(f"{custom_casting[0]}({c_parentheses})", + json.dumps(inner_result)) + else: + parse_string = inner_result + + return parse_string, True # Looks for parantheses to grab special cases within a string, e.g: # int(1) lower(HELLO) or length(what's the length) From 07bbf19d33853459f16ac6b8c161b47c0a46b2e4 Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 4 Apr 2021 17:28:32 +0200 Subject: [PATCH 07/96] #285: Made apps send available versions and mapped them in workflow view --- backend/go-app/go.mod | 4 +- backend/go-app/go.sum | 2 + backend/go-app/main.go | 2 +- backend/go-app/walkoff.go | 207 ------------------------- frontend/src/views/AngularWorkflow.jsx | 100 +++++++++++- frontend/src/views/Apps.jsx | 77 ++++++++- 6 files changed, 174 insertions(+), 218 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 02af7a49..55b11a4d 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -3,6 +3,7 @@ module shuffle go 1.13 replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared + //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi require ( @@ -10,14 +11,15 @@ require ( cloud.google.com/go/datastore v1.4.0 cloud.google.com/go/pubsub v1.3.1 cloud.google.com/go/storage v1.12.0 + github.com/Masterminds/semver v1.5.0 // indirect github.com/Microsoft/go-winio v0.4.14 // indirect github.com/basgys/goxml2json v1.1.0 - github.com/frikky/kin-openapi v0.38.0 github.com/carlescere/scheduler v0.0.0-20170109141437-ee74d2f83d82 github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/docker v1.13.1 github.com/docker/go-connections v0.4.0 github.com/docker/go-units v0.4.0 // indirect + github.com/frikky/kin-openapi v0.38.0 github.com/frikky/shuffle-shared v0.0.23 github.com/ghodss/yaml v1.0.0 github.com/go-git/go-billy/v5 v5.0.0 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 21fd33ff..8d8cf6ec 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -48,6 +48,8 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 56b4ec22..edd8e863 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -6189,11 +6189,11 @@ func initHandlers() { // From here down isnt checked for org specific r.HandleFunc("/api/v1/apps/{appId}", shuffle.UpdateWorkflowAppConfig).Methods("PATCH", "OPTIONS") r.HandleFunc("/api/v1/apps/{appId}", shuffle.DeleteWorkflowApp).Methods("DELETE", "OPTIONS") + r.HandleFunc("/api/v1/apps/{appId}/config", shuffle.GetWorkflowAppConfig).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/apps/run_hotload", handleAppHotloadRequest).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/apps/get_existing", loadSpecificApps).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/download_remote", loadSpecificApps).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/validate", validateAppInput).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/apps/{appId}/config", getWorkflowAppConfig).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/apps", getWorkflowApps).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/apps", setNewWorkflowApp).Methods("PUT", "OPTIONS") r.HandleFunc("/api/v1/apps/search", getSpecificApps).Methods("POST", "OPTIONS") diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 49b550d2..3b780cdf 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -3396,113 +3396,6 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(`{"success": true}`)) } -func getWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - ctx := context.Background() - - location := strings.Split(request.URL.String(), "/") - var fileId string - if location[1] == "api" { - if len(location) <= 4 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - app, err := shuffle.GetApp(ctx, fileId, shuffle.User{}) - if err != nil { - log.Printf("[WARNING] Error getting app %s (app config): %s", fileId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`)) - return - } - - //if IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"` - // Sharing bool `json:"sharing" yaml:"sharing" required:false datastore:"sharing"` - //log.Printf("Sharing: %s", app.Sharing) - //log.Printf("Generated: %s", app.Generated) - //log.Printf("Downloaded: %s", app.Downloaded) - - // FIXME - Handle sharing and such PROPERLY - if app.Sharing && app.Generated { - log.Printf("CAN SHARE APP!") - parsedApi, err := getOpenApiDatastore(ctx, fileId) - if err != nil { - log.Printf("[WARNING] OpenApi doesn't exist for: %s - err: %s", fileId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if len(parsedApi.ID) > 0 { - parsedApi.Success = true - } else { - parsedApi.Success = false - } - - //log.Printf("PARSEDAPI: %#v", parsedApi) - data, err := json.Marshal(parsedApi) - if err != nil { - log.Printf("[WARNING] Error parsing api json: %s", err) - resp.WriteHeader(422) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed marshalling new parsed swagger: %s"}`, err))) - return - } - - resp.WriteHeader(200) - resp.Write(data) - return - } - - user, userErr := shuffle.HandleApiAuthentication(resp, request) - if userErr != nil { - log.Printf("[WARNING] Api authentication failed in get app: %s", userErr) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if user.Id != app.Owner { - log.Printf("[WARNING] Wrong user (%s) for app %s", user.Username, app.Name) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - log.Printf("[INFO] Getting app %s (OpenAPI)", fileId) - parsedApi, err := getOpenApiDatastore(ctx, fileId) - if err != nil { - log.Printf("OpenApi doesn't exist for: %s - err: %s", fileId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - //log.Printf("Parsed API: %#v", parsedApi) - if len(parsedApi.ID) > 0 { - parsedApi.Success = true - } else { - parsedApi.Success = false - } - - data, err := json.Marshal(parsedApi) - if err != nil { - resp.WriteHeader(422) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed marshalling new parsed swagger: %s"}`, err))) - return - } - - resp.WriteHeader(200) - resp.Write(data) -} - func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -3525,34 +3418,6 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { //return } - //if item, err := memcache.Get(ctx, memcacheName); err == memcache.ErrCacheMiss { - // // Not in cache - // log.Printf("Apps not in cache.") - //} else if err != nil { - // log.Printf("Error getting item: %v", err) - //} else { - // // FIXME - verify if value is ok? Can unmarshal etc. - // allApps := item.Value - - // if userErr == nil && len(user.PrivateApps) > 0 { - // var parsedApps []WorkflowApp - // err = json.Unmarshal(allApps, &parsedApps) - // if err == nil { - // log.Printf("Shouldve added %d apps", len(user.PrivateApps)) - // user.PrivateApps = append(user.PrivateApps, parsedApps...) - - // tmpApps, err := json.Marshal(user.PrivateApps) - // if err == nil { - // allApps = tmpApps - // } - // } - // } - - // resp.WriteHeader(200) - // resp.Write(allApps) - // return - //} - workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500) if err != nil { log.Printf("Failed getting apps (getworkflowapps): %s", err) @@ -3560,52 +3425,8 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(`{"success": false}`)) return } - //log.Printf("Length: %d", len(workflowapps)) - - // FIXME - this is really garbage, but is here to protect again null values etc. newapps := workflowapps - /* - skipApps := []string{"Shuffle Subflow"} - newapps := []WorkflowApp{} - baseApps := []WorkflowApp{} - for _, workflowapp := range workflowapps { - //if !workflowapp.Activated && workflowapp.Generated { - // continue - //} - - if workflowapp.Owner != user.Id && user.Role != "admin" && !workflowapp.Sharing { - continue - } - - continueOuter := false - for _, skip := range skipApps { - if workflowapp.Name == skip { - continueOuter = true - break - } - } - - if continueOuter { - continue - } - - //workflowapp.Environment = "cloud" - newactions := []WorkflowAppAction{} - for _, action := range workflowapp.Actions { - //action.Environment = workflowapp.Environment - if len(action.Parameters) == 0 { - action.Parameters = []WorkflowAppActionParameter{} - } - - newactions = append(newactions, action) - } - - workflowapp.Actions = newactions - newapps = append(newapps, workflowapp) - baseApps = append(baseApps, workflowapp) - } - */ if len(user.PrivateApps) > 0 { found := false @@ -3625,7 +3446,6 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { // Double unmarshal because of user apps newbody, err := json.Marshal(newapps) - //newbody, err := json.Marshal(workflowapps) if err != nil { log.Printf("Failed unmarshalling all newapps: %s", err) resp.WriteHeader(401) @@ -3633,33 +3453,6 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { return } - //basebody, err := json.Marshal(baseApps) - ////newbody, err := json.Marshal(workflowapps) - //if err != nil { - // log.Printf("Failed unmarshalling all baseapps: %s", err) - // resp.WriteHeader(401) - // resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking workflow apps"}`))) - // return - //} - - // Refreshed every hour - //item := &memcache.Item{ - // Key: memcacheName, - // Value: basebody, - // Expiration: time.Minute * 60, - //} - //if err := memcache.Add(ctx, item); err == memcache.ErrNotStored { - // if err := memcache.Set(ctx, item); err != nil { - // log.Printf("Error setting item: %v", err) - // } - //} else if err != nil { - // log.Printf("error adding item: %v", err) - //} else { - // log.Printf("Set cache for %s", item.Key) - //} - - //log.Println(string(body)) - //log.Println(string(newbody)) resp.WriteHeader(200) resp.Write(newbody) } diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 156e476c..96bdb439 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1302,9 +1302,10 @@ const AngularWorkflow = (props) => { } - const curapp = apps.find(a => a.name === curaction.app_name && a.app_version === curaction.app_version) + console.log(apps) + const curapp = apps.find(a => a.name === curaction.app_name && (a.app_version === curaction.app_version || a.loop_versions.includes(curaction.app_version))) if (!curapp || curapp === undefined) { - alert.error("App "+curaction.app_name+" not found. Is it activated?") + alert.error(`App ${curaction.app_name}:${curaction.app_version} not found. Is it activated?`) //return } else { @@ -2169,9 +2170,9 @@ const AngularWorkflow = (props) => {
What are WORKFLOW variables? {workflow.workflow_variables === null ? - null : workflow.workflow_variables.map(variable=> { + null : workflow.workflow_variables.map((variable, index) => { return ( -
+
{ }}>
@@ -2906,7 +2907,6 @@ const AngularWorkflow = (props) => { id="appsearch" onKeyPress={(event) => { if (event.key === "Enter") { - console.log("ENTER!") runSearch(event.target.value) } }} @@ -4215,6 +4215,66 @@ const AngularWorkflow = (props) => { borderRadius: borderRadius, } + const getApp = (appId, setApp) => { + fetch(globalUrl+"/api/v1/apps/"+appId+"/config?openapi=false", { + headers: { + 'Accept': 'application/json', + }, + credentials: "include", + }) + .then((response) => { + if (response.status === 200) { + //alert.success("Successfully GOT app "+appId) + } else { + alert.error("Failed getting app") + } + + return response.json() + }) + .then((responseJson) => { + console.log(responseJson) + + if (setApp && responseJson.actions !== undefined && responseJson.actions !== null) { + if (selectedApp.versions !== undefined && selectedApp.versions !== null) { + responseJson.versions = selectedApp.versions + } + + if (selectedApp.loop_versions !== undefined && selectedApp.loop_versions !== null) { + responseJson.loop_versions = selectedApp.loop_versions + } + + var foundAction = responseJson.actions.find(action => action.name === selectedAction.name) + console.log("Old : ", selectedAction) + console.log("Found: ", foundAction) + if (foundAction !== null && foundAction !== undefined) { + for (var paramkey in foundAction.parameters) { + const param = foundAction.parameters[paramkey] + + const foundParam = selectedAction.parameters.find(item => item.name === param.name) + if (foundParam === undefined) { + console.log("COULDNT find Param: ", param) + } else { + console.log("FoundP: ", foundParam) + foundAction.parameters[paramkey] = foundParam + } + } + } else { + alert.error("Couldn't find action "+selectedAction.name) + } + + // Updating params for the new action + selectedAction.parameters = foundAction.parameters + selectedAction.app_id = appId + selectedAction.app_version = responseJson.app_version + + setSelectedAction(selectedAction) + setSelectedApp(responseJson) + } + }) + .catch(error => { + alert.error(error.toString()) + }); + } const innerTextfieldStyle = { color: "white", @@ -4267,16 +4327,42 @@ const AngularWorkflow = (props) => {
-
+
{selectedAction.id === workflow.start ? null : - } + {selectedApp.versions !== null && selectedApp.versions !== undefined && selectedApp.versions.length > 0 ? + + : null }
diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 40d2cb4e..03b98a5f 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -631,6 +631,37 @@ const Apps = (props) => {
+ {selectedApp.versions !== null && selectedApp.versions !== undefined && selectedApp.versions.length > 0 ? + + : null } {isCloud ? @@ -835,7 +866,7 @@ const Apps = (props) => { var tmpapps = searchableApps.filter(data => data.name.toLowerCase().includes(searchfield) || data.description.toLowerCase().includes(searchfield)) newapps.push(...tmpapps) - console.log(newapps) + //console.log(newapps) setFilteredApps(newapps) //if ((newapps.length === 0 || searchBackend) && !appSearchLoading) { @@ -1108,7 +1139,7 @@ const Apps = (props) => { setValidation(true) var cors = "cors" - if (openApi.includes("localhost")) { + if (openApi.includes("= localhost")) { cors = "no-cors" } @@ -1126,6 +1157,48 @@ const Apps = (props) => { }); } + const getApp = (appId, setApp) => { + fetch(globalUrl+"/api/v1/apps/"+appId+"/config?openapi=false", { + headers: { + 'Accept': 'application/json', + }, + credentials: "include", + }) + .then((response) => { + if (response.status === 200) { + //alert.success("Successfully GOT app "+appId) + } else { + alert.error("Failed getting app") + } + + return response.json() + }) + .then((responseJson) => { + console.log(responseJson) + + if (setApp) { + if (selectedApp.versions !== undefined && selectedApp.versions !== null) { + responseJson.versions = selectedApp.versions + } + + if (selectedApp.loop_versions !== undefined && selectedApp.loop_versions !== null) { + responseJson.loop_versions = selectedApp.loop_versions + } + + //alert.info("Should set app to selected") + if (responseJson.actions !== undefined && responseJson.actions !== null && responseJson.actions.length > 0) { + setSelectedAction(responseJson.actions[0]) + } else { + setSelectedAction({}) + } + setSelectedApp(responseJson) + } + }) + .catch(error => { + alert.error(error.toString()) + }); + } + const deleteApp = (appId) => { alert.info("Attempting to delete app") fetch(globalUrl+"/api/v1/apps/"+appId, { From 6690a8c68b18a1ec612e40f1676ee01c6adeb65f Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 5 Apr 2021 17:14:39 +0200 Subject: [PATCH 08/96] #287: Basic import configuration done --- backend/go-app/main.go | 55 +-- backend/go-app/walkoff.go | 185 +------- frontend/src/components/ConfigureWorkflow.jsx | 413 ++++++++++++++++-- frontend/src/views/Admin.jsx | 78 ++-- frontend/src/views/AngularWorkflow.jsx | 90 ++-- frontend/src/views/Workflows.jsx | 6 +- 6 files changed, 496 insertions(+), 331 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index edd8e863..e775ebf5 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -1506,49 +1506,6 @@ func getUserCount() (int, error) { return count, nil } -func handleGetSchedules(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - user, err := shuffle.HandleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in set new workflowhandler: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if user.Role != "admin" { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Admin required"}`)) - return - } - - ctx := context.Background() - schedules, err := getAllSchedules(ctx, user.ActiveOrg.Id) - if err != nil { - log.Printf("Failed getting schedules: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Couldn't get schedules"}`)) - return - } - - newjson, err := json.Marshal(schedules) - if err != nil { - log.Printf("Failed unmarshal: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking environments"}`))) - return - } - - //log.Printf("Existing environments: %s", string(newjson)) - - resp.WriteHeader(200) - resp.Write(newjson) -} - func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -5379,7 +5336,7 @@ func runInit(ctx context.Context) { // Gets schedules and starts them log.Printf("Relaunching schedules") - schedules, err := getAllSchedules(ctx, "ALL") + schedules, err := shuffle.GetAllSchedules(ctx, "ALL") if err != nil { log.Printf("Failed getting schedules during service init: %s", err) } else { @@ -6214,18 +6171,18 @@ func initHandlers() { /* Everything below here increases the counters*/ r.HandleFunc("/api/v1/workflows", shuffle.GetWorkflows).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/workflows", shuffle.SetNewWorkflow).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/workflows/{key}", shuffle.GetSpecificWorkflow).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/workflows/{key}", shuffle.SaveWorkflow).Methods("PUT", "OPTIONS") - r.HandleFunc("/api/v1/workflows/schedules", handleGetSchedules).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/workflows/schedules", shuffle.HandleGetSchedules).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/workflows/{key}/executions", shuffle.GetWorkflowExecutions).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/workflows/{key}/executions/{key}/abort", shuffle.AbortExecution).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/schedule", scheduleWorkflow).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/download_remote", loadSpecificWorkflows).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/execute", executeWorkflow).Methods("GET", "POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/schedule/{schedule}", stopSchedule).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/outlook", createOutlookSub).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/outlook/{triggerId}", handleDeleteOutlookSub).Methods("DELETE", "OPTIONS") - r.HandleFunc("/api/v1/workflows/{key}/executions", shuffle.GetWorkflowExecutions).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/workflows/{key}/executions/{key}/abort", shuffle.AbortExecution).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}", deleteWorkflow).Methods("DELETE", "OPTIONS") + r.HandleFunc("/api/v1/workflows/{key}", shuffle.SaveWorkflow).Methods("PUT", "OPTIONS") + r.HandleFunc("/api/v1/workflows/{key}", shuffle.GetSpecificWorkflow).Methods("GET", "OPTIONS") // Triggers r.HandleFunc("/api/v1/hooks/new", shuffle.HandleNewHook).Methods("POST", "OPTIONS") diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 3b780cdf..20ab9791 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -1079,7 +1079,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { } if actionResult.Status == "WAITING" && actionResult.Action.AppName == "User Input" { - log.Printf("SHOULD WAIT A BIT AND RUN CLOUD STUFF WITH USER INPUT! WAITING!") + log.Printf("[INFO] SHOULD WAIT A BIT AND RUN CLOUD STUFF WITH USER INPUT! WAITING!") var trigger shuffle.Trigger err = json.Unmarshal([]byte(actionResult.Result), &trigger) @@ -4919,187 +4919,6 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) { resp.Write(newjson) } -func getAllSchedules(ctx context.Context, orgId string) ([]ScheduleOld, error) { - var schedules []ScheduleOld - - q := datastore.NewQuery("schedules").Filter("org = ", orgId) - if orgId == "ALL" { - q = datastore.NewQuery("schedules") - } - - _, err := dbclient.GetAll(ctx, q, &schedules) - if err != nil { - return []ScheduleOld{}, err - } - - return schedules, nil -} - -//FIXME: Add cursor -//func shuffle.GetAllWorkflowApps(ctx context.Context, maxLen int) ([]shuffle.WorkflowApp, error) { -// var apps []WorkflowApp -// query := datastore.NewQuery("workflowapp").Order("-edited").Limit(10) -// //query := datastore.NewQuery("workflowapp").Order("-edited").Limit(40) -// -// cacheKey := fmt.Sprintf("workflowapps-sorted-%d", maxLen) -// if value, found := requestCache.Get(cacheKey); found { -// parsedValue := value.(*[]WorkflowApp) -// log.Printf("[INFO] Returning %d apps from cache", len(*parsedValue)) -// return *parsedValue, nil -// } -// -// cursorStr := "" -// -// // NOT BEING UPDATED -// // FIXME: Update the app with the correct actions. HOW DOES THIS WORK?? -// // Seems like only actions are wrong. Could get the app individually. -// // Guessing it's a memory issue. -// //Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions,noindex"` -// //errors.New(nil) -// var err error -// for { -// it := dbclient.Run(ctx, query) -// //_, err = it.Next(&app) -// for { -// var app WorkflowApp -// _, err := it.Next(&app) -// if err != nil { -// break -// } -// -// if app.Name == "Shuffle Subflow" { -// continue -// } -// -// found := false -// //log.Printf("ACTIONS: %d - %s", len(app.Actions), app.Name) -// for _, innerapp := range apps { -// if innerapp.Name == app.Name { -// found = true -// break -// } -// } -// -// if !found { -// apps = append(apps, app) -// } -// } -// -// if err != iterator.Done { -// //log.Printf("[INFO] Failed fetching results: %v", err) -// //break -// } -// -// // Get the cursor for the next page of results. -// nextCursor, err := it.Cursor() -// if err != nil { -// log.Printf("Cursorerror: %s", err) -// break -// } else { -// //log.Printf("NEXTCURSOR: %s", nextCursor) -// nextStr := fmt.Sprintf("%s", nextCursor) -// if cursorStr == nextStr { -// break -// } -// -// cursorStr = nextStr -// query = query.Start(nextCursor) -// //cursorStr = nextCursor -// //break -// } -// -// if len(apps) > maxLen { -// break -// } -// } -// -// if len(apps) > 20 { -// log.Printf("[INFO] Setting %d apps in cache", len(apps)) -// requestCache.Set(cacheKey, &apps, cache.DefaultExpiration) -// } -// -// //var allworkflowapps []WorkflowApp -// //_, err := dbclient.GetAll(ctx, query, &allworkflowapps) -// //if err != nil { -// // if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") { -// // //datastore.NewQuery("workflowapp").Limit(30).Order("-edited") -// // query = datastore.NewQuery("workflowapp").Order("-edited").Limit(25) -// // //q := q.Limit(25) -// // _, err := dbclient.GetAll(ctx, query, &allworkflowapps) -// // if err != nil { -// // return []WorkflowApp{}, err -// // } -// // } else { -// // return []WorkflowApp{}, err -// // } -// //} -// -// return apps, nil -//} - -//func shuffle.GetAllWorkflowAppAuth(ctx context.Context, OrgId string) ([]shuffle.AppAuthenticationStorage, error) { -// var allworkflowapps []AppAuthenticationStorage -// q := datastore.NewQuery("workflowappauth").Filter("org_id = ", OrgId) -// -// _, err := dbclient.GetAll(ctx, q, &allworkflowapps) -// if err != nil { -// return []AppAuthenticationStorage{}, err -// } -// -// return allworkflowapps, nil -//} -// -//func getWorkflowAppAuthDatastore(ctx context.Context, id string) (*AppAuthenticationStorage, error) { -// -// key := datastore.NameKey("workflowappauth", id, nil) -// appAuth := &AppAuthenticationStorage{} -// // New struct, to not add body, author etc -// if err := dbclient.Get(ctx, key, appAuth); err != nil { -// return &AppAuthenticationStorage{}, err -// } -// -// return appAuth, nil -//} -// -//func shuffle.SetWorkflowAppAuthDatastore(ctx context.Context, workflowappauth AppAuthenticationStorage, id string) error { -// timeNow := int64(time.Now().Unix()) -// if workflowappauth.Created == 0 { -// workflowappauth.Created = timeNow -// } -// -// workflowappauth.Edited = timeNow -// -// key := datastore.NameKey("workflowappauth", id, nil) -// -// // New struct, to not add body, author etc -// if _, err := dbclient.Put(ctx, key, &workflowappauth); err != nil { -// log.Printf("Error adding workflow app auth: %s", err) -// return err -// } -// -// return nil -//} -// -//// Hmm, so I guess this should use uuid :( -//// Consistency PLX -//func SetWorkflowAppDatastore(ctx context.Context, workflowapp WorkflowApp, id string) error { -// timeNow := int64(time.Now().Unix()) -// if workflowapp.Created == 0 { -// workflowapp.Created = timeNow -// } -// -// workflowapp.Edited = timeNow -// key := datastore.NameKey("workflowapp", id, nil) -// -// // New struct, to not add body, author etc -// if _, err := dbclient.Put(ctx, key, &workflowapp); err != nil { -// log.Printf("Error adding workflow app: %s", err) -// return err -// } -// -// return nil -//} - // Starts a new webhook func handleStopHook(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) @@ -5372,7 +5191,7 @@ func handleUserInput(trigger shuffle.Trigger, organizationId string, workflowId return err } - log.Printf("Should send email to %s during execution.", email) + log.Printf("[INFO] Should send email to %s during execution.", email) } if strings.Contains(triggerType, "sms") { action := shuffle.CloudSyncJob{ diff --git a/frontend/src/components/ConfigureWorkflow.jsx b/frontend/src/components/ConfigureWorkflow.jsx index 0e83f81b..b311dc95 100644 --- a/frontend/src/components/ConfigureWorkflow.jsx +++ b/frontend/src/components/ConfigureWorkflow.jsx @@ -1,13 +1,24 @@ import React, {useState} from 'react'; -import {Typography, } from '@material-ui/core'; +import { InputAdornment, Tooltip, TextField, CircularProgress, ButtonGroup, Button, Avatar, ListItemAvatar, Typography, List, ListItem, ListItemText} from '@material-ui/core'; +import {FavoriteBorder as FavoriteBorderIcon} from '@material-ui/icons'; +// Handles workflow updates on first open to highlight the issues of the workflow +// Variables +// Action (exists, missing fields) +// Action auth +// Triggers +// +// Specifically used for UNSAVED workflows only? const Workflow = (props) => { - const { workflow, appAuthentication, apps } = props + const { globalUrl, theme, workflow, appAuthentication, setSelectedAction, setAuthenticationModalOpen, setSelectedApp, apps, selectedAction,setConfigureWorkflowModalOpen, saveWorkflow, newWebhook, submitSchedule, referenceUrl, isCloud, } = props const [requiredActions, setRequiredActions] = React.useState([]) + const [requiredVariables, setRequiredVariables] = React.useState([]) + const [requiredTriggers, setRequiredTriggers] = React.useState([]) + const [previousAuth, setPreviousAuth] = React.useState(appAuthentication) const [firstLoad, setFirstLoad] = React.useState("") - - // Rofl + var finished = false + if (workflow === undefined || workflow === null) { return null } @@ -20,63 +31,349 @@ const Workflow = (props) => { return null } + const getApp = (actionId, appId) => { + fetch(globalUrl+"/api/v1/apps/"+appId+"/config?openapi=false", { + headers: { + 'Accept': 'application/json', + }, + credentials: "include", + }) + .then((response) => { + if (response.status === 200) { + //alert.success("Successfully GOT app "+appId) + } else { + alert.error("Failed getting app") + } + + return response.json() + }) + .then((responseJson) => { + console.log("ACTION: ", responseJson) + if (responseJson.actions !== undefined && responseJson.actions !== null) { + } + }) + .catch(error => { + alert.error(error.toString()) + }); + } + if (firstLoad.length === 0 || firstLoad !== workflow.id) { + if (finished) { + setConfigureWorkflowModalOpen(false) + return null + } + setFirstLoad(workflow.id) const newactions = [] for (var key in workflow.actions) { + const action = workflow.actions[key] var newaction = { - "large_image": "", - "app_name": "", - "app_version": "", + "large_image": action.large_image, + "app_name": action.app_name, + "app_version": action.app_version, + "activation_done": false, "must_activate": false, "must_authenticate": false, + "auth_done": false, "action_ids": [], + "action": action, + "app": {}, } - const action = workflow.actions[key] - console.log(action) - const app = apps.find(app => app.name === action.app_name && app.app_version === action.app_version) + const app = apps.find(app => app.name === action.app_name && (app.app_version === action.app_version || app.loop_versions.includes(action.app_version))) if (app === undefined || app === null) { - console.log("COULDNT FIND APP - SEARCH BACKEND") + console.log("App not found!") - newaction.app_name = action.app_name - newaction.app_version = action.app_version + newaction.must_activate = true } else { - newaction.app_name = app.name - newaction.app_version = app.app_version - - console.log("APP: ", app) if (action.authentication_id === "" && app.authentication.required === true) { - console.log("Requires auth!") newaction.must_authenticate = true newaction.action_ids.push(action.id) } - //newaction.app_name = action.app_name - //newaction.app_name = action.app_version + newaction.app = app } if (action.errors !== undefined && action.errors !== null && action.errors.length > 0) { - console.log("Has errors!") + console.log("Node has errors!: ", action.errors) } - console.log("NEWACTION: ", newaction) - if (newaction.must_authenticate || newaction.must_activate) { - newactions.push(newaction) + if (newaction.must_authenticate) { + var authenticationOptions = [] + for (var key in appAuthentication) { + const auth = appAuthentication[key] + if (auth.app.name === app.name && auth.active) { + console.log("Found auth: ", auth) + authenticationOptions.push(auth) + newaction.authenticationId = auth.id + break + } + } + + console.log("APPAUTH: ", app.authentication, action) + if (newaction.authenticationId === null || newaction.authenticationId === undefined || newaction.authenticationId.length === "") { + console.log("FAILED to authentication node!") + newactions.push(newaction) + } else { + console.log("Skipping node as it's already authenticated.") + newaction.authentication = authenticationOptions + workflow.actions[key] = newaction + } + } else if (newaction.must_activate) { + + if (newactions.find(tmpaction => tmpaction.app_id === newaction.app_id && tmpaction.app_name === newaction.app_name) !== undefined) { + console.log("Action already found.") + } else { + newactions.push(newaction) + } } } + for (var key in workflow.workflow_variables) { + const variable = workflow.workflow_variables[key] + if (variable.value === undefined || variable.value === undefined || variable.value.length < 2) { + variable.value = "" + variable.index = key + requiredVariables.push(variable) + } + } + + for (var key in workflow.triggers) { + var trigger = workflow.triggers[key] + trigger.index = key + + if (trigger.status === "running") { + continue + } + + if (trigger.trigger_type === "SUBFLOW" || trigger.trigger_type === "USERINPUT") { + continue + } + + requiredTriggers.push(trigger) + } + + if (requiredTriggers.length === 0 && requiredVariables.length === 0 && newactions.length === 0) { + setConfigureWorkflowModalOpen(false) + } + + console.log("VARIABLES: ", requiredVariables) console.log("ACTIONS: ", newactions) + setRequiredTriggers(requiredTriggers) + setRequiredVariables(requiredVariables) setRequiredActions(newactions) } + console.log("AUTH: ", appAuthentication) + if (appAuthentication.length !== previousAuth.length) { + console.log("APP AUTH CHANGED!") + var newactions = [] + for (var actionkey in requiredActions) { + var newaction = requiredActions[actionkey] + const app = newaction.app + + for (var key in appAuthentication) { + const auth = appAuthentication[key] + if (auth.app.name === app.name && auth.active) { + console.log("FOUND AUTH FOR: ", auth.app.name) + newaction.auth_done = true + break + } + } + + newactions.push(newaction) + } + + setRequiredActions(newactions) + setPreviousAuth(appAuthentication) + // Set auth done to true + //"auth_done": false + } + + const TriggerSection = (props) => { + const {trigger} = props + + console.log(trigger) + + return ( +
+ + + + {trigger.label} + + + + {trigger.trigger_type === "WEBHOOK" && trigger.status !== "running" ? + + : + trigger.trigger_type === "SCHEDULE" && trigger.status !== "running" ? + + : + null} + {/* + + + ) + }} + fullWidth + color="primary" + type={"text"} + placeholder={`New value for ${trigger.name}`} + onChange={(event) => { + console.log("NEW VALUE ON INDEX", trigger.value) + }} + onBlur={(event) => { + //workflow.variables[variable.index] = event.target.value + }} + /> + } + style={{}} + /> + */} + +
+ ) + } + + const VariableSection = (props) => { + const {variable} = props + + //Name: {variable.name} - {variable.value}. + return ( + + + + + + + + + + ) + }} + fullWidth + color="primary" + type={"text"} + placeholder={`New value for ${variable.name}`} + onChange={(event) => { + console.log("NEW VALUE ON INDEX", variable.index, variable.value) + }} + onBlur={(event) => { + workflow.workflow_variables[variable.index].value = event.target.value + }} + /> + } + style={{}} + /> + + ) + } + const AppSection = (props) => { const {action} = props return ( -
- Name: {action.app_name}:{action.app_version}. -
+ + + + {action.app_name} + + + + {action.must_authenticate ? + action.auth_done ? +
+ +
+ : + selectedAction.app_name === action.app_name ? + + : + + : + null} + {action.must_activate ? + + : null} + ) } @@ -84,12 +381,62 @@ const Workflow = (props) => { return (
- Workflow: {workflow.id} - {requiredActions.map((data, index) => { - return ( - - ) - })} + {workflow.name} + + The following configuration makes the workflow ready immediately. + + {requiredActions.length > 0 ? + + Actions + + {requiredActions.map((data, index) => { + return ( + + ) + })} + + + : null} + + {requiredVariables.length > 0 ? + + Variables + {requiredVariables.map((data, index) => { + return ( + + ) + })} + + : null} + + + {requiredTriggers.length > 0 ? + + Triggers + {requiredTriggers.map((data, index) => { + return ( + + ) + })} + + : null } +
+ + + + +
) } diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index b73af2fb..3749f4d0 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -459,9 +459,9 @@ const Admin = (props) => { // FIXME: Set up features - Object.keys(responseJson.sync_features).map(function(key, index) { - //console.log(responseJson.sync_features[key]) - }) + //Object.keys(responseJson.sync_features).map(function(key, index) { + // //console.log(responseJson.sync_features[key]) + //}) //setOrgName(responseJson.name) //setOrgDescription(responseJson.description) @@ -849,7 +849,6 @@ const Admin = (props) => { return response.json() }) .then((responseJson) => { - console.log(responseJson) setSchedules(responseJson) }) .catch(error => { @@ -988,34 +987,45 @@ const Admin = (props) => { }); } + const views = { + 0: "organization", + 1: "users", + 2: "app_auth", + 3: "files", + 4: "schedules", + 5: "environments", + 6: "categories", + } const setConfig = (event, newValue) => { + console.log("Value: ", newValue) + + setCurTab(parseInt(newValue)) if (newValue === 1) { + document.title = "Shuffle - admin - users" getUsers() } else if (newValue === 2) { + document.title = "Shuffle - admin - app authentication" getAppAuthentication() } else if (newValue === 3) { + document.title = "Shuffle - admin - files" getFiles() } else if (newValue === 4) { + document.title = "Shuffle - admin - schedules" getSchedules() } else if (newValue === 5) { + document.title = "Shuffle - admin - environments" getEnvironments() } else if (newValue === 6) { + document.title = "Shuffle - admin - orgs" getOrgs() + } else { + document.title = "Shuffle - admin" } if (newValue === 6) { console.log("Should get apps for categories.") } - const views = { - 0: "organization", - 1: "users", - 2: "app_auth", - 3: "environments", - 4: "schedules", - 5: "files", - 6: "categories", - } //var theURL = window.location.pathname //FIXME: Add url edits @@ -1027,33 +1037,21 @@ const Admin = (props) => { //window.location.pathame = newpath setModalUser({}) - setCurTab(newValue) } if (firstRequest) { setFirstRequest(false) + document.title = "Shuffle - admin" if (!isCloud) { getUsers() } else { getSettings() } - const views = { - "organization": 0, - "users": 1, - "app_auth": 2, - "environments": 3, - "schedules": 4, - "files": 5, - } - if (props.match.params.key !== undefined) { - const tmpitem = views[props.match.params.key] - if (tmpitem !== undefined) { - //setCurTab(tmpitem) - setConfig("", tmpitem) - } + //const tmpitem = views[props.match.params.key] + setConfig("", props.match.params.key) } } @@ -2092,11 +2090,13 @@ const Admin = (props) => { : - - - - - + + + + + + + } style={{minWidth: 100, maxWidth: 100, overflow: "hidden"}} @@ -2116,11 +2116,13 @@ const Admin = (props) => { - { - downloadFile(file) - }}> - - + + { + downloadFile(file) + }}> + + + style={{minWidth: 75, maxWidth: 75, overflow: "hidden"}} /> diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 96bdb439..b32fb4b2 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -128,7 +128,7 @@ const AngularWorkflow = (props) => { const [rightSideBarOpen, setRightSideBarOpen] = React.useState(false) const [showSkippedActions, setShowSkippedActions] = React.useState(false) const [lastExecution, setLastExecution] = React.useState("") - const [configureWorkflowModalOpen, setConfigureWorkflowModalOpen] = React.useState(false) + const [configureWorkflowModalOpen, setConfigureWorkflowModalOpen] = React.useState(true) const [curpath, setCurpath] = useState(typeof window === 'undefined' || window.location === undefined ? "" : window.location.pathname) // 0 = normal, 1 = just done, 2 = normal @@ -707,8 +707,10 @@ const AngularWorkflow = (props) => { newBranches.push(parsedElement) } else { if (type === "ACTION") { - // FIXME - check whether position is new to not fuck up params etc. - var curworkflowAction = useworkflow.actions.find(a => a.id === cyelements[key].data()["id"]) + const cyelement = cyelements[key].data() + const elementid = cyelement.id === undefined || cyelement.id === null ? cyelement["_id"] : cyelement.id + + var curworkflowAction = useworkflow.actions.find(a => a !== undefined && (a["id"] === elementid || a["_id"] === elementid)) if (curworkflowAction === undefined) { curworkflowAction = cyelements[key].data() } @@ -1114,13 +1116,13 @@ const AngularWorkflow = (props) => { const getWorkflow = () => { fetch(globalUrl+"/api/v1/workflows/"+props.match.params.key, { - method: 'GET', + method: 'GET', headers: { 'Content-Type': 'application/json', 'Accept': 'application/json', }, - credentials: "include", - }) + credentials: "include", + }) .then((response) => { if (response.status !== 200) { console.log("Status not 200 for workflows :O!") @@ -1146,6 +1148,12 @@ const AngularWorkflow = (props) => { setWorkflow(responseJson) setWorkflowDone(true) + + //console.log(responseJson) + // Add error checks + if (!responseJson.public && (!responseJson.previously_saved || (!responseJson.is_valid || (responseJson.errors !== undefined || responseJson.errors !== null || responseJson.errors !== responseJson.errors.length > 0)))) { + setConfigureWorkflowModalOpen(true) + } }) .catch(error => { alert.error(error.toString()) @@ -1289,11 +1297,6 @@ const AngularWorkflow = (props) => { //console.log("BRANCHES: ", branch) if (data.type === "ACTION") { - - - // FIXME - unselect - //console.log(cy.elements('[_id!="${data._id}"]`)) - // Does it choose the wrong action? var curaction = workflow.actions.find(a => a.id === data.id) if (!curaction || curaction === undefined) { //event.target.unselect() @@ -1301,15 +1304,27 @@ const AngularWorkflow = (props) => { return } - - console.log(apps) + //console.log(apps) const curapp = apps.find(a => a.name === curaction.app_name && (a.app_version === curaction.app_version || a.loop_versions.includes(curaction.app_version))) if (!curapp || curapp === undefined) { alert.error(`App ${curaction.app_name}:${curaction.app_version} not found. Is it activated?`) + + const tmpapp = { + name: curaction.app_name, + app_name: curaction.app_name, + app_version: curaction.app_version, + id: curaction.app_id, + actions: [curaction], + } + + console.log(tmpapp) + console.log(curaction) + setSelectedApp(tmpapp) + setSelectedAction(curaction) //return } else { - console.log("AUTHENTICATION: ", curapp.authentication) + //console.log("AUTHENTICATION: ", curapp.authentication) setRequiresAuthentication(curapp.authentication.required && curapp.authentication.parameters !== undefined && curapp.authentication.parameters !== null) if (curapp.authentication.required) { // Setup auth here :) @@ -2147,6 +2162,7 @@ const AngularWorkflow = (props) => { } const deleteVariable = (variableName) => { + console.log("Delete:" ,variableName) workflow.workflow_variables = workflow.workflow_variables.filter(data => data.name !== variableName) setWorkflow(workflow) } @@ -2208,7 +2224,7 @@ const AngularWorkflow = (props) => { setOpen(false) setAnchorEl(null) }} - > + > { setOpen(false) @@ -4232,8 +4248,6 @@ const AngularWorkflow = (props) => { return response.json() }) .then((responseJson) => { - console.log(responseJson) - if (setApp && responseJson.actions !== undefined && responseJson.actions !== null) { if (selectedApp.versions !== undefined && selectedApp.versions !== null) { responseJson.versions = selectedApp.versions @@ -4244,17 +4258,14 @@ const AngularWorkflow = (props) => { } var foundAction = responseJson.actions.find(action => action.name === selectedAction.name) - console.log("Old : ", selectedAction) - console.log("Found: ", foundAction) if (foundAction !== null && foundAction !== undefined) { for (var paramkey in foundAction.parameters) { const param = foundAction.parameters[paramkey] const foundParam = selectedAction.parameters.find(item => item.name === param.name) if (foundParam === undefined) { - console.log("COULDNT find Param: ", param) + //console.log("COULDNT find Param: ", param) } else { - console.log("FoundP: ", foundParam) foundAction.parameters[paramkey] = foundParam } } @@ -7878,9 +7889,15 @@ const AngularWorkflow = (props) => { } } + console.log("Action: ", selectedAction) selectedAction.authentication_id = authenticationOption.id selectedAction.selectedAuthentication = authenticationOption - selectedAction.authentication.push(authenticationOption) + if (selectedAction.authentication === undefined || selectedAction.authentication === null) { + selectedAction.authentication = [authenticationOption] + } else { + selectedAction.authentication.push(authenticationOption) + } + setSelectedAction(selectedAction) var newAuthOption = JSON.parse(JSON.stringify(authenticationOption)) @@ -7898,6 +7915,12 @@ const AngularWorkflow = (props) => { setNewAppAuth(newAuthOption) //appAuthentication.push(newAuthOption) //setAppAuthentication(appAuthentication) + // + + if (configureWorkflowModalOpen) { + setSelectedAction({}) + } + setUpdate(authenticationOption.id) /* @@ -8024,18 +8047,23 @@ const AngularWorkflow = (props) => { { - setConfigureWorkflowModalOpen(false) + //setConfigureWorkflowModalOpen(false) }} PaperProps={{ style: { backgroundColor: surfaceColor, color: "white", minWidth: 600, - padding: 15, + padding: 50, }, }} > - + { + setConfigureWorkflowModalOpen(false) + }}> + + + : null @@ -8046,6 +8074,10 @@ const AngularWorkflow = (props) => { open={authenticationModalOpen} onClose={() => { //setAuthenticationModalOpen(false) + // + if (configureWorkflowModalOpen) { + setSelectedAction({}) + } }} PaperProps={{ style: { @@ -8056,6 +8088,14 @@ const AngularWorkflow = (props) => { }, }} > + { + setAuthenticationModalOpen(false) + if (configureWorkflowModalOpen) { + setSelectedAction({}) + } + }}> + +
Authentication for {selectedApp.name}
: null diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 5d1dad88..8a2c9176 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -585,7 +585,7 @@ const Workflows = (props) => { for (var subkey in data.actions[key].parameters) { const param = data.actions[key].parameters[subkey] - if (param.name.includes("key") || param.name.includes("user") || param.name.includes("pass") || param.name.includes("api") || param.name.includes("auth") || param.name.includes("secret")) { + if (param.name.includes("key") || param.name.includes("user") || param.name.includes("pass") || param.name.includes("api") || param.name.includes("auth") || param.name.includes("secret") || param.name.includes("domain") || param.name.includes("url")) { // FIXME: This may be a vuln if api-keys are generated that start with $ if (param.value.startsWith("$")) { console.log("Skipping field, as it's referencing a variable") @@ -645,8 +645,8 @@ const Workflows = (props) => { let exportFileDefaultName = data.name+'.json'; data = sanitizeWorkflow(data) - //console.log("EXPORT: ", data) - //return + // Add correct ID's for triggers + // Add mag let dataStr = JSON.stringify(data) let dataUri = 'data:application/json;charset=utf-8,'+ encodeURIComponent(dataStr); From 8112bd16c507bf40728a0a92d096d36a2a04b679 Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 5 Apr 2021 23:27:45 +0200 Subject: [PATCH 09/96] Moved Action Parsing out of AngularWorkflow. Should be reusable in App explorer UI --- backend/go-app/main.go | 5 - frontend/src/components/ConfigureWorkflow.jsx | 64 +- frontend/src/components/ParsedAction.jsx | 1426 +++++++++++++++ frontend/src/theme.js | 57 + frontend/src/views/AngularWorkflow.jsx | 1580 ++--------------- 5 files changed, 1615 insertions(+), 1517 deletions(-) create mode 100644 frontend/src/components/ParsedAction.jsx create mode 100644 frontend/src/theme.js diff --git a/backend/go-app/main.go b/backend/go-app/main.go index e775ebf5..f86a557c 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -6085,11 +6085,6 @@ func initHandlers() { panic(fmt.Sprintf("DBclient error during init: %s", err)) } - //dbclient, err := shuffle.GetDatastoreClient(ctx, gceProject) - //if err != nil { - // panic(fmt.Sprintf("Error setting datastore connector: %s", err)) - //} - _ = shuffle.RunInit(*dbclient, storage.Client{}, gceProject, "onprem", true) log.Printf("Finished Shuffle database init") diff --git a/frontend/src/components/ConfigureWorkflow.jsx b/frontend/src/components/ConfigureWorkflow.jsx index b311dc95..95457675 100644 --- a/frontend/src/components/ConfigureWorkflow.jsx +++ b/frontend/src/components/ConfigureWorkflow.jsx @@ -95,7 +95,7 @@ const Workflow = (props) => { } if (action.errors !== undefined && action.errors !== null && action.errors.length > 0) { - console.log("Node has errors!: ", action.errors) + //console.log("Node has errors!: ", action.errors) } if (newaction.must_authenticate) { @@ -103,19 +103,23 @@ const Workflow = (props) => { for (var key in appAuthentication) { const auth = appAuthentication[key] if (auth.app.name === app.name && auth.active) { - console.log("Found auth: ", auth) + //console.log("Found auth: ", auth) authenticationOptions.push(auth) newaction.authenticationId = auth.id break } } - console.log("APPAUTH: ", app.authentication, action) if (newaction.authenticationId === null || newaction.authenticationId === undefined || newaction.authenticationId.length === "") { - console.log("FAILED to authentication node!") - newactions.push(newaction) + //console.log("FAILED to authenticate node!") + + if (newactions.find(tmpaction => tmpaction.app_id === newaction.app_id && tmpaction.app_name === newaction.app_name) !== undefined) { + console.log("Action already found.") + } else { + newactions.push(newaction) + } } else { - console.log("Skipping node as it's already authenticated.") + //console.log("Skipping node as it's already authenticated.") newaction.authentication = authenticationOptions workflow.actions[key] = newaction } @@ -157,16 +161,12 @@ const Workflow = (props) => { setConfigureWorkflowModalOpen(false) } - console.log("VARIABLES: ", requiredVariables) - console.log("ACTIONS: ", newactions) setRequiredTriggers(requiredTriggers) setRequiredVariables(requiredVariables) setRequiredActions(newactions) } - console.log("AUTH: ", appAuthentication) if (appAuthentication.length !== previousAuth.length) { - console.log("APP AUTH CHANGED!") var newactions = [] for (var actionkey in requiredActions) { var newaction = requiredActions[actionkey] @@ -175,7 +175,6 @@ const Workflow = (props) => { for (var key in appAuthentication) { const auth = appAuthentication[key] if (auth.app.name === app.name && auth.active) { - console.log("FOUND AUTH FOR: ", auth.app.name) newaction.auth_done = true break } @@ -193,10 +192,7 @@ const Workflow = (props) => { const TriggerSection = (props) => { const {trigger} = props - console.log(trigger) - return ( -
@@ -275,7 +271,6 @@ const Workflow = (props) => { /> */} -
) } @@ -343,16 +338,13 @@ const Workflow = (props) => { {action.must_authenticate ? action.auth_done ? -
- -
+ : selectedAction.app_name === action.app_name ? @@ -377,8 +369,6 @@ const Workflow = (props) => { ) } - console.log(requiredActions) - return (
{workflow.name} @@ -401,11 +391,13 @@ const Workflow = (props) => { {requiredVariables.length > 0 ? Variables - {requiredVariables.map((data, index) => { - return ( - - ) - })} + + {requiredVariables.map((data, index) => { + return ( + + ) + })} + : null} @@ -413,21 +405,25 @@ const Workflow = (props) => { {requiredTriggers.length > 0 ? Triggers - {requiredTriggers.map((data, index) => { - return ( - - ) - })} + + {requiredTriggers.map((data, index) => { + return ( + + ) + })} + : null }
+ {/* + */} +
+
+ } else { + // FIXME - this is a shitty solution that needs re-renders all the time + datafield = + + } + } + + // Shows nested list of nodes > their JSON lists + const ActionlistWrapper = (props) => { + + const handleMenuClose = () => { + setShowAutocomplete(false) + + if (!selectedActionParameters[count].value[selectedActionParameters[count].value.length-1] === "$") { + setShowDropdown(false) + } + + setUpdate(Math.random()) + setMenuPosition(null) + } + + const handleItemClick = (values) => { + if (values === undefined || values === null || values.length === 0) { + return + } + + var toComplete = selectedActionParameters[count].value.trim().endsWith("$") ? values[0].autocomplete : "$"+values[0].autocomplete + for (var key in values) { + if (key == 0 || values[key].autocomplete.length === 0) { + continue + } + + toComplete += values[key].autocomplete + } + + // Handles the fields under OpenAPI body to be parsed. + if (data.name.startsWith("${") && data.name.endsWith("}")) { + console.log("INSIDE VALUE REPLACE: ", data.name, toComplete) + // PARAM FIX - Gonna use the ID field, even though it's a hack + const paramcheck = selectedAction.parameters.find(param => param.name === "body") + if (paramcheck !== undefined) { + if (paramcheck["value_replace"] === undefined || paramcheck["value_replace"] === null) { + paramcheck["value_replace"] = [{ + "key": data.name, + "value": toComplete, + }] + + } else { + const subparamindex = paramcheck["value_replace"].findIndex(param => param.key === data.name) + if (subparamindex === -1) { + paramcheck["value_replace"].push({ + "key": data.name, + "value": toComplete, + }) + } else { + paramcheck["value_replace"][subparamindex]["value"] += toComplete + } + } + + selectedActionParameters[count]["value_replace"] = paramcheck + selectedAction.parameters[count]["value_replace"] = paramcheck + setSelectedAction(selectedAction) + setUpdate(Math.random()) + + setShowDropdown(false) + setMenuPosition(null) + return + } + } + + selectedActionParameters[count].value += toComplete + selectedAction.parameters[count].value = selectedActionParameters[count].value + setSelectedAction(selectedAction) + setUpdate(Math.random()) + + setShowDropdown(false) + setMenuPosition(null) + } + + const iconStyle = { + marginRight: 15, + } + + return ( + { + handleMenuClose() + }} + open={!!menuPosition} + style={{ + border: `2px solid #f85a3e`, + color: "white", + marginTop: 2, + }} + > + {actionlist.map(innerdata => { + const icon = innerdata.type === "action" ? : innerdata.type === "workflow_variable" || innerdata.type === "execution_variable" ? : + + const handleExecArgumentHover = (inside) => { + var exec_text_field = document.getElementById("execution_argument_input_field") + if (exec_text_field !== null) { + if (inside) { + exec_text_field.style.border = "2px solid #f85a3e" + } else { + exec_text_field.style.border = "" + } + } + + // Also doing arguments + if (workflow.triggers !== undefined && workflow.triggers !== null && workflow.triggers.length > 0) { + for (var key in workflow.triggers) { + const item = workflow.triggers[key] + + var node = cy.getElementById(item.id) + if (node.length > 0) { + if (inside) { + node.addClass('shuffle-hover-highlight') + } else { + node.removeClass('shuffle-hover-highlight') + } + } + + } + } + } + + const handleActionHover = (inside, actionId) => { + var node = cy.getElementById(actionId) + if (node.length > 0) { + if (inside) { + node.addClass('shuffle-hover-highlight') + } else { + node.removeClass('shuffle-hover-highlight') + } + } + } + + const handleMouseover = () => { + if (innerdata.type === "Execution Argument") { + handleExecArgumentHover(true) + } else if (innerdata.type === "action") { + handleActionHover(true, innerdata.id) + } + } + + const handleMouseOut = () => { + if (innerdata.type === "Execution Argument") { + handleExecArgumentHover(false) + } else if (innerdata.type === "action") { + handleActionHover(false, innerdata.id) + } + } + + var parsedPaths = [] + if (typeof(innerdata.example) === "object") { + parsedPaths = GetParsedPaths(innerdata.example, "") + } + + return ( + parsedPaths.length > 0 ? + + {icon} {innerdata.name} +
+ } + parentMenuOpen={!!menuPosition} + style={{backgroundColor: theme.palette.inputColor, color: "white", minWidth: 250,}} + onClick={() => { + handleItemClick([innerdata]) + }} + > + {parsedPaths.map((pathdata, index) => { + // FIXME: Should be recursive in here + const icon = pathdata.type === "value" ? : pathdata.type === "list" ? : + return ( + {}} + onClick={() => { + handleItemClick([innerdata, pathdata]) + }} + > + +
+ {icon} {pathdata.name} +
+ + + ) + + })} + + : + handleMouseover()} onMouseOut={() => {handleMouseOut()}} + onClick={() => { + handleItemClick([innerdata]) + }} + > + +
+ {icon} {innerdata.name} +
+ + + + ) + })} + + ) + } + + var itemColor = "#f85a3e" + if (!data.required) { + itemColor = "#ffeb3b" + } + + var tmpitem = data.name.valueOf() + if (data.name.startsWith("${") && data.name.endsWith("}")) { + tmpitem = tmpitem.slice(2, data.name.length-1) + } + + tmpitem = tmpitem.charAt(0).toUpperCase()+tmpitem.substring(1) + tmpitem = tmpitem.replaceAll("_", " ") + const description = data.description === undefined ? "" : data.description + + return ( +
+
+ + + {data.configuration === true ? + + { + setAuthenticationModalOpen(true) + }}/> + + : +
+ } +
+ + {tmpitem} + +
+ + {/*selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0 ? null : +
+ +
{ + e.preventDefault() + changeActionParameterVariant("STATIC_VALUE", count) + }}> + +
+ +  |  + +
{ + e.preventDefault() + changeActionParameterVariant("ACTION_RESULT", count) + }}> + +
+ +  |  + +
{ + e.preventDefault() + changeActionParameterVariant("WORKFLOW_VARIABLE", count) + }}> + +
+ +
+ */} + {selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0 && selectedActionParameters[count].required === true && selectedActionParameters[count].unique_toggled !== undefined ? null : +
+ +
{}}> + { + //console.log("CHECKED!: ", selectedActionParameters[count]) + selectedActionParameters[count].unique_toggled = !selectedActionParameters[count].unique_toggled + selectedAction.parameters[count].unique_toggled = selectedActionParameters[count].unique_toggled + setSelectedActionParameters(selectedActionParameters) + setSelectedAction(selectedAction) + setUpdate(Math.random()) + }} + name="requires_unique" + /> +
+ +
+ } +
+ {datafield} + {showDropdown && showDropdownNumber === count && data.variant === "STATIC_VALUE" && jsonList.length > 0 ? + + Autocomplete + + + : null} + {showDropdown && showDropdownNumber === count && data.variant === "STATIC_VALUE" && jsonList.length === 0 ? + + : null} + + +
+ )})} +
+ ) + } + return null + } + return ( +
+
+
+

{selectedAction.app_name.replaceAll("_", " ")}

+
+ { + console.log("FIND EXAMPLE RESULTS FOR ", selectedAction) + if (workflowExecutions.length > 0) { + // Look for the ID + const found = false + for (var key in workflowExecutions) { + if (workflowExecutions[key].results === undefined || workflowExecutions[key].results === null) { + continue + } + + var foundResult = workflowExecutions[key].results.find(result => result.action.id === selectedAction.id) + if (foundResult === undefined || foundResult === null) { + continue + } + + setSelectedResult(foundResult) + setCodeModalOpen(true) + break + } + } + }}> + + + + + + What are actions? + {selectedAction.errors !== null && selectedAction.errors.length > 0 ? +
+ Errors: {selectedAction.errors.join("\n")} +
+ : null + } + +
+
+
+ {selectedAction.id === workflow.start ? null : + + + + } + {selectedApp.versions !== null && selectedApp.versions !== undefined && selectedApp.versions.length > 0 ? + + : null } +
+
+ + + Name + + + {selectedApp.name !== undefined && selectedAction.authentication !== undefined && selectedAction.authentication.length === 0 && requiresAuthentication ? +
+ + + + + +
+ : null} + {selectedAction.authentication !== undefined && selectedAction.authentication.length > 0 ? +
+ Authentication +
+ + + {/* + + + curaction.authentication = authenticationOptions + if (curaction.selectedAuthentication === null || curaction.selectedAuthentication === undefined || curaction.selectedAuthentication.length === "") + */} + + { + setAuthenticationModalOpen(true) + }}> + + + +
+
+ : null} + {showEnvironment ? +
+ + Environment + + +
+ : null} + {workflow.execution_variables !== undefined && workflow.execution_variables !== null && workflow.execution_variables.length > 0 ? +
+ Set execution variable (optional) + +
+ : null} + +
+
+ Actions +
+ + {selectedAction.description !== undefined && selectedAction.description.length > 0 ? +
+ {selectedAction.description} +
: null} +
+ +
+
+
+ ) + } + + +export default ParsedAction; diff --git a/frontend/src/theme.js b/frontend/src/theme.js new file mode 100644 index 00000000..13b47d64 --- /dev/null +++ b/frontend/src/theme.js @@ -0,0 +1,57 @@ +import React from 'react'; +import { createMuiTheme } from '@material-ui/core/styles'; + +const theme = createMuiTheme({ + palette: { + primary: { + main: "#f85a3e" + }, + secondary: { + main: '#e8eaf6', + }, + text: { + secondary: "rgba(255,255,255,0.7)", + }, + surfaceColor: "#27292d", + inputColor: "#383B40", + borderRadius: 5, + textFieldStyle: { + backgroundColor: "#383B40", + borderRadius: 5, + }, + innerTextfieldStyle: { + color: "white", + minHeight: 50, + marginLeft: "5px", + maxWidth: "95%", + fontSize: "1em", + borderRadius: 5, + } + }, + typography: { + "fontFamily": `"Roboto", "Helvetica", "Arial", sans-serif`, + useNextVariants: true, + h1: { + fontSize: 40, + }, + h4: { + fontSize: 30, + fontWeight: 500, + }, + h6: { + fontSize: 22, + }, + body1: { + fontSize: 18, + }, + }, + overrides: { + MuiMenu: { + list: { + backgroundColor: "#383B40", + }, + }, + }, +}); + +export default theme; diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index b32fb4b2..ed8e2295 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -27,6 +27,7 @@ import { useAlert } from "react-alert"; import { validateJson } from "./Workflows.jsx"; import { GetParsedPaths } from "./Apps.jsx"; import ConfigureWorkflow from '../components/ConfigureWorkflow.jsx'; +import ParsedAction from '../components/ParsedAction.jsx'; const surfaceColor = "#27292D" const inputColor = "#383B40" @@ -87,7 +88,6 @@ const AngularWorkflow = (props) => { const { globalUrl, isLoggedIn, isLoaded, userdata } = props; const referenceUrl = globalUrl+"/api/v1/hooks/" const alert = useAlert() - const borderRadius = 5 const theme = useTheme(); const green = "#86c142" const yellow = "#FECC00" @@ -128,7 +128,7 @@ const AngularWorkflow = (props) => { const [rightSideBarOpen, setRightSideBarOpen] = React.useState(false) const [showSkippedActions, setShowSkippedActions] = React.useState(false) const [lastExecution, setLastExecution] = React.useState("") - const [configureWorkflowModalOpen, setConfigureWorkflowModalOpen] = React.useState(true) + const [configureWorkflowModalOpen, setConfigureWorkflowModalOpen] = React.useState(false) const [curpath, setCurpath] = useState(typeof window === 'undefined' || window.location === undefined ? "" : window.location.pathname) // 0 = normal, 1 = just done, 2 = normal @@ -1151,8 +1151,11 @@ const AngularWorkflow = (props) => { //console.log(responseJson) // Add error checks - if (!responseJson.public && (!responseJson.previously_saved || (!responseJson.is_valid || (responseJson.errors !== undefined || responseJson.errors !== null || responseJson.errors !== responseJson.errors.length > 0)))) { - setConfigureWorkflowModalOpen(true) + console.log(responseJson) + if (!responseJson.public) { + if ((!responseJson.previously_saved || (!responseJson.is_valid || (responseJson.errors !== undefined || responseJson.errors !== null || responseJson.errors !== responseJson.errors.length > 0)))) { + setConfigureWorkflowModalOpen(true) + } } }) .catch(error => { @@ -2127,7 +2130,7 @@ const AngularWorkflow = (props) => { const paperAppStyle = { - borderRadius: borderRadius, + borderRadius: theme.palette.borderRadius, minHeight: 100, maxHeight: 100, minWidth: "100%", @@ -2140,7 +2143,7 @@ const AngularWorkflow = (props) => { } const paperVariableStyle = { - borderRadius: borderRadius, + borderRadius: theme.palette.borderRadius, minHeight: 50, maxHeight: 50, minWidth: "100%", @@ -2629,6 +2632,7 @@ const AngularWorkflow = (props) => { const handleAppDrag = (e, app) => { const cycontainer = cy.container() + // Chrome lol //if (e.srcElement !== undefined && e.srcElement.localName === "canvas") { if (e.pageX > cycontainer.offsetLeft && e.pageX < cycontainer.offsetLeft+cycontainer.offsetWidth && e.pageY > cycontainer.offsetTop && e.pageY < cycontainer.offsetTop+cycontainer.offsetHeight) { @@ -2641,6 +2645,10 @@ const AngularWorkflow = (props) => { currentnode[0].renderedPosition("x", e.pageX-cycontainer.offsetLeft) currentnode[0].renderedPosition("y", e.pageY-cycontainer.offsetTop) } else{ + if (workflow.public) { + return + } + if (app.actions === undefined || app.actions === null || app.actions.length === 0) { alert.error("App "+app.name+" currently has no actions to perform. Please go to https://shuffler.io/apps to edit it.") return @@ -2669,13 +2677,14 @@ const AngularWorkflow = (props) => { setUpdate(Math.random()) */ + console.log("ENVS: ", environments) const newAppData = { app_name: app.name, app_version: app.app_version, app_id: app.id, sharing: app.sharing, private_id: app.private_id, - environment: environments === null ? "cloud" : environments[defaultEnvironmentIndex].Name, + environment: environments === null || environments === [] ? "cloud" : environments[defaultEnvironmentIndex].Name, errors: [], id_: newNodeId, _id_: newNodeId, @@ -2865,7 +2874,7 @@ const AngularWorkflow = (props) => { {setHover(true)}} onMouseOut={() => {setHover(false)}}> - {newAppname} + {newAppname} @@ -2900,7 +2909,7 @@ const AngularWorkflow = (props) => {
{ // Dropdown -> static, action, local env, global env // VALUE (JSON) // {data.name}, {data.description}, {data.required}, {data.schema.type} - const AppActionArguments = (props) => { - const [selectedActionParameters, setSelectedActionParameters] = React.useState([]) - const [selectedVariableParameter, setSelectedVariableParameter] = React.useState("") - const [actionlist, setActionlist] = React.useState([]) - const [jsonList, setJsonList] = React.useState([]) - const [showDropdown, setShowDropdown] = React.useState(false) - const [showDropdownNumber, setShowDropdownNumber] = React.useState(0) - const [showAutocomplete, setShowAutocomplete] = React.useState(false) - const [menuPosition, setMenuPosition] = useState(null) - - useEffect(() => { - if (selectedActionParameters !== null && selectedActionParameters.length === 0) { - if (selectedAction.parameters !== null && selectedAction.parameters.length > 0) { - setSelectedActionParameters(selectedAction.parameters) - } - } - - if ((selectedVariableParameter === null || selectedVariableParameter === undefined) && (workflow.workflow_variables !== null && workflow.workflow_variables.length > 0)) { - // FIXME - this is the bad thing - setSelectedVariableParameter(workflow.workflow_variables[0].name) - } - - if (actionlist.length === 0) { - // FIXME: Have previous execution values in here - actionlist.push({"type": "Execution Argument", "name": "Execution Argument", "value": "$exec", "highlight": "exec", "autocomplete": "exec", "example": "hello"}) - if (workflow.workflow_variables !== null && workflow.workflow_variables !== undefined && workflow.workflow_variables.length > 0) { - for (var key in workflow.workflow_variables) { - const item = workflow.workflow_variables[key] - actionlist.push({"type": "workflow_variable", "name": item.name, "value": item.value, "id": item.id, "autocomplete": `${item.name.split(" ").join("_")}`, "example": item.value}) - } - } - - // FIXME: Add values from previous executions if they exist - if (workflow.execution_variables !== null && workflow.execution_variables !== undefined && workflow.execution_variables.length > 0) { - for (var key in workflow.execution_variables) { - const item = workflow.execution_variables[key] - actionlist.push({"type": "execution_variable", "name": item.name, "value": item.value, "id": item.id, "autocomplete": `${item.name.split(" ").join("_")}`, "example": ""}) - } - } - - // Loops parent nodes' old results to fix autocomplete - var parents = getParents(selectedAction) - if (parents.length > 1) { - for (var key in parents) { - const item = parents[key] - if (item.label === "Execution Argument") { - continue - } - - var exampledata = item.example === undefined ? "" : item.example - // Find previous execution and their variables - //exampledata === "" && - if (workflowExecutions.length > 0) { - // Look for the ID - const found = false - for (var key in workflowExecutions) { - if (workflowExecutions[key].results === undefined || workflowExecutions[key].results === null) { - continue - } - - var foundResult = workflowExecutions[key].results.find(result => result.action.id === item.id) - if (foundResult === undefined) { - continue - } - - foundResult.result = foundResult.result.trim() - foundResult.result = foundResult.result.split(" None").join(" \"None\"") - foundResult.result = foundResult.result.split(" False").join(" false") - foundResult.result = foundResult.result.split(" True").join(" true") - - var jsonvalid = true - try { - const tmp = String(JSON.parse(foundResult.result)) - if (!foundResult.result.includes("{") && !foundResult.result.includes("[")) { - jsonvalid = false - } - } catch (e) { - try { - foundResult.result = foundResult.result.split("\'").join("\"") - const tmp = String(JSON.parse(foundResult.result)) - if (!foundResult.result.includes("{") && !foundResult.result.includes("[")) { - jsonvalid = false - } - } catch (e) { - jsonvalid = false - } - } - - // Finds the FIRST json only - if (jsonvalid) { - exampledata = JSON.parse(foundResult.result) - break - } - //else { - // console.log("Invalid JSON: ", foundResult.result) - //} - } - } - - // 1. Take - const actionvalue = {"type": "action", "id": item.id, "name": item.label, "autocomplete": `${item.label.split(" ").join("_")}`, "example": exampledata} - actionlist.push(actionvalue) - } - } - - setActionlist(actionlist) - } - }) - - const changeActionParameter = (event, count, data) => { - if (data.name.startsWith("${") && data.name.endsWith("}")) { - // PARAM FIX - Gonna use the ID field, even though it's a hack - const paramcheck = selectedAction.parameters.find(param => param.name === "body") - if (paramcheck !== undefined) { - if (paramcheck["value_replace"] === undefined || paramcheck["value_replace"] === null) { - paramcheck["value_replace"] = [{ - "key": data.name, - "value": event.target.value, - }] - - } else { - const subparamindex = paramcheck["value_replace"].findIndex(param => param.key === data.name) - if (subparamindex === -1) { - paramcheck["value_replace"].push({ - "key": data.name, - "value": event.target.value, - }) - } else { - paramcheck["value_replace"][subparamindex]["value"] = event.target.value - } - } - //console.log("PARAM: ", paramcheck) - - selectedActionParameters[count]["value_replace"] = paramcheck - selectedAction.parameters[count]["value_replace"] = paramcheck - setSelectedAction(selectedAction) - //setUpdate(Math.random()) - return - } - } - - if (event.target.value[event.target.value.length-1] === "$") { - if (!showDropdown) { - setShowAutocomplete(false) - setShowDropdown(true) - setShowDropdownNumber(count) - } - } else { - if (showDropdown) { - setShowDropdown(false) - } - } - - // bad detection mechanism probably - if (event.target.value[event.target.value.length-1] === "." && actionlist.length > 0) { - console.log("GET THE LAST ARGUMENT FOR NODE!") - // THIS IS AN EXAMPLE OF SHOWING IT - /* - const inputdata = {"data": "1.2.3.4", "dataType": "4.5.6.6"} - setJsonList(GetParsedPaths(inputdata, "")) - if (!showDropdown) { - setShowAutocomplete(false) - setShowDropdown(true) - setShowDropdownNumber(count) - } - console.log(jsonList) - */ - - // Search for the item backwards - // 1. Reverse search backwards from . -> $ - // 2. Search the actionlist for the item - // 3. Find the data for the specific item - - var curstring = "" - var record = false - for (var key in selectedActionParameters[count].value) { - const item = selectedActionParameters[count].value[key] - if (record) { - curstring += item - } - - if (item === "$") { - record = true - curstring = "" - } - } - - //console.log("CURSTRING: ", curstring) - if (curstring.length > 0 && actionlist !== null) { - // Search back in the action list - curstring = curstring.split(" ").join("_").toLowerCase() - var actionItem = actionlist.find(data => data.autocomplete.split(" ").join("_").toLowerCase() === curstring) - if (actionItem !== undefined) { - console.log("Found item: ", actionItem) - - //actionItem.example = actionItem.example.trim() - //actionItem.example = actionItem.example.split(" None").join(" \"None\"") - //actionItem.example = actionItem.example.split("\'").join("\"") - - var jsonvalid = true - try { - const tmp = String(JSON.parse(actionItem.example)) - if (!actionItem.example.includes("{") && !actionItem.example.includes("[")) { - jsonvalid = false - } - } catch (e) { - jsonvalid = false - } - - if (jsonvalid) { - setJsonList(GetParsedPaths(JSON.parse(actionItem.example), "")) - - if (!showDropdown) { - setShowAutocomplete(false) - setShowDropdown(true) - setShowDropdownNumber(count) - } - } - } - } - } else { - if (jsonList.length > 0) { - setJsonList([]) - } - } - - selectedActionParameters[count].value = event.target.value - selectedAction.parameters[count].value = event.target.value - setSelectedAction(selectedAction) - //setUpdate(Math.random()) - //setUpdate(event.target.value) - } - - const changeActionParameterVariable = (fieldvalue, count) => { - //console.log("CALLED THIS ONE WITH VALUE!", fieldvalue) - //if (selectedVariableParameter === fieldvalue) { - // return - //} - - setSelectedVariableParameter(fieldvalue) - - selectedActionParameters[count].action_field = fieldvalue - selectedAction.parameters = selectedActionParameters - - setSelectedApp(selectedApp) - setSelectedAction(selectedAction) - setUpdate(fieldvalue) - } - - // Sets ACTION_RESULT things - const changeActionParameterActionResult = (fieldvalue, count) => { - //cy.nodes().forEach(function( ele ) { - // if (ele.data()["label"] === fieldvalue) { - // selectedActionParameters[count].action_field = ele.id() - // return - // } - //}); - - selectedActionParameters[count].action_field = fieldvalue - selectedAction.parameters = selectedActionParameters - - // FIXME - check if startnode - - // Set value - setSelectedApp(selectedApp) - - setSelectedAction(selectedAction) - setUpdate(Math.random()) - } - - const changeActionParameterVariant = (data, count) => { - selectedActionParameters[count].variant = data - selectedActionParameters[count].value = "" - - if (data === "ACTION_RESULT") { - var parents = getParents(selectedAction) - if (parents.length > 0) { - selectedActionParameters[count].action_field = parents[0].label - } else { - selectedActionParameters[count].action_field = "" - } - } else if (data === "WORKFLOW_VARIABLE") { - if (workflow.workflow_variables !== null && workflow.workflow_variables !== undefined && workflow.workflow_variables.length > 0) { - selectedActionParameters[count].action_field = workflow.workflow_variables[0].name - } - } - - selectedAction.parameters = selectedActionParameters - - // This is a stupid workaround to make it refresh rofl - setSelectedAction({}) - setSelectedTrigger({}) - setSelectedApp({}) - setSelectedEdge({}) - // FIXME - check if startnode - - // Set value - setSelectedApp(selectedApp) - setSelectedAction(selectedAction) - setUpdate(Math.random()) - } - - // FIXME: Issue #40 - selectedActionParameters not reset - if (Object.getOwnPropertyNames(selectedAction).length > 0 && selectedActionParameters.length > 0) { - return ( -
- Parameters - {selectedActionParameters.map((data, count) => { - if (data.variant === "") { - data.variant = "STATIC_VALUE" - } - - - // selectedAction.selectedAuthentication = e.target.value - // selectedAction.authentication_id = e.target.value.id - if (!selectedAction.auth_not_required && selectedAction.selectedAuthentication !== undefined && selectedAction.selectedAuthentication.fields !== undefined && selectedAction.selectedAuthentication.fields[data.name] !== undefined) { - // This sets the placeholder in the frontend. (Replaced in backend) - selectedActionParameters[count].value = selectedAction.selectedAuthentication.fields[data.name] - selectedAction.parameters[count].value = selectedAction.selectedAuthentication.fields[data.name] - setSelectedAction(selectedAction) - //setUpdate(Math.random()) - - return null - } - - var staticcolor = "inherit" - var actioncolor = "inherit" - var varcolor = "inherit" - var multiline - if (data.multiline !== undefined && data.multiline !== null && data.multiline === true) { - multiline = true - } - - if (data.value !== undefined && data.value !== null && data.value.startsWith("{") && data.value.endsWith("}")) { - multiline = true - } - - var placeholder = "Static value" - if (data.example !== undefined && data.example !== null && data.example.length > 0) { - placeholder = data.example - } - - if (data.name.startsWith("${") && data.name.endsWith("}")) { - const paramcheck = selectedAction.parameters.find(param => param.name === "body") - if (paramcheck !== undefined && paramcheck !== null) { - if (paramcheck["value_replace"] !== undefined && paramcheck["value_replace"] !== null) { - //console.log("IN THE VALUE REPLACE: ", paramcheck["value_replace"]) - const subparamindex = paramcheck["value_replace"].findIndex(param => param.key === data.name) - if (subparamindex !== -1) { - data.value = paramcheck["value_replace"][subparamindex]["value"] - } - } - } - } - - var disabled = false - var rows = "5" - var openApiHelperText = "This is an OpenAPI specific field" - if (selectedApp.generated && selectedApp.activated && data.name === "body") { - const regex = /\${(\w+)}/g - const found = placeholder.match(regex) - if (found === null) { - //setExtraBodyFields([]) - } else { - rows = "1" - disabled = true - openApiHelperText = "OpenAPI spec: fill the following fields." - //console.log("SHOULD ADD TO selectedActionParameters!: ", found, selectedActionParameters) - var changed = false - for (var specKey in found) { - const tmpitem = found[specKey] - var skip = false - for (var innerkey in selectedActionParameters) { - if (selectedActionParameters[innerkey].name === tmpitem) { - skip = true - break - } - } - - if (skip) { - //console.log("SKIPPING ", tmpitem) - continue - } - - changed = true - selectedActionParameters.push({ - action_field: "", - configuration: false, - description: "Generated by OpenAPI body example", - example: "", - id: "", - multiline: false, - name: tmpitem, - options: null, - required: false, - schema: {type: "string"}, - skip_multicheck: false, - tags: null, - value: "", - variant: "STATIC_VALUE", - }) - } - - if (changed) { - setSelectedActionParameters(selectedActionParameters) - } - - return - } - } - - //console.log("Data: ", data) - var datafield = - - - { - setMenuPosition({ - top: event.pageY+10, - left: event.pageX+10, - }) - setShowDropdownNumber(count) - setShowDropdown(true) - setShowAutocomplete(true) - }}/> - - - ) - }} - fullWidth - multiline={multiline} - rows={rows} - color="primary" - defaultValue={data.value} - type={placeholder.includes("***") ? "password" : "text"} - placeholder={placeholder} - onChange={(event) => { - changeActionParameter(event, count, data) - }} - helperText={selectedApp.generated && selectedApp.activated && data.name === "body" ? - - {openApiHelperText} - - : - data.name.startsWith("${") && data.name.endsWith("}") ? - - OpenAPI helperfield - - : - null - } - onBlur={(event) => { - // Super basic check - //if (event.target.value.startsWith("{")) { - // console.log("VALIDATING JSON") - // try { - // JSON.parse(event.target.value) - // } catch (e) { - // alert.error("Failed to parse json: ", e) - // } - //} - }} - /> - - if (selectedActionParameters[count].schema !== undefined && selectedActionParameters[count].schema !== null && selectedActionParameters[count].schema.type === "file") { - datafield = - - - { - setMenuPosition({ - top: event.pageY+10, - left: event.pageX+10, - }) - setShowDropdownNumber(count) - setShowDropdown(true) - setShowAutocomplete(true) - }}/> - - - ) - }} - fullWidth - multiline={multiline} - rows="5" - color="primary" - defaultValue={data.value} - type={"text"} - placeholder={"The file ID to get"} - onChange={(event) => { - changeActionParameter(event, count, data) - }} - onBlur={(event) => { - }} - /> - //const fileId = "6daabec1-892b-469c-b603-c902e47223a9" - //datafield = `SHOW FILES FROM OTHER NODES? Filename: ${selectedActionParameters[count].value}` - /* - if (selectedActionParameters[count].value != fileId) { - changeActionParameter(fileId, count, data) - setUpdate(Math.random()) - - } - */ - } else if (selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0) { - if (selectedActionParameters[count].value === "" && selectedActionParameters[count].required) { - // Rofl, dirty workaround :) - const e = { - target: { - value: selectedActionParameters[count].options[0], - } - } - - changeActionParameter(e, count, data) - } - - datafield = - - - } else if (data.variant === "STATIC_VALUE") { - staticcolor = "#f85a3e" - } else if (data.variant === "ACTION_RESULT") { - // Gets the parents of the current node - var parents = getParents(selectedAction) - actioncolor = "#f85a3e" - // set the datafield - //var datafieldvalue = "Error: No parents. Action not eligible" - //if (parents.length > 0) { - // datafieldvalue = parents[0].label - //} - const fixedActionText = selectedActionParameters[count].value - - datafield = -
- - Example: $.body will get "data" from {'{"body": "data"}'}
} - placeholder="Action variable ($.)" - onChange={(event) => { - changeActionParameter(event, count, data) - }} - /> -
- - } else if (data.variant === "WORKFLOW_VARIABLE") { - varcolor = "#f85a3e" - if ((workflow.workflow_variables === null || workflow.workflow_variables === undefined || workflow.workflow_variables.length === 0) && (workflow.execution_variables === null || workflow.execution_variables === undefined || workflow.execution_variables.length === 0)) { - setCurrentView(2) - datafield = -
-
- Looks like you don't have any variables yet. -
-
- -
-
- } else { - // FIXME - this is a shitty solution that needs re-renders all the time - datafield = - - } - } - - // Shows nested list of nodes > their JSON lists - const ActionlistWrapper = (props) => { - - const handleMenuClose = () => { - setShowAutocomplete(false) - - if (!selectedActionParameters[count].value[selectedActionParameters[count].value.length-1] === "$") { - setShowDropdown(false) - } - - setUpdate(Math.random()) - setMenuPosition(null) - } - - const handleItemClick = (values) => { - if (values === undefined || values === null || values.length === 0) { - return - } - - var toComplete = selectedActionParameters[count].value.trim().endsWith("$") ? values[0].autocomplete : "$"+values[0].autocomplete - for (var key in values) { - if (key == 0 || values[key].autocomplete.length === 0) { - continue - } - - toComplete += values[key].autocomplete - } - - // Handles the fields under OpenAPI body to be parsed. - if (data.name.startsWith("${") && data.name.endsWith("}")) { - console.log("INSIDE VALUE REPLACE: ", data.name, toComplete) - // PARAM FIX - Gonna use the ID field, even though it's a hack - const paramcheck = selectedAction.parameters.find(param => param.name === "body") - if (paramcheck !== undefined) { - if (paramcheck["value_replace"] === undefined || paramcheck["value_replace"] === null) { - paramcheck["value_replace"] = [{ - "key": data.name, - "value": toComplete, - }] - - } else { - const subparamindex = paramcheck["value_replace"].findIndex(param => param.key === data.name) - if (subparamindex === -1) { - paramcheck["value_replace"].push({ - "key": data.name, - "value": toComplete, - }) - } else { - paramcheck["value_replace"][subparamindex]["value"] += toComplete - } - } - - selectedActionParameters[count]["value_replace"] = paramcheck - selectedAction.parameters[count]["value_replace"] = paramcheck - setSelectedAction(selectedAction) - setUpdate(Math.random()) - - setShowDropdown(false) - setMenuPosition(null) - return - } - } - - selectedActionParameters[count].value += toComplete - selectedAction.parameters[count].value = selectedActionParameters[count].value - setSelectedAction(selectedAction) - setUpdate(Math.random()) - - setShowDropdown(false) - setMenuPosition(null) - } - - const iconStyle = { - marginRight: 15, - } - - return ( - { - handleMenuClose() - }} - open={!!menuPosition} - style={{ - border: `2px solid #f85a3e`, - color: "white", - marginTop: 2, - }} - > - {actionlist.map(innerdata => { - const icon = innerdata.type === "action" ? : innerdata.type === "workflow_variable" || innerdata.type === "execution_variable" ? : - - const handleExecArgumentHover = (inside) => { - var exec_text_field = document.getElementById("execution_argument_input_field") - if (exec_text_field !== null) { - if (inside) { - exec_text_field.style.border = "2px solid #f85a3e" - } else { - exec_text_field.style.border = "" - } - } - - // Also doing arguments - if (workflow.triggers !== undefined && workflow.triggers !== null && workflow.triggers.length > 0) { - for (var key in workflow.triggers) { - const item = workflow.triggers[key] - - var node = cy.getElementById(item.id) - if (node.length > 0) { - if (inside) { - node.addClass('shuffle-hover-highlight') - } else { - node.removeClass('shuffle-hover-highlight') - } - } - - } - } - } - - const handleActionHover = (inside, actionId) => { - var node = cy.getElementById(actionId) - if (node.length > 0) { - if (inside) { - node.addClass('shuffle-hover-highlight') - } else { - node.removeClass('shuffle-hover-highlight') - } - } - } - - const handleMouseover = () => { - if (innerdata.type === "Execution Argument") { - handleExecArgumentHover(true) - } else if (innerdata.type === "action") { - handleActionHover(true, innerdata.id) - } - } - - const handleMouseOut = () => { - if (innerdata.type === "Execution Argument") { - handleExecArgumentHover(false) - } else if (innerdata.type === "action") { - handleActionHover(false, innerdata.id) - } - } - - var parsedPaths = [] - if (typeof(innerdata.example) === "object") { - parsedPaths = GetParsedPaths(innerdata.example, "") - } - - return ( - parsedPaths.length > 0 ? - - {icon} {innerdata.name} -
- } - parentMenuOpen={!!menuPosition} - style={{backgroundColor: inputColor, color: "white", minWidth: 250,}} - onClick={() => { - handleItemClick([innerdata]) - }} - > - {parsedPaths.map((pathdata, index) => { - // FIXME: Should be recursive in here - const icon = pathdata.type === "value" ? : pathdata.type === "list" ? : - return ( - {}} - onClick={() => { - handleItemClick([innerdata, pathdata]) - }} - > - -
- {icon} {pathdata.name} -
- - - ) - - })} - - : - handleMouseover()} onMouseOut={() => {handleMouseOut()}} - onClick={() => { - handleItemClick([innerdata]) - }} - > - -
- {icon} {innerdata.name} -
- - - - ) - })} - - ) - } - - var itemColor = "#f85a3e" - if (!data.required) { - itemColor = "#ffeb3b" - } - - var tmpitem = data.name.valueOf() - if (data.name.startsWith("${") && data.name.endsWith("}")) { - tmpitem = tmpitem.slice(2, data.name.length-1) - } - - tmpitem = tmpitem.charAt(0).toUpperCase()+tmpitem.substring(1) - tmpitem = tmpitem.replaceAll("_", " ") - const description = data.description === undefined ? "" : data.description - - return ( -
-
- - - {data.configuration === true ? - - { - setAuthenticationModalOpen(true) - }}/> - - : -
- } -
- - {tmpitem} - -
- - {/*selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0 ? null : -
- -
{ - e.preventDefault() - changeActionParameterVariant("STATIC_VALUE", count) - }}> - -
- -  |  - -
{ - e.preventDefault() - changeActionParameterVariant("ACTION_RESULT", count) - }}> - -
- -  |  - -
{ - e.preventDefault() - changeActionParameterVariant("WORKFLOW_VARIABLE", count) - }}> - -
- -
- */} - {selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0 && selectedActionParameters[count].required === true && selectedActionParameters[count].unique_toggled !== undefined ? null : -
- -
{}}> - { - //console.log("CHECKED!: ", selectedActionParameters[count]) - selectedActionParameters[count].unique_toggled = !selectedActionParameters[count].unique_toggled - selectedAction.parameters[count].unique_toggled = selectedActionParameters[count].unique_toggled - setSelectedActionParameters(selectedActionParameters) - setSelectedAction(selectedAction) - setUpdate(Math.random()) - }} - name="requires_unique" - /> -
- -
- } -
- {datafield} - {showDropdown && showDropdownNumber === count && data.variant === "STATIC_VALUE" && jsonList.length > 0 ? - - Autocomplete - - - : null} - {showDropdown && showDropdownNumber === count && data.variant === "STATIC_VALUE" && jsonList.length === 0 ? - - : null} - - -
- )})} -
- ) - } - return null - } + //height: "100%", const appApiViewStyle = { @@ -4148,49 +3124,7 @@ const AngularWorkflow = (props) => { overflow: "auto", } - const defineStartnode = () => { - var oldstartnode = cy.getElementById(workflow.start) - if (oldstartnode.length > 0) { - oldstartnode[0].data("isStartNode", false) - var oldnodecnt = workflow.actions.findIndex(a => a.id === workflow.start) - if (workflow.actions[oldnodecnt] !== undefined) { - workflow.actions[oldnodecnt].isStartNode = false - } - } - - var newstartnode = cy.getElementById(selectedAction.id) - if (newstartnode.length > 0) { - newstartnode[0].data("isStartNode", true) - var newnodecnt = workflow.actions.findIndex(a => a.id === selectedAction.id) - console.log("NEW NODE CNT: ", newnodecnt) - if (workflow.actions[newnodecnt] !== undefined) { - workflow.actions[newnodecnt].isStartNode = true - console.log(workflow.actions[newnodecnt]) - } - } - - // Find branches with triggers as source nodes - // Move these targets to be the new node - // Set arrows pointing to new startnode with errors - //for (var key in workflow.branches) { - // var item = workflow.branches[key] - // if (item.destination_id === oldstartnode[0].data()["id"]) { - // var curbranch = cy.getElementById(item.id) - // if (curbranch.length > 0) { - // //console.log(curbranch[0].data()) - // //curbranch[0].data("target", selectedAction.id) - // //curbranch[0].data("hasErrors", true) - // //workflow.branches[key].destination_id = selectedAction.id - // //console.log(curbranch[0].data()) - // } - // } - //} - - setUpdate("start_node"+selectedAction.id) - workflow.start = selectedAction.id - setWorkflow(workflow) - //setStartNode(selectedAction.id) - } + function sortByKey(array, key) { if (array === undefined) { @@ -4226,353 +3160,43 @@ const AngularWorkflow = (props) => { zIndex: 1000, } - const textFieldStyle = { - backgroundColor: inputColor, - borderRadius: borderRadius, - } - - const getApp = (appId, setApp) => { - fetch(globalUrl+"/api/v1/apps/"+appId+"/config?openapi=false", { - headers: { - 'Accept': 'application/json', - }, - credentials: "include", - }) - .then((response) => { - if (response.status === 200) { - //alert.success("Successfully GOT app "+appId) - } else { - alert.error("Failed getting app") - } - - return response.json() - }) - .then((responseJson) => { - if (setApp && responseJson.actions !== undefined && responseJson.actions !== null) { - if (selectedApp.versions !== undefined && selectedApp.versions !== null) { - responseJson.versions = selectedApp.versions - } - - if (selectedApp.loop_versions !== undefined && selectedApp.loop_versions !== null) { - responseJson.loop_versions = selectedApp.loop_versions - } - - var foundAction = responseJson.actions.find(action => action.name === selectedAction.name) - if (foundAction !== null && foundAction !== undefined) { - for (var paramkey in foundAction.parameters) { - const param = foundAction.parameters[paramkey] - - const foundParam = selectedAction.parameters.find(item => item.name === param.name) - if (foundParam === undefined) { - //console.log("COULDNT find Param: ", param) - } else { - foundAction.parameters[paramkey] = foundParam - } - } - } else { - alert.error("Couldn't find action "+selectedAction.name) - } - - // Updating params for the new action - selectedAction.parameters = foundAction.parameters - selectedAction.app_id = appId - selectedAction.app_version = responseJson.app_version - - setSelectedAction(selectedAction) - setSelectedApp(responseJson) - } - }) - .catch(error => { - alert.error(error.toString()) - }); - } - - const innerTextfieldStyle = { - color: "white", - minHeight: 50, - marginLeft: "5px", - maxWidth: "95%", - fontSize: "1em", - borderRadius: borderRadius, - } - const appApiView = Object.getOwnPropertyNames(selectedAction).length > 0 ? -
-
-
-

{selectedAction.app_name.replaceAll("_", " ")}

-
- { - console.log("FIND EXAMPLE RESULTS FOR ", selectedAction) - if (workflowExecutions.length > 0) { - // Look for the ID - const found = false - for (var key in workflowExecutions) { - if (workflowExecutions[key].results === undefined || workflowExecutions[key].results === null) { - continue - } - - var foundResult = workflowExecutions[key].results.find(result => result.action.id === selectedAction.id) - if (foundResult === undefined || foundResult === null) { - continue - } - - setSelectedResult(foundResult) - setCodeModalOpen(true) - break - } - } - }}> - - - - - - What are actions? - {selectedAction.errors !== null && selectedAction.errors.length > 0 ? -
- Errors: {selectedAction.errors.join("\n")} -
- : null - } - -
-
-
- {selectedAction.id === workflow.start ? null : - - - - } - {selectedApp.versions !== null && selectedApp.versions !== undefined && selectedApp.versions.length > 0 ? - - : null } -
-
- - - Name - - - {selectedApp.name !== undefined && selectedAction.authentication !== undefined && selectedAction.authentication.length === 0 && requiresAuthentication ? -
- - - - - -
- : null} - {selectedAction.authentication !== undefined && selectedAction.authentication.length > 0 ? -
- Authentication -
- - - {/* - - - curaction.authentication = authenticationOptions - if (curaction.selectedAuthentication === null || curaction.selectedAuthentication === undefined || curaction.selectedAuthentication.length === "") - */} - - { - setAuthenticationModalOpen(true) - }}> - - - -
-
- : null} - {showEnvironment ? -
- - Environment - - -
- : null} - {workflow.execution_variables !== undefined && workflow.execution_variables !== null && workflow.execution_variables.length > 0 ? -
- Set execution variable (optional) - -
- : null} - -
-
- Actions -
- - {selectedAction.description !== undefined && selectedAction.description.length > 0 ? -
- {selectedAction.description} -
: null} -
- - -
-
- -
-
-
- : null + + + : + null const setTriggerFolderWrapperMulti = event => { const { options } = event.target @@ -4763,7 +3387,7 @@ const AngularWorkflow = (props) => { var datafield = { Name
{
Environment: { Name
{ { { Name { { var copyText = document.getElementById("webhook_uri_field"); @@ -6224,7 +4848,7 @@ const AngularWorkflow = (props) => { Name {
Environment: {
{ {workflow.triggers[selectedTriggerIndex].parameters[2] !== undefined && workflow.triggers[selectedTriggerIndex].parameters[2].value.includes("email") ? { } {workflow.triggers[selectedTriggerIndex].parameters[2] !== undefined && workflow.triggers[selectedTriggerIndex].parameters[2].value.includes("sms") ? { Name { { { { - - - - - - - - - - - - - - - {workflow.public ? @@ -6830,6 +5428,32 @@ const AngularWorkflow = (props) => { : null} + + + + + + + + + + + + + + + {/* */} {workflow.configuration !== null && workflow.configuration !== undefined && workflow.configuration.exit_on_error !== undefined ? : null} @@ -7942,7 +6566,7 @@ const AngularWorkflow = (props) => {
Name - what is this used for? { {data.name} {
The API endpoint to use (URL) - predefined in the app Date: Tue, 6 Apr 2021 22:00:24 +0200 Subject: [PATCH 10/96] Fixed action parser view for multiple locations --- frontend/src/components/ParsedAction.jsx | 536 +++++++++++------------ frontend/src/views/AngularWorkflow.jsx | 44 +- frontend/src/views/AppCreator.jsx | 387 ++++++++-------- frontend/src/views/Apps.jsx | 16 +- 4 files changed, 485 insertions(+), 498 deletions(-) diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index 56103e95..b4eabc2a 100644 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -1,18 +1,65 @@ import React, {useState, useEffect, useLayoutEffect} from 'react'; -import { useTheme } from '@material-ui/core/styles'; import { GetParsedPaths } from "../views/Apps.jsx"; +import { sortByKey } from "../views/AngularWorkflow.jsx"; +import { useTheme } from '@material-ui/core/styles'; import NestedMenuItem from "material-ui-nested-menu-item"; import {TextField, Drawer, Button, Paper, Grid, Tabs, InputAdornment, Tab, ButtonBase, Tooltip, Select, MenuItem, Divider, Dialog, Modal, DialogActions, DialogTitle, InputLabel, DialogContent, FormControl, IconButton, Menu, Input, FormGroup, FormControlLabel, Typography, Checkbox, Breadcrumbs, CircularProgress, Switch, Fade} from '@material-ui/core'; import {GetApp as GetAppIcon, Search as SearchIcon, ArrowUpward as ArrowUpwardIcon, Visibility as VisibilityIcon, Done as DoneIcon, Close as CloseIcon, Error as ErrorIcon, FindReplace as FindreplaceIcon, ArrowLeft as ArrowLeftIcon, Cached as CachedIcon, DirectionsRun as DirectionsRunIcon, Add as AddIcon, Polymer as PolymerIcon, FormatListNumbered as FormatListNumberedIcon, Create as CreateIcon, PlayArrow as PlayArrowIcon, AspectRatio as AspectRatioIcon, MoreVert as MoreVertIcon, Apps as AppsIcon, Schedule as ScheduleIcon, FavoriteBorder as FavoriteBorderIcon, Pause as PauseIcon, Delete as DeleteIcon, AddCircleOutline as AddCircleOutlineIcon, Save as SaveIcon, KeyboardArrowLeft as KeyboardArrowLeftIcon, KeyboardArrowRight as KeyboardArrowRightIcon, ArrowBack as ArrowBackIcon, Settings as SettingsIcon, LockOpen as LockOpenIcon, ExpandMore as ExpandMoreIcon, VpnKey as VpnKeyIcon} from '@material-ui/icons'; - - const ParsedAction = (props) => { - const {workflow, setWorkflow, setAction, setSelectedAction, setUpdate, appActionArguments, selectedApp, workflowExecutions, setSelectedResult, selectedAction, getParents, setSelectedApp, setSelectedTrigger, setSelectedEdge, setCurrentView, cy, setAuthenticationModalOpen,setVariablesModalOpen, setLastSaved, setCodeModalOpen, selectedNameChange, rightsidebarStyle, showEnvironment, selectedActionEnvironment, environments, setNewSelectedAction, sortByKey, appApiViewStyle, globalUrl, setSelectedActionEnvironment, requiresAuthentication } = props + const {workflow, setWorkflow, setAction, setSelectedAction, setUpdate, appActionArguments, selectedApp, workflowExecutions, setSelectedResult, selectedAction, setSelectedApp, setSelectedTrigger, setSelectedEdge, setCurrentView, cy, setAuthenticationModalOpen,setVariablesModalOpen, setCodeModalOpen, selectedNameChange, rightsidebarStyle, showEnvironment, selectedActionEnvironment, environments, setNewSelectedAction, appApiViewStyle, globalUrl, setSelectedActionEnvironment, requiresAuthentication, hideExtraTypes } = props const theme = useTheme(); + const getParents = (action) => { + if (cy === undefined) { + return [] + } + + var allkeys = [action.id] + var handled = [] + var results = [] + + while(true) { + for (var key in allkeys) { + var currentnode = cy.getElementById(allkeys[key]) + if (handled.includes(currentnode.data().id)) { + continue + } else { + // Get the name / label here too? + handled.push(currentnode.data().id) + results.push(currentnode.data()) + } + + if (currentnode.length === 0) { + continue + } + + const incomingEdges = currentnode.incomers('edge') + if (incomingEdges.length === 0) { + continue + } + + for (var i = 0; i < incomingEdges.length; i++) { + var tmp = incomingEdges[i] + if (!allkeys.includes(tmp.data().source)) { + allkeys.push(tmp.data().source) + } + } + } + if (results.length === allkeys.length) { + break + } + } + + // Remove self + results = results.filter(data => data.id !== action.id) + results = results.filter(data => data.type !== "TRIGGER") + results.push({"label": "Execution Argument", "type": "INTERNAL"}) + return results + } + const getApp = (appId, setApp) => { fetch(globalUrl+"/api/v1/apps/"+appId+"/config?openapi=false", { @@ -71,6 +118,10 @@ const ParsedAction = (props) => { } const defineStartnode = () => { + if (cy === undefined) { + return + } + var oldstartnode = cy.getElementById(workflow.start) if (oldstartnode.length > 0) { oldstartnode[0].data("isStartNode", false) @@ -405,9 +456,12 @@ const ParsedAction = (props) => { // This is a stupid workaround to make it refresh rofl setSelectedAction({}) - setSelectedTrigger({}) - setSelectedApp({}) - setSelectedEdge({}) + + if (setSelectedTrigger !== undefined) { + setSelectedTrigger({}) + setSelectedApp({}) + setSelectedEdge({}) + } // FIXME - check if startnode // Set value @@ -419,7 +473,7 @@ const ParsedAction = (props) => { // FIXME: Issue #40 - selectedActionParameters not reset if (Object.getOwnPropertyNames(selectedAction).length > 0 && selectedActionParameters.length > 0) { return ( -
+
Parameters {selectedActionParameters.map((data, count) => { if (data.variant === "") { @@ -539,19 +593,21 @@ const ParsedAction = (props) => { fontSize: "1em", }, endAdornment: ( - - - { - setMenuPosition({ - top: event.pageY+10, - left: event.pageX+10, - }) - setShowDropdownNumber(count) - setShowDropdown(true) - setShowAutocomplete(true) - }}/> - - + hideExtraTypes ? null : + + + { + setMenuPosition({ + top: event.pageY+10, + left: event.pageX+10, + }) + setShowDropdownNumber(count) + setShowDropdown(true) + setShowAutocomplete(true) + }}/> + + + ) }} fullWidth @@ -559,7 +615,7 @@ const ParsedAction = (props) => { rows={rows} color="primary" defaultValue={data.value} - type={placeholder.includes("***") ? "password" : "text"} + type={placeholder.includes("***") || (data.configuration && (data.name.toLowerCase().includes("api") || data.name.toLowerCase().includes("key") || data.name.toLowerCase().includes("pass"))) ? "password" : "text"} placeholder={placeholder} onChange={(event) => { changeActionParameter(event, count, data) @@ -602,19 +658,20 @@ const ParsedAction = (props) => { fontSize: "1em", }, endAdornment: ( - - - { - setMenuPosition({ - top: event.pageY+10, - left: event.pageX+10, - }) - setShowDropdownNumber(count) - setShowDropdown(true) - setShowAutocomplete(true) - }}/> - - + hideExtraTypes ? null : + + + { + setMenuPosition({ + top: event.pageY+10, + left: event.pageX+10, + }) + setShowDropdownNumber(count) + setShowDropdown(true) + setShowAutocomplete(true) + }}/> + + ) }} fullWidth @@ -684,106 +741,6 @@ const ParsedAction = (props) => { } else if (data.variant === "STATIC_VALUE") { staticcolor = "#f85a3e" - } else if (data.variant === "ACTION_RESULT") { - // Gets the parents of the current node - var parents = getParents(selectedAction) - actioncolor = "#f85a3e" - // set the datafield - //var datafieldvalue = "Error: No parents. Action not eligible" - //if (parents.length > 0) { - // datafieldvalue = parents[0].label - //} - const fixedActionText = selectedActionParameters[count].value - - datafield = -
- - Example: $.body will get "data" from {'{"body": "data"}'}
} - placeholder="Action variable ($.)" - onChange={(event) => { - changeActionParameter(event, count, data) - }} - /> -
- - } else if (data.variant === "WORKFLOW_VARIABLE") { - varcolor = "#f85a3e" - if ((workflow.workflow_variables === null || workflow.workflow_variables === undefined || workflow.workflow_variables.length === 0) && (workflow.execution_variables === null || workflow.execution_variables === undefined || workflow.execution_variables.length === 0)) { - setCurrentView(2) - datafield = -
-
- Looks like you don't have any variables yet. -
-
- -
-
- } else { - // FIXME - this is a shitty solution that needs re-renders all the time - datafield = - - } } // Shows nested list of nodes > their JSON lists @@ -894,26 +851,29 @@ const ParsedAction = (props) => { for (var key in workflow.triggers) { const item = workflow.triggers[key] - var node = cy.getElementById(item.id) - if (node.length > 0) { - if (inside) { - node.addClass('shuffle-hover-highlight') - } else { - node.removeClass('shuffle-hover-highlight') + if (cy !== undefined) { + var node = cy.getElementById(item.id) + if (node.length > 0) { + if (inside) { + node.addClass('shuffle-hover-highlight') + } else { + node.removeClass('shuffle-hover-highlight') + } } } - } } } const handleActionHover = (inside, actionId) => { - var node = cy.getElementById(actionId) - if (node.length > 0) { - if (inside) { - node.addClass('shuffle-hover-highlight') - } else { - node.removeClass('shuffle-hover-highlight') + if (cy !== undefined) { + var node = cy.getElementById(actionId) + if (node.length > 0) { + if (inside) { + node.addClass('shuffle-hover-highlight') + } else { + node.removeClass('shuffle-hover-highlight') + } } } } @@ -1009,8 +969,6 @@ const ParsedAction = (props) => { return (
- - {data.configuration === true ? { @@ -1056,7 +1014,7 @@ const ParsedAction = (props) => {
*/} - {selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0 && selectedActionParameters[count].required === true && selectedActionParameters[count].unique_toggled !== undefined ? null : + {(selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0 && selectedActionParameters[count].required === true && selectedActionParameters[count].unique_toggled !== undefined) || hideExtraTypes ? null :
{}}> @@ -1150,99 +1108,106 @@ const ParsedAction = (props) => { } return (
-
-
-

{selectedAction.app_name.replaceAll("_", " ")}

-
- { - console.log("FIND EXAMPLE RESULTS FOR ", selectedAction) - if (workflowExecutions.length > 0) { - // Look for the ID - const found = false - for (var key in workflowExecutions) { - if (workflowExecutions[key].results === undefined || workflowExecutions[key].results === null) { - continue - } + {hideExtraTypes === true ? null : + +
+
+

{selectedAction.app_name.replaceAll("_", " ")}

+
+ { + console.log("FIND EXAMPLE RESULTS FOR ", selectedAction) + if (workflowExecutions.length > 0) { + // Look for the ID + const found = false + for (var key in workflowExecutions) { + if (workflowExecutions[key].results === undefined || workflowExecutions[key].results === null) { + continue + } - var foundResult = workflowExecutions[key].results.find(result => result.action.id === selectedAction.id) - if (foundResult === undefined || foundResult === null) { - continue - } + var foundResult = workflowExecutions[key].results.find(result => result.action.id === selectedAction.id) + if (foundResult === undefined || foundResult === null) { + continue + } - setSelectedResult(foundResult) - setCodeModalOpen(true) - break + setSelectedResult(foundResult) + + if (setCodeModalOpen !== undefined) { + setCodeModalOpen(true) + } + break + } } - } - }}> - - + }}> + + + + + + What are actions? + {selectedAction.errors !== undefined && selectedAction.errors !== null && selectedAction.errors.length > 0 ? +
+ Errors: {selectedAction.errors.join("\n")} +
+ : null + } + +
+
+
+ {selectedAction.id === workflow.start ? null : + + - - - What are actions? - {selectedAction.errors !== null && selectedAction.errors.length > 0 ? -
- Errors: {selectedAction.errors.join("\n")} -
- : null - } - + } + {selectedApp.versions !== null && selectedApp.versions !== undefined && selectedApp.versions.length > 1 ? + + : null }
-
- {selectedAction.id === workflow.start ? null : - - - - } - {selectedApp.versions !== null && selectedApp.versions !== undefined && selectedApp.versions.length > 0 ? - - : null } -
-
- - - Name - - - {selectedApp.name !== undefined && selectedAction.authentication !== undefined && selectedAction.authentication.length === 0 && requiresAuthentication ? + + + Name + + + + } + {selectedApp.name !== undefined && selectedAction.authentication !== null && selectedAction.authentication !== undefined && selectedAction.authentication.length === 0 && requiresAuthentication ?
@@ -1255,8 +1220,8 @@ const ParsedAction = (props) => {
: null} - {selectedAction.authentication !== undefined && selectedAction.authentication.length > 0 ? -
+ {selectedAction.authentication !== undefined && selectedAction.authentication !== null && selectedAction.authentication.length > 0 ? +
Authentication
{/* @@ -1302,7 +1270,7 @@ const ParsedAction = (props) => {
: null} - {showEnvironment ? + {showEnvironment !== undefined && showEnvironment ?
Environment @@ -1375,46 +1343,50 @@ const ParsedAction = (props) => { : null}
-
- Actions -
- + {sortByKey(selectedApp.actions, "label").map(data => { + var newActionname = data.name + if (data.label !== undefined && data.label !== null && data.label.length > 0) { + newActionname = data.label + } + // ROFL FIXME - loop + newActionname = newActionname.replaceAll("_", " ") + newActionname = newActionname.replaceAll("_", " ") + newActionname = newActionname.replaceAll("_", " ") + newActionname = newActionname.replaceAll("_", " ") + newActionname = newActionname.charAt(0).toUpperCase()+newActionname.substring(1) + return ( + + {newActionname} - - ) - })} - - {selectedAction.description !== undefined && selectedAction.description.length > 0 ? + + ) + })} + + : null} + {selectedAction.description !== undefined && selectedAction.description.length > 0 && hideExtraTypes !== true ?
{selectedAction.description}
: null} -
+
diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index ed8e2295..5c76fd77 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -5,8 +5,8 @@ import { useTheme } from '@material-ui/core/styles'; import uuid from "uuid"; import {Link} from 'react-router-dom'; import { Prompt } from 'react-router' -import ReactJson from 'react-json-view' import { useBeforeunload } from 'react-beforeunload'; +import ReactJson from 'react-json-view' import NestedMenuItem from "material-ui-nested-menu-item"; import {TextField, Drawer, Button, Paper, Grid, Tabs, InputAdornment, Tab, ButtonBase, Tooltip, Select, MenuItem, Divider, Dialog, Modal, DialogActions, DialogTitle, InputLabel, DialogContent, FormControl, IconButton, Menu, Input, FormGroup, FormControlLabel, Typography, Checkbox, Breadcrumbs, CircularProgress, Switch, Fade} from '@material-ui/core'; @@ -52,6 +52,25 @@ function useWindowSize() { return size; } +export function sortByKey(array, key) { + if (array === undefined) { + return [] + } + + if (key.startsWith("-") && key.length > 2) { + key = key.slice(1, key.length) + return array.sort(function(a, b) { + var x = a[key]; var y = b[key] + return ((x < y) ? -1 : ((x > y) ? 1 : 0)) + }).reverse() + } + + return array.sort(function(a, b) { + var x = a[key]; var y = b[key] + return ((x < y) ? -1 : ((x > y) ? 1 : 0)) + }) +} + function removeParam(key, sourceURL) { if (sourceURL === undefined) { return @@ -2678,13 +2697,14 @@ const AngularWorkflow = (props) => { */ console.log("ENVS: ", environments) + const parsedEnvironments = environments === null || environments === [] ? "cloud" : environments[defaultEnvironmentIndex] === undefined ? "cloud" : environments[defaultEnvironmentIndex].Name const newAppData = { app_name: app.name, app_version: app.app_version, app_id: app.id, sharing: app.sharing, private_id: app.private_id, - environment: environments === null || environments === [] ? "cloud" : environments[defaultEnvironmentIndex].Name, + environment: parsedEnvironments, errors: [], id_: newNodeId, _id_: newNodeId, @@ -3126,24 +3146,7 @@ const AngularWorkflow = (props) => { - function sortByKey(array, key) { - if (array === undefined) { - return [] - } - - if (key.startsWith("-") && key.length > 2) { - key = key.slice(1, key.length) - return array.sort(function(a, b) { - var x = a[key]; var y = b[key] - return ((x < y) ? -1 : ((x > y) ? 1 : 0)) - }).reverse() - } - - return array.sort(function(a, b) { - var x = a[key]; var y = b[key] - return ((x < y) ? -1 : ((x > y) ? 1 : 0)) - }) - } + const rightsidebarStyle = { position: "fixed", @@ -3170,7 +3173,6 @@ const AngularWorkflow = (props) => { selectedApp={selectedApp} workflowExecutions={workflowExecutions} setSelectedResult={setSelectedResult} - getParents={getParents} setSelectedApp={setSelectedApp} setSelectedTrigger={setSelectedTrigger} setSelectedEdge={setSelectedEdge} diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index d3c2be72..34149bd0 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -301,12 +301,12 @@ const AppCreator = (props) => { return response.json() }) .then((responseJson) => { - if (!responseJson.success) { + if (responseJson.success === false) { alert.error("Failed to get the app") setIsAppLoaded(true) + window.location.pathname = "/search" } else { - const data = JSON.parse(responseJson.body) - parseIncomingOpenapiData(data) + parseIncomingOpenapiData(responseJson) } }) .catch(error => { @@ -346,27 +346,8 @@ const AppCreator = (props) => { if (!responseJson.success) { alert.error("Failed to verify") } else{ - var jsonvalid = false - var tmpvalue = "" - try { - tmpvalue = JSON.parse(responseJson.body) - jsonvalid = true - } catch (e) { - console.log("Error JSON: ", e) - } - - if (!jsonvalid) { - try { - tmpvalue = YAML.parse(responseJson.body, ) - jsonvalid = true - } catch(e) { - console.log("Error YAML: ", e) - } - } - - if (jsonvalid) { - parseIncomingOpenapiData(tmpvalue) - } + parseIncomingOpenapiData(responseJson) + } }) .catch(error => { @@ -427,9 +408,33 @@ const AppCreator = (props) => { // Sets the data up as it should be at later points // This is the data FROM the database, not what's being saved const parseIncomingOpenapiData = (data) => { - //console.log("DATA: ", data.info) - setBasedata(data) + const parsedapp = data.openapi === undefined ? data : JSON.parse(atob(data.openapi)) + data = parsedapp.body === undefined ? parsedapp : parsedapp.body + var jsonvalid = false + var tmpvalue = "" + try { + data = JSON.parse(data) + jsonvalid = true + } catch (e) { + console.log("Error JSON: ", e) + } + + if (!jsonvalid) { + try { + data = YAML.parse(data) + jsonvalid = true + } catch(e) { + console.log("Error YAML: ", e) + } + } + + if (!jsonvalid) { + alert.info("OpenAPI data is invalid.") + return + } + + setBasedata(data) if (data.info !== null && data.info !== undefined) { setName(data.info.title) setDescription(data.info.description) @@ -490,210 +495,211 @@ const AppCreator = (props) => { "PUT", ] - // FIXME - headers? var newActions = [] var wordlist = {} - for (let [path, pathvalue] of Object.entries(data.paths)) { - for (let [method, methodvalue] of Object.entries(pathvalue)) { - if (methodvalue === null) { - alert.info("Skipped method (null)"+method) - continue - } + if (data.paths !== null && data.paths !== undefined) { + for (let [path, pathvalue] of Object.entries(data.paths)) { + for (let [method, methodvalue] of Object.entries(pathvalue)) { + if (methodvalue === null) { + alert.info("Skipped method (null)"+method) + continue + } - if (!allowedfunctions.includes(method.toUpperCase())) { - console.log("Invalid method: ", method, "data: ", methodvalue) - alert.info("Skipped method (not allowed): "+method) - continue - } + if (!allowedfunctions.includes(method.toUpperCase())) { + console.log("Invalid method: ", method, "data: ", methodvalue) + alert.info("Skipped method (not allowed): "+method) + continue + } - var tmpname = methodvalue.summary - if (methodvalue.operationId !== undefined && methodvalue.operationId !== null && methodvalue.operationId.length > 0 && (tmpname === undefined || tmpname.length === 0)) { - tmpname = methodvalue.operationId - } + var tmpname = methodvalue.summary + if (methodvalue.operationId !== undefined && methodvalue.operationId !== null && methodvalue.operationId.length > 0 && (tmpname === undefined || tmpname.length === 0)) { + tmpname = methodvalue.operationId + } - var newaction = { - "name": tmpname, - "description": methodvalue.description, - "url": path, - "file_field": "", - "method": method.toUpperCase(), - "headers": "", - "queries": [], - "paths": [], - "body": "", - "errors": [], - "example_response": "", - } + var newaction = { + "name": tmpname, + "description": methodvalue.description, + "url": path, + "file_field": "", + "method": method.toUpperCase(), + "headers": "", + "queries": [], + "paths": [], + "body": "", + "errors": [], + "example_response": "", + } - if (methodvalue["requestBody"] !== undefined) { - //console.log("Handle requestbody: ", methodvalue["requestBody"]) - if (methodvalue["requestBody"]["content"] !== undefined) { - if (methodvalue["requestBody"]["content"]["application/json"] !== undefined) { - if (methodvalue["requestBody"]["content"]["application/json"]["schema"] !== undefined && methodvalue["requestBody"]["content"]["application/json"]["schema"] !== null) { - if (methodvalue["requestBody"]["content"]["application/json"]["schema"]["properties"] !== undefined) { - var tmpobject = {} - for (let [prop, propvalue] of Object.entries(methodvalue["requestBody"]["content"]["application/json"]["schema"]["properties"])) { - tmpobject[prop] = `\$\{${prop}\}` + if (methodvalue["requestBody"] !== undefined) { + //console.log("Handle requestbody: ", methodvalue["requestBody"]) + if (methodvalue["requestBody"]["content"] !== undefined) { + if (methodvalue["requestBody"]["content"]["application/json"] !== undefined) { + if (methodvalue["requestBody"]["content"]["application/json"]["schema"] !== undefined && methodvalue["requestBody"]["content"]["application/json"]["schema"] !== null) { + if (methodvalue["requestBody"]["content"]["application/json"]["schema"]["properties"] !== undefined) { + var tmpobject = {} + for (let [prop, propvalue] of Object.entries(methodvalue["requestBody"]["content"]["application/json"]["schema"]["properties"])) { + tmpobject[prop] = `\$\{${prop}\}` + } + + for (var subkey in methodvalue["requestBody"]["content"]["application/json"]["schema"]["required"]) { + const tmpitem = methodvalue["requestBody"]["content"]["application/json"]["schema"]["required"][subkey] + tmpobject[tmpitem] = `\$\{${tmpitem}\}` + } + + newaction["body"] = JSON.stringify(tmpobject, null, 2) } - - for (var subkey in methodvalue["requestBody"]["content"]["application/json"]["schema"]["required"]) { - const tmpitem = methodvalue["requestBody"]["content"]["application/json"]["schema"]["required"][subkey] - tmpobject[tmpitem] = `\$\{${tmpitem}\}` - } - - newaction["body"] = JSON.stringify(tmpobject, null, 2) } - } - } else if (methodvalue["requestBody"]["content"]["application/xml"] !== undefined) { - console.log("METHOD XML: ", methodvalue) - if (methodvalue["requestBody"]["content"]["application/xml"]["schema"] !== undefined && methodvalue["requestBody"]["content"]["application/xml"]["schema"] !== null) { - if (methodvalue["requestBody"]["content"]["application/xml"]["schema"]["properties"] !== undefined) { - var tmpobject = {} - for (let [prop, propvalue] of Object.entries(methodvalue["requestBody"]["content"]["application/xml"]["schema"]["properties"])) { - tmpobject[prop] = `\$\{${prop}\}` + } else if (methodvalue["requestBody"]["content"]["application/xml"] !== undefined) { + console.log("METHOD XML: ", methodvalue) + if (methodvalue["requestBody"]["content"]["application/xml"]["schema"] !== undefined && methodvalue["requestBody"]["content"]["application/xml"]["schema"] !== null) { + if (methodvalue["requestBody"]["content"]["application/xml"]["schema"]["properties"] !== undefined) { + var tmpobject = {} + for (let [prop, propvalue] of Object.entries(methodvalue["requestBody"]["content"]["application/xml"]["schema"]["properties"])) { + tmpobject[prop] = `\$\{${prop}\}` + } + + for (var subkey in methodvalue["requestBody"]["content"]["application/xml"]["schema"]["required"]) { + const tmpitem = methodvalue["requestBody"]["content"]["application/xml"]["schema"]["required"][subkey] + tmpobject[tmpitem] = `\$\{${tmpitem}\}` + } + + //console.log("OBJ XML: ", tmpobject) + //newaction["body"] = XML.stringify(tmpobject, null, 2) } - - for (var subkey in methodvalue["requestBody"]["content"]["application/xml"]["schema"]["required"]) { - const tmpitem = methodvalue["requestBody"]["content"]["application/xml"]["schema"]["required"][subkey] - tmpobject[tmpitem] = `\$\{${tmpitem}\}` + } + } else { + if (methodvalue["requestBody"]["content"]["example"] !== undefined) { + if (methodvalue["requestBody"]["content"]["example"]["example"] !== undefined) { + newaction["body"] = methodvalue["requestBody"]["content"]["example"]["example"] + //JSON.stringify(tmpobject, null, 2) } - - //console.log("OBJ XML: ", tmpobject) - //newaction["body"] = XML.stringify(tmpobject, null, 2) } - } - } else { - if (methodvalue["requestBody"]["content"]["example"] !== undefined) { - if (methodvalue["requestBody"]["content"]["example"]["example"] !== undefined) { - newaction["body"] = methodvalue["requestBody"]["content"]["example"]["example"] - //JSON.stringify(tmpobject, null, 2) - } - } - //console.log(methodvalue["requestBody"]["content"]) - if (methodvalue["requestBody"]["content"]["multipart/form-data"] !== undefined) { - if (methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"] !== undefined && methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"] !== null) { - if (methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"]["type"] === "object") { - const fieldname = methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"]["properties"]["fieldname"] - if (fieldname !== undefined) { - console.log("FIELDNAME: ", fieldname) - newaction.file_field = fieldname["value"] + //console.log(methodvalue["requestBody"]["content"]) + if (methodvalue["requestBody"]["content"]["multipart/form-data"] !== undefined) { + if (methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"] !== undefined && methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"] !== null) { + if (methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"]["type"] === "object") { + const fieldname = methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"]["properties"]["fieldname"] + if (fieldname !== undefined) { + console.log("FIELDNAME: ", fieldname) + newaction.file_field = fieldname["value"] + } } } } } } } - } - // HAHAHA wtf is this. - if (methodvalue.responses !== undefined && methodvalue.responses !== null) { - if (methodvalue.responses.default !== undefined) { - if (methodvalue.responses.default.content !== undefined) { - if (methodvalue.responses.default.content["text/plain"] !== undefined) { - if (methodvalue.responses.default.content["text/plain"]["schema"] !== undefined) { - if (methodvalue.responses.default.content["text/plain"]["schema"]["example"] !== undefined) { - newaction.example_response = methodvalue.responses.default.content["text/plain"]["schema"]["example"] + // HAHAHA wtf is this. + if (methodvalue.responses !== undefined && methodvalue.responses !== null) { + if (methodvalue.responses.default !== undefined) { + if (methodvalue.responses.default.content !== undefined) { + if (methodvalue.responses.default.content["text/plain"] !== undefined) { + if (methodvalue.responses.default.content["text/plain"]["schema"] !== undefined) { + if (methodvalue.responses.default.content["text/plain"]["schema"]["example"] !== undefined) { + newaction.example_response = methodvalue.responses.default.content["text/plain"]["schema"]["example"] + } } } } } } - } - for (var key in methodvalue.parameters) { - const parameter = handleGetRef(methodvalue.parameters[key], data) - if (parameter.in === "query") { - var tmpaction = { - "description": parameter.description, - "name": parameter.name, - "required": parameter.required, - "in": "query", + for (var key in methodvalue.parameters) { + const parameter = handleGetRef(methodvalue.parameters[key], data) + if (parameter.in === "query") { + var tmpaction = { + "description": parameter.description, + "name": parameter.name, + "required": parameter.required, + "in": "query", + } + + if (parameter.required === undefined) { + tmpaction.required = false + } + + newaction.queries.push(tmpaction) + } else if (parameter.in === "path") { + // FIXME - parse this to the URL too + newaction.paths.push(parameter.name) + + // FIXME: This doesn't follow OpenAPI3 exactly. + // https://swagger.io/docs/specification/describing-request-body/ + // https://swagger.io/docs/specification/describing-parameters/ + // Need to split the data. + } else if (parameter.in === "body") { + // FIXME: Add tracking for components + // E.G: https://raw.githubusercontent.com/owentl/Shuffle/master/gosecure.yaml + if (parameter.example !== undefined) { + newaction.body = parameter.example + } + } else if (parameter.in === "header") { + newaction.headers += `${parameter.name}=${parameter.example}\n` + } else { + console.log("WARNING: don't know how to handle this param: ", parameter) } - - if (parameter.required === undefined) { - tmpaction.required = false - } - - newaction.queries.push(tmpaction) - } else if (parameter.in === "path") { - // FIXME - parse this to the URL too - newaction.paths.push(parameter.name) - - // FIXME: This doesn't follow OpenAPI3 exactly. - // https://swagger.io/docs/specification/describing-request-body/ - // https://swagger.io/docs/specification/describing-parameters/ - // Need to split the data. - } else if (parameter.in === "body") { - // FIXME: Add tracking for components - // E.G: https://raw.githubusercontent.com/owentl/Shuffle/master/gosecure.yaml - if (parameter.example !== undefined) { - newaction.body = parameter.example - } - } else if (parameter.in === "header") { - newaction.headers += `${parameter.name}=${parameter.example}\n` - } else { - console.log("WARNING: don't know how to handle this param: ", parameter) } - } - if (newaction.name === "" || newaction.name === undefined) { - // Find a unique part of the string - // FIXME: Looks for length between /, find the one where they differ - // Should find others with the same START to their path - // Make a list of reserved names? Aka things that show up only once - if (Object.getOwnPropertyNames(wordlist).length === 0) { - for (let [newpath, pathvalue] of Object.entries(data.paths)) { - const newpathsplit = newpath.split("/") - for(var key in newpathsplit) { - const pathitem = newpathsplit[key].toLowerCase() - if (wordlist[pathitem] === undefined) { - wordlist[pathitem] = 1 - } else { - wordlist[pathitem] += 1 + if (newaction.name === "" || newaction.name === undefined) { + // Find a unique part of the string + // FIXME: Looks for length between /, find the one where they differ + // Should find others with the same START to their path + // Make a list of reserved names? Aka things that show up only once + if (Object.getOwnPropertyNames(wordlist).length === 0) { + for (let [newpath, pathvalue] of Object.entries(data.paths)) { + const newpathsplit = newpath.split("/") + for(var key in newpathsplit) { + const pathitem = newpathsplit[key].toLowerCase() + if (wordlist[pathitem] === undefined) { + wordlist[pathitem] = 1 + } else { + wordlist[pathitem] += 1 + } } } - } - } + } - //console.log("WORDLIST: ", wordlist) + //console.log("WORDLIST: ", wordlist) - // Remove underscores and make it normal with upper case etc - const urlsplit = path.split("/") - if (urlsplit.length > 0) { - var curname = "" - for(var key in urlsplit) { - var subpath = urlsplit[key] - if (wordlist[subpath] > 2 || subpath.length < 1) { - continue + // Remove underscores and make it normal with upper case etc + const urlsplit = path.split("/") + if (urlsplit.length > 0) { + var curname = "" + for(var key in urlsplit) { + var subpath = urlsplit[key] + if (wordlist[subpath] > 2 || subpath.length < 1) { + continue + } + + curname = subpath + break } - - curname = subpath - break - } - // FIXME: If name exists, - // FIXME: Check if first part of parsedname is verb, otherwise use method - const parsedname = curname.split("_").join(" ").split("-").join(" ").split("{").join(" ").split("}").join(" ").trim() - if (parsedname.length === 0) { - newaction.errors.push("Missing name") - } else { - const newname = method.charAt(0).toUpperCase() + method.slice(1) + " " + parsedname - const searchactions = newActions.find(data => data.name === newname) - console.log("SEARCH: ", searchactions) - if (searchactions !== undefined) { + // FIXME: If name exists, + // FIXME: Check if first part of parsedname is verb, otherwise use method + const parsedname = curname.split("_").join(" ").split("-").join(" ").split("{").join(" ").split("}").join(" ").trim() + if (parsedname.length === 0) { newaction.errors.push("Missing name") } else { - newaction.name = newname + const newname = method.charAt(0).toUpperCase() + method.slice(1) + " " + parsedname + const searchactions = newActions.find(data => data.name === newname) + console.log("SEARCH: ", searchactions) + if (searchactions !== undefined) { + newaction.errors.push("Missing name") + } else { + newaction.name = newname + } } - } - } else { - newaction.errors.push("Missing name") + } else { + newaction.errors.push("Missing name") + } } + newActions.push(newaction) } - newActions.push(newaction) } if (data.servers !== undefined && data.servers.length > 0) { @@ -721,8 +727,7 @@ const AppCreator = (props) => { } } - - // FIXME: Have multiple authentication options? + console.log("INVALID9: ", data) if (securitySchemes !== undefined) { for (const [key, value] of Object.entries(securitySchemes)) { if (value.scheme === "bearer") { diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 03b98a5f..d13c068a 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -280,11 +280,19 @@ const Apps = (props) => { if (!responseJson.success) { alert.error("Failed to download file") } else { - const inputdata = YAML.parse(responseJson.body) + console.log(responseJson) + const basedata = atob(responseJson.openapi) + console.log("BASE: ", basedata) + var inputdata = JSON.parse(basedata) + console.log("POST INPUT: ", inputdata) + inputdata = JSON.parse(inputdata.body) + const newpaths = {} - Object.keys(inputdata["paths"]).forEach(function(key) { - newpaths[key.split("?")[0]] = inputdata.paths[key] - }) + if (inputdata["paths"] !== undefined) { + Object.keys(inputdata["paths"]).forEach(function(key) { + newpaths[key.split("?")[0]] = inputdata.paths[key] + }) + } inputdata.paths = newpaths console.log("INPUT: ", inputdata) From e72ba27fb61b901c324b55267fa8f2b2fdb7e454 Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 11 Apr 2021 10:34:48 +0200 Subject: [PATCH 11/96] Added reference/component parsing of OpenAPI --- backend/go-app/main.go | 36 ++--- backend/go-app/oauth2.go | 2 +- backend/go-app/walkoff.go | 31 +++-- frontend/src/components/ConfigureWorkflow.jsx | 15 ++- frontend/src/components/ParsedAction.jsx | 37 +++-- frontend/src/views/AngularWorkflow.jsx | 2 +- frontend/src/views/AppCreator.jsx | 126 +++++++++++++++++- frontend/src/views/Workflows.jsx | 28 +++- 8 files changed, 225 insertions(+), 52 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index f86a557c..501384cc 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -728,9 +728,8 @@ func handleRegisterVerification(resp http.ResponseWriter, request *http.Request) Userdata := users[0] - // FIXME: Not for cloud! Userdata.Verified = true - err = shuffle.SetUser(ctx, &Userdata) + err = shuffle.SetUser(ctx, &Userdata, true) if err != nil { log.Printf("Failed adding verification for user %s: %s", Userdata.Username, err) resp.WriteHeader(401) @@ -740,7 +739,7 @@ func handleRegisterVerification(resp http.ResponseWriter, request *http.Request) resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, defaultMessage))) - log.Printf("%s SUCCESSFULLY FINISHED REGISTRATION", Userdata.Username) + log.Printf("[INFO] %s SUCCESSFULLY FINISHED REGISTRATION", Userdata.Username) } func handleSetEnvironments(resp http.ResponseWriter, request *http.Request) { @@ -928,7 +927,7 @@ func createNewUser(username, password, role, apikey string, org shuffle.OrgMini) newUser.Id = ID.String() newUser.VerificationToken = verifyToken.String() - err = shuffle.SetUser(ctx, newUser) + err = shuffle.SetUser(ctx, newUser, true) if err != nil { log.Printf("Error adding User %s: %s", username, err) return err @@ -941,14 +940,14 @@ func createNewUser(username, password, role, apikey string, org shuffle.OrgMini) if err != nil { log.Printf("Failed updating org with user %s", newUser.Username) } else { - log.Printf("Successfully updated org with user %s!", newUser.Username) + log.Printf("[INFO] Successfully updated org with user %s!", newUser.Username) } } - err = increaseStatisticsField(ctx, "successful_register", username, 1, org.Id) - if err != nil { - log.Printf("Failed to increase total apps loaded stats: %s", err) - } + //err = increaseStatisticsField(ctx, "successful_register", username, 1, org.Id) + //if err != nil { + // log.Printf("Failed to increase total apps loaded stats: %s", err) + //} return nil } @@ -1152,7 +1151,7 @@ func handleUpdateUser(resp http.ResponseWriter, request *http.Request) { foundUser.Username = t.Username } - err = shuffle.SetUser(ctx, foundUser) + err = shuffle.SetUser(ctx, foundUser, true) if err != nil { log.Printf("Error patching user %s: %s", foundUser.Username, err) resp.WriteHeader(401) @@ -1287,7 +1286,7 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { userInfo.Orgs = newStringOrgs - err = shuffle.SetUser(ctx, &userInfo) + err = shuffle.SetUser(ctx, &userInfo, true) if err != nil { log.Printf("Error patching User for activeOrg: %s", err) } else { @@ -1304,7 +1303,7 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { userInfo.ActiveOrg = shuffle.OrgMini{ Id: userInfo.Orgs[0], } - err = shuffle.SetUser(ctx, &userInfo) + err = shuffle.SetUser(ctx, &userInfo, true) if err != nil { log.Printf("Error patching User for activeOrg: %s", err) } @@ -1423,7 +1422,7 @@ func handlePasswordReset(resp http.ResponseWriter, request *http.Request) { Userdata.Password = string(hashedPassword) Userdata.ResetTimeout = 0 Userdata.ResetReference = "" - err = shuffle.SetUser(ctx, &Userdata) + err = shuffle.SetUser(ctx, &Userdata, true) if err != nil { log.Printf("Error adding User %s: %s", Userdata.Username, err) resp.WriteHeader(200) @@ -1630,7 +1629,7 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) { } Userdata.Session = sessionToken - err = shuffle.SetUser(ctx, &Userdata) + err = shuffle.SetUser(ctx, &Userdata, true) if err != nil { log.Printf("Failed updating user when setting session: %s", err) resp.WriteHeader(500) @@ -1641,7 +1640,7 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) { loginData = fmt.Sprintf(`{"success": true, "cookies": [{"key": "session_token", "value": "%s", "expiration": %d}]}`, sessionToken, expiration.Unix()) } - log.Printf("%s SUCCESSFULLY LOGGED IN with session %s", data.Username, Userdata.Session) + log.Printf("[INFO] %s SUCCESSFULLY LOGGED IN with session %s", data.Username, Userdata.Session) resp.WriteHeader(200) resp.Write([]byte(loginData)) @@ -4204,6 +4203,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { swaggerLoader.IsExternalRefsAllowed = true swagger, err := swaggerLoader.LoadSwaggerFromData(body) if err != nil { + log.Println(string(body)) log.Printf("[ERROR] Swagger validation error: %s", err) resp.WriteHeader(500) resp.Write([]byte(`{"success": false, "reason": "Failed verifying openapi"}`)) @@ -4389,7 +4389,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { user.PrivateApps[foundNumber] = api } - err = shuffle.SetUser(ctx, &user) + err = shuffle.SetUser(ctx, &user, true) if err != nil { log.Printf("[ERROR] Failed adding verification for user %s: %s", user.Username, err) resp.WriteHeader(500) @@ -5083,7 +5083,7 @@ func runInit(ctx context.Context) { } } - err = shuffle.SetUser(ctx, &user) + err = shuffle.SetUser(ctx, &user, true) if err != nil { log.Printf("Failed to reset user") } else { @@ -5132,7 +5132,7 @@ func runInit(ctx context.Context) { Name: activeOrgs[0].Name, } - err = shuffle.SetUser(ctx, &user) + err = shuffle.SetUser(ctx, &user, true) if err != nil { log.Printf("Failed updating user %s with org", user.Username) } else { diff --git a/backend/go-app/oauth2.go b/backend/go-app/oauth2.go index 164c803d..71d3cbbd 100644 --- a/backend/go-app/oauth2.go +++ b/backend/go-app/oauth2.go @@ -458,7 +458,7 @@ func handleNewOutlookRegister(resp http.ResponseWriter, request *http.Request) { }) // Set apikey for the user if they don't have one - err = shuffle.SetUser(ctx, Userdata) + err = shuffle.SetUser(ctx, Userdata, true) if err != nil { log.Printf("Failed setting user data for %s: %s", Userdata.Username, err) resp.WriteHeader(401) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 20ab9791..4952f40f 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -2051,7 +2051,7 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request //log.Println(body) //if string(body)[0] == "\"" && string(body)[string(body) - log.Printf("Body: %s", string(body)) + log.Printf("[INFO] Body: %s", string(body)) } var execution shuffle.ExecutionRequest @@ -2093,9 +2093,9 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request return shuffle.WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", workflow.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", workflow.Start)) } } else if len(execution.Start) > 0 { - - log.Printf("[ERROR] START ACTION %s IS WRONG ID LENGTH %d!", execution.Start, len(execution.Start)) - return shuffle.WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", execution.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", execution.Start)) + //log.Printf("[INFO] !") + //log.Printf("[ERROR] START ACTION %s IS WRONG ID LENGTH %d!", execution.Start, len(execution.Start)) + //return shuffle.WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", execution.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", execution.Start)) } if len(execution.ExecutionId) == 36 { @@ -2245,10 +2245,10 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request //log.Println(string(mappedData)) - if len(workflowExecution.Start) == 0 { + if len(workflowExecution.Start) == 0 && len(workflowExecution.Workflow.Start) > 0 { workflowExecution.Start = workflowExecution.Workflow.Start } - log.Printf("[INFO] New startnode: %s", workflowExecution.Start) + //log.Printf("[INFO] New startnode: %s", workflowExecution.Start) childNodes := findChildNodes(workflowExecution, workflowExecution.Start) @@ -2411,13 +2411,20 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request } } } - //childNodes := findChildNodes(workflowExecution, workflowExecution.Start) if !startFound { - log.Printf("[ERROR] Startnode %s doesn't exist!!", workflowExecution.Start) - return shuffle.WorkflowExecution{}, fmt.Sprintf("Workflow action %s doesn't exist in workflow", workflowExecution.Start), errors.New(fmt.Sprintf(`Workflow start node "%s" doesn't exist. Exiting!`, workflowExecution.Start)) + if len(workflowExecution.Start) == 0 && len(workflowExecution.Workflow.Start) > 0 { + workflowExecution.Start = workflow.Start + } else if len(workflowExecution.Workflow.Actions) > 0 { + workflowExecution.Start = workflowExecution.Workflow.Actions[0].ID + } else { + log.Printf("[ERROR] Startnode %s doesn't exist!!", workflowExecution.Start) + return shuffle.WorkflowExecution{}, fmt.Sprintf("Workflow action %s doesn't exist in workflow", workflowExecution.Start), errors.New(fmt.Sprintf(`Workflow start node "%s" doesn't exist. Exiting!`, workflowExecution.Start)) + } } + //log.Printf("EXECUTION START: %s", workflowExecution.Start) + // Verification for execution environments workflowExecution.Results = defaultResults workflowExecution.Workflow.Actions = newActions @@ -3364,7 +3371,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { } user.PrivateApps = privateApps - err = shuffle.SetUser(ctx, &user) + err = shuffle.SetUser(ctx, &user, true) if err != nil { log.Printf("[ERROR] Failed removing %s app for user %s: %s", app.Name, user.Username, err) resp.WriteHeader(401) @@ -4197,7 +4204,7 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, if !found { err = shuffle.SetWorkflowAppDatastore(ctx, api, api.ID) if err != nil { - log.Printf("Failed setting workflowapp in loop: %s", err) + log.Printf("[WARNING] Failed setting workflowapp in loop: %s", err) continue } else { appCounter += 1 @@ -4584,7 +4591,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin err = shuffle.SetWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) if err != nil { - log.Printf("Failed setting workflowapp: %s", err) + log.Printf("[WARNING] Failed setting workflowapp in intro: %s", err) continue } diff --git a/frontend/src/components/ConfigureWorkflow.jsx b/frontend/src/components/ConfigureWorkflow.jsx index 95457675..5dba1179 100644 --- a/frontend/src/components/ConfigureWorkflow.jsx +++ b/frontend/src/components/ConfigureWorkflow.jsx @@ -17,6 +17,7 @@ const Workflow = (props) => { const [requiredTriggers, setRequiredTriggers] = React.useState([]) const [previousAuth, setPreviousAuth] = React.useState(appAuthentication) const [firstLoad, setFirstLoad] = React.useState("") + const [itemChanged, setItemChanged] = React.useState(false) var finished = false if (workflow === undefined || workflow === null) { @@ -216,6 +217,7 @@ const Workflow = (props) => { newWebhook(workflow.triggers[trigger.index]) saveWorkflow(workflow) + setItemChanged(true) }}> {trigger.status !== "running" ? "Start" : "Running"} @@ -232,6 +234,7 @@ const Workflow = (props) => { submitSchedule(workflow.triggers[trigger.index], trigger.index) saveWorkflow(workflow) + setItemChanged(true) }}> {trigger.status !== "running" ? "Start" : "Running"} @@ -350,6 +353,7 @@ const Workflow = (props) => { : @@ -426,8 +431,12 @@ const Workflow = (props) => { */} diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index b4eabc2a..c5295876 100644 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -87,17 +87,21 @@ const ParsedAction = (props) => { responseJson.loop_versions = selectedApp.loop_versions } - var foundAction = responseJson.actions.find(action => action.name === selectedAction.name) + var foundAction = responseJson.actions.find(action => action.name.toLowerCase() === selectedAction.name.toLowerCase()) + console.log("FOUNDACTION: ", foundAction) if (foundAction !== null && foundAction !== undefined) { + var foundparams = [] for (var paramkey in foundAction.parameters) { const param = foundAction.parameters[paramkey] - const foundParam = selectedAction.parameters.find(item => item.name === param.name) + const foundParam = selectedAction.parameters.find(item => item.name.toLowerCase() === param.name.toLowerCase()) if (foundParam === undefined) { - //console.log("COULDNT find Param: ", param) + console.log("COULDNT find Param: ", param) } else { foundAction.parameters[paramkey] = foundParam } + + //foundparams.push(param.name) } } else { alert.error("Couldn't find action "+selectedAction.name) @@ -285,6 +289,8 @@ const ParsedAction = (props) => { "value": event.target.value, }] + console.log("IN IF: ", paramcheck) + } else { const subparamindex = paramcheck["value_replace"].findIndex(param => param.key === data.name) if (subparamindex === -1) { @@ -295,11 +301,25 @@ const ParsedAction = (props) => { } else { paramcheck["value_replace"][subparamindex]["value"] = event.target.value } + + console.log("IN ELSE: ", paramcheck) } //console.log("PARAM: ", paramcheck) - - selectedActionParameters[count]["value_replace"] = paramcheck - selectedAction.parameters[count]["value_replace"] = paramcheck + //if (paramcheck.id === undefined) { + // console.log("Normal paramcheck") + //} else { + // selectedActionParameters[count]["value_replace"] = paramcheck + // selectedAction.parameters[count]["value_replace"] = paramcheck + //} + + if (paramcheck["value_replace"] === undefined) { + selectedActionParameters[count]["value_replace"] = paramcheck + selectedAction.parameters[count]["value_replace"] = paramcheck + } else { + selectedActionParameters[count]["value_replace"] = paramcheck["value_replace"] + selectedAction.parameters[count]["value_replace"] = paramcheck["value_replace"] + } + console.log("RESULT: ", selectedAction) setSelectedAction(selectedAction) //setUpdate(Math.random()) return @@ -718,7 +738,6 @@ const ParsedAction = (props) => { value={selectedActionParameters[count].value} fullWidth onChange={(e) => { - console.log("VAL: ", e.target.value) changeActionParameter(e, count, data) setUpdate(Math.random()) }} @@ -1234,7 +1253,7 @@ const ParsedAction = (props) => { }} fullWidth onChange={(e) => { - console.log("CHOSE AN AUTHENTICATION OPTION: ", e.target.value) + //console.log("CHOSE AN AUTHENTICATION OPTION: ", e.target.value) selectedAction.selectedAuthentication = e.target.value selectedAction.authentication_id = e.target.value.id setSelectedAction(selectedAction) @@ -1243,7 +1262,7 @@ const ParsedAction = (props) => { style={{backgroundColor: theme.palette.inputColor, color: "white", height: 50, maxWidth: rightsidebarStyle.maxWidth-80,}} > {selectedAction.authentication.map(data => { - console.log("DATA: ", data) + //console.log("AUTH DATA: ", data) return( {data.label} - ({data.app.app_version}) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 5c76fd77..bc03a65b 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -5730,7 +5730,7 @@ const AngularWorkflow = (props) => { const timestamp = new Date(data.started_at*1000).toISOString().split('.')[0].split("T").join(" ") - var calculatedResult = data.workflow.actions.length + var calculatedResult = data.workflow.actions !== undefined && data.workflow.actions !== null ? data.workflow.actions.length : 0 for (var key in data.workflow.triggers) { const trigger = data.workflow.triggers[key] if ((trigger.app_name === "User Input" && trigger.trigger_type === "USERINPUT") || (trigger.app_name === "Shuffle Workflow" && trigger.trigger_type === "SUBFLOW")) { diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 34149bd0..27c3c9a2 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -338,7 +338,6 @@ const AppCreator = (props) => { throw new Error("NOT 200 :O") } - //console.log("DATA: ", response.text()) return response.json() }) .then((responseJson) => { @@ -373,6 +372,7 @@ const AppCreator = (props) => { const handleGetRef = (parameter, data) => { if (parameter["$ref"] === undefined) { + console.log("$ref not found in getref: ") return parameter } @@ -530,6 +530,8 @@ const AppCreator = (props) => { "example_response": "", } + //console.log("Schema is application/json: ", methodvalue) + //console.log("DATA", data) if (methodvalue["requestBody"] !== undefined) { //console.log("Handle requestbody: ", methodvalue["requestBody"]) if (methodvalue["requestBody"]["content"] !== undefined) { @@ -541,12 +543,22 @@ const AppCreator = (props) => { tmpobject[prop] = `\$\{${prop}\}` } + //console.log("Data: ", data) for (var subkey in methodvalue["requestBody"]["content"]["application/json"]["schema"]["required"]) { const tmpitem = methodvalue["requestBody"]["content"]["application/json"]["schema"]["required"][subkey] tmpobject[tmpitem] = `\$\{${tmpitem}\}` } newaction["body"] = JSON.stringify(tmpobject, null, 2) + } else if (methodvalue["requestBody"]["content"]["application/json"]["schema"]["$ref"] !== undefined && methodvalue["requestBody"]["content"]["application/json"]["schema"]["$ref"] !== null) { + const retRef = handleGetRef(methodvalue["requestBody"]["content"]["application/json"]["schema"], data) + var newbody = {} + // Can handle default, required, description and type + for (var propkey in retRef.properties) { + const parsedkey = propkey.replaceAll(" ", "_").toLowerCase() + newbody[parsedkey] = "${"+parsedkey+"}" + } + newaction["body"] = JSON.stringify(newbody, null, 2) } } } else if (methodvalue["requestBody"]["content"]["application/xml"] !== undefined) { @@ -603,6 +615,117 @@ const AppCreator = (props) => { } } } + } else { + var selectedReturn = "" + if (methodvalue.responses["200"] !== undefined) { + selectedReturn = "200" + } else if (methodvalue.responses["201"] !== undefined) { + selectedReturn = "201" + } + + // Parsing examples. This should be standardized lol + if (methodvalue.responses[selectedReturn] !== undefined) { + const selectedExample = methodvalue.responses[selectedReturn] + if (selectedExample["content"] !== undefined) { + if (selectedExample["content"]["application/json"] !== undefined) { + if (selectedExample["content"]["application/json"]["schema"] !== undefined) { + if (selectedExample["content"]["application/json"]["schema"]["$ref"] !== undefined) { + //console.log("REF EXAMPLE: ", selectedExample["content"]["application/json"]["schema"]) + const parameter = handleGetRef(selectedExample["content"]["application/json"]["schema"], data) + //console.log("GOT REF RETURN AS EXAMPLE: ", parameter) + + if (parameter.properties !== undefined && parameter["type"] === "object") { + var newbody = {} + for (var propkey in parameter.properties) { + const parsedkey = propkey.replaceAll(" ", "_").toLowerCase() + if (parameter.properties[propkey].type === "string") { + if (parameter.properties[propkey].description !== undefined) { + newbody[parsedkey] = parameter.properties[propkey].description + } else { + newbody[parsedkey] = "" + } + } else if (parameter.properties[propkey].type.includes("int")) { + newbody[parsedkey] = 0 + } else { + console.log("CANT HANDLE TYPE ", parameter.properties[propkey].type) + newbody[parsedkey] = [] + } + } + newaction.example_response = JSON.stringify(newbody, null, 2) + } else { + console.log("CANT HANDLE PARAM: (1) ", parameter.properties) + } + } else { + // Just selecting the first one. bleh. + if (selectedExample["content"]["application/json"]["schema"]["allOf"] !== undefined) { + //console.log("ALLOF: ", selectedExample["content"]["application/json"]["schema"]["allOf"]) + //console.log("BAD EXAMPLE: (SKIP ALLOF) ", selectedExample["content"]["application/json"]["schema"]["allOf"]) + var selectedComponent = selectedExample["content"]["application/json"]["schema"]["allOf"] + if (selectedComponent.length >= 1) { + selectedComponent = selectedComponent[0] + + const parameter = handleGetRef(selectedComponent, data) + if (parameter.properties !== undefined && parameter["type"] === "object") { + var newbody = {} + for (var propkey in parameter.properties) { + const parsedkey = propkey.replaceAll(" ", "_").toLowerCase() + if (parameter.properties[propkey].type === "string") { + if (parameter.properties[propkey].description !== undefined) { + newbody[parsedkey] = parameter.properties[propkey].description + } else { + newbody[parsedkey] = "" + } + } else if (parameter.properties[propkey].type.includes("int")) { + newbody[parsedkey] = 0 + } else { + console.log("CANT HANDLE TYPE ", parameter.properties[propkey].type) + newbody[parsedkey] = [] + } + } + + newaction.example_response = JSON.stringify(newbody, null, 2) + //newaction.example_response = JSON.stringify(parameter.properties, null, 2) + } else { + //newaction.example_response = parameter.properties + console.log("CANT HANDLE PARAM: (3) ", parameter.properties) + } + } else { + + } + } else if (selectedExample["content"]["application/json"]["schema"]["properties"] !== undefined) { + if (selectedExample["content"]["application/json"]["schema"]["properties"]["data"] !== undefined) { + const parameter = handleGetRef(selectedExample["content"]["application/json"]["schema"]["properties"]["data"], data) + if (parameter.properties !== undefined && parameter["type"] === "object") { + var newbody = {} + for (var propkey in parameter.properties) { + const parsedkey = propkey.replaceAll(" ", "_").toLowerCase() + if (parameter.properties[propkey].type === "string") { + if (parameter.properties[propkey].description !== undefined) { + newbody[parsedkey] = parameter.properties[propkey].description + } else { + newbody[parsedkey] = "" + } + } else if (parameter.properties[propkey].type.includes("int")) { + newbody[parsedkey] = 0 + } else { + console.log("CANT HANDLE TYPE ", parameter.properties[propkey].type) + newbody[parsedkey] = [] + } + } + + newaction.example_response = JSON.stringify(newbody, null, 2) + //newaction.example_response = JSON.stringify(parameter.properties, null, 2) + } else { + //newaction.example_response = parameter.properties + console.log("CANT HANDLE PARAM: (2) ", parameter.properties) + } + } + } + } + } + } + } + } } } @@ -727,7 +850,6 @@ const AppCreator = (props) => { } } - console.log("INVALID9: ", data) if (securitySchemes !== undefined) { for (const [key, value] of Object.entries(securitySchemes)) { if (value.scheme === "bearer") { diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 8a2c9176..ddd62883 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -136,6 +136,7 @@ export const validateJson = (showResult) => { const Workflows = (props) => { const { globalUrl, isLoggedIn, isLoaded, removeCookie, cookies, userdata} = props; document.title = "Shuffle - Workflows" + const referenceUrl = globalUrl+"/api/v1/hooks/" const alert = useAlert() const classes = useStyles(); @@ -544,9 +545,7 @@ const Workflows = (props) => { } } - const sanitizeWorkflow = (data) => { - data["owner"] = "" - console.log("Sanitize start: ", data) + const deduplicateIds = (data) => { if (data.triggers !== null && data.triggers !== undefined) { for (var key in data.triggers) { const trigger = data.triggers[key] @@ -561,6 +560,17 @@ const Workflows = (props) => { } const newId = uuid.v4() + if (trigger.trigger_type === "WEBHOOK") { + //"http://localhost:5002/api/v1/hooks/webhook_db179eb0-cb0c-4d2a-9c4b-53d2625a5008" + const hookname = "webhook_"+newId + if (trigger.parameters.length === 2) { + trigger.parameters[0].value = referenceUrl+"webhook_"+trigger.id + trigger.parameters[1].value = "webhook_"+trigger.id + } else { + alert.info("Something is wrong with the webhook in the copy") + } + } + for (var branchkey in data.branches) { const branch = data.branches[branchkey] if (branch.source_id === trigger.id) { @@ -626,8 +636,13 @@ const Workflows = (props) => { } } - //console.log(data) - //return + return data + } + + const sanitizeWorkflow = (data) => { + data["owner"] = "" + console.log("Sanitize start: ", data) + data = deduplicateIds(data) data["org"] = [] data["org_id"] = "" @@ -701,7 +716,8 @@ const Workflows = (props) => { alert.success("Copying workflow "+data.name) data.id = "" data.name = data.name+"_copy" - console.log("COPIED DATA: ", data) + data = deduplicateIds(data) + //console.log("COPIED DATA: ", data) //return fetch(globalUrl+"/api/v1/workflows", { From 09ec48686a9c00c69c8d2df54ab36f35faec305c Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 11 Apr 2021 20:37:41 +0200 Subject: [PATCH 12/96] 0.8.72 release --- backend/app_sdk/app_base.py | 58 +++++++++++++++++------- backend/app_sdk/build.sh | 2 +- backend/go-app/go.mod | 4 +- backend/go-app/main.go | 24 +++++++++- docker-compose.yml | 8 ++-- frontend/src/components/ParsedAction.jsx | 2 +- frontend/src/views/AngularWorkflow.jsx | 2 +- frontend/src/views/AppCreator.jsx | 12 +++-- frontend/src/views/Workflows.jsx | 4 +- 9 files changed, 84 insertions(+), 32 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 9d397aeb..75d0acb4 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -1708,23 +1708,21 @@ class AppBase: if values != None: added = 0 for val in values: - #print(f"VAL: {val}") - #parameter["value"].replace(val["key"], val["value"], -1) - #print(f'PARAM1: {action["parameters"][counter]["value"]}') - action["parameters"][counter]["value"] = action["parameters"][counter]["value"].replace(val["key"], val["value"], 1) - #action["parameters"][counter]["value"].replace(r"${url}", r"$Find_URLs.valid.#.data", 1) - #print(f'PARAM2: {action["parameters"][counter]["value"]}') - #newparams.append({ - # "name": val["key"], - # "value": val["value"], - # "variant": "STATIC_VALUE", - # "id": "body_replacement", - # "schema": { - # "type": "string", - # }, - #}) + replace_value = val["value"] + replace_key = val["key"] + if (val["value"].startswith("{") and val["value"].endswith("}")) or (val["value"].startswith("[") and val["value"].endswith("]")): + print(f"""Trying to parse as JSON: {val["value"]}""") + try: + value_replace = json.loads(val["value"]) + # If it gets here, remove the "" infront and behind the key as well since this is preventing the JSON from being loaded + replace_key = f"\"{replace_key}\"" + except json.decoder.JSONDecodeError as e: + print("Failed JSON replacement for OpenAPI %s", val["key"]) + elif val["value"].lower() == "true" or val["value"].lower() == "false": + replace_key = f"\"{replace_key}\"" + + action["parameters"][counter]["value"] = action["parameters"][counter]["value"].replace(replace_key, replace_value, 1) - #print(f'[INFO] Added param {val["key"]} for body with value {val["value"]} (using OpenAPI)') print(f'[INFO] Added param {val["key"]} for body (using OpenAPI)') added += 1 @@ -1735,6 +1733,34 @@ class AppBase: print("KeyError body OpenAPI: %s" % e) pass + try: + newvalue = json.loads(action["parameters"][counter]["value"]) + deletekeys = [] + for key, value in newvalue.items(): + if isinstance(value, str) and len(value) == 0: + deletekeys.append(key) + continue + + for deletekey in deletekeys: + del newvalue[deletekey] + + action["parameters"][counter]["value"] = json.dumps(newvalue) + + except json.decoder.JSONDecodeError as e: + print("Failed JSON replacement for OpenAPI keys (2) %s", val["key"]) + + #if "\n" in action["parameters"][counter]["value"]: + # print("MODIFYING BODY!!") + # newbody = "" + # for line in action["parameters"][counter]["value"].split("\n"): + # if ": \"\"" in line: + # print("Skipping line %s" % line) + # continue + + # newbody += line + + # print("New body: %s" % newbody) + break #print(action["parameters"]) diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 695cacd6..ca82bf90 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,6 +1,6 @@ #!/bin/bash NAME=shuffle-app_sdk -VERSION=0.8.71 +VERSION=0.8.72 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 55b11a4d..58cb8459 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -2,7 +2,7 @@ module shuffle go 1.13 -replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared +//replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi @@ -20,7 +20,7 @@ require ( github.com/docker/go-connections v0.4.0 github.com/docker/go-units v0.4.0 // indirect github.com/frikky/kin-openapi v0.38.0 - github.com/frikky/shuffle-shared v0.0.23 + github.com/frikky/shuffle-shared v0.0.27 github.com/ghodss/yaml v1.0.0 github.com/go-git/go-billy/v5 v5.0.0 github.com/go-git/go-git/v5 v5.0.0 diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 501384cc..dd437a88 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -10,6 +10,8 @@ import ( "crypto/md5" "encoding/hex" "encoding/json" + //"unicode/utf8" + "errors" "path/filepath" @@ -4191,7 +4193,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("[INFO] EDITING APP WITH ID %s", app.ID) + log.Printf("[INFO] EDITING APP WITH ID %s and md5 %s", app.ID, newmd5) newmd5 = app.ID } @@ -4199,11 +4201,29 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { // Test = client side with fetch? ctx := context.Background() + //s := string(body) + //if !utf8.ValidString(s) { + // v := make([]rune, 0, len(s)) + // for i, r := range s { + // if r == utf8.RuneError { + // _, size := utf8.DecodeRuneInString(s[i:]) + // if size == 1 { + // continue + // } + // } + // v = append(v, r) + // } + // s = string(v) + //} + //fmt.Printf("%q\n", s) + + //body = []byte(strings.Replace(string(body), '<80>', '', -1)) + + //log.Println(string(body)) swaggerLoader := openapi3.NewSwaggerLoader() swaggerLoader.IsExternalRefsAllowed = true swagger, err := swaggerLoader.LoadSwaggerFromData(body) if err != nil { - log.Println(string(body)) log.Printf("[ERROR] Swagger validation error: %s", err) resp.WriteHeader(500) resp.Write([]byte(`{"success": false, "reason": "Failed verifying openapi"}`)) diff --git a/docker-compose.yml b/docker-compose.yml index 56fd9a3c..f9399d8c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: frontend: #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.71 + image: ghcr.io/frikky/shuffle-frontend:0.8.72 container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -17,7 +17,7 @@ services: - backend backend: #build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.71 + image: ghcr.io/frikky/shuffle-backend:0.8.72 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: @@ -47,7 +47,7 @@ services: - database orborus: #build: ./functions/onprem/orborus - image: ghcr.io/frikky/shuffle-orborus:0.8.71 + image: ghcr.io/frikky/shuffle-orborus:0.8.72 container_name: shuffle-orborus hostname: shuffle-orborus networks: @@ -56,7 +56,7 @@ services: - /var/run/docker.sock:/var/run/docker.sock environment: - SHUFFLE_APP_SDK_VERSION=0.8.60 - - SHUFFLE_WORKER_VERSION=0.8.71 + - SHUFFLE_WORKER_VERSION=0.8.72 - ORG_ID=${ORG_ID} - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT} diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index c5295876..6b8e4938 100644 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -1405,7 +1405,7 @@ const ParsedAction = (props) => {
{selectedAction.description}
: null} -
+
diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index bc03a65b..eb2cf843 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -4412,7 +4412,7 @@ const AngularWorkflow = (props) => {
- API-key: + API-key
{ //console.log("Handle requestbody: ", methodvalue["requestBody"]) if (methodvalue["requestBody"]["content"] !== undefined) { if (methodvalue["requestBody"]["content"]["application/json"] !== undefined) { + newaction["headers"] = "Content-Type=application/json\nAccept=application/json" if (methodvalue["requestBody"]["content"]["application/json"]["schema"] !== undefined && methodvalue["requestBody"]["content"]["application/json"]["schema"] !== null) { if (methodvalue["requestBody"]["content"]["application/json"]["schema"]["properties"] !== undefined) { var tmpobject = {} @@ -558,11 +559,13 @@ const AppCreator = (props) => { const parsedkey = propkey.replaceAll(" ", "_").toLowerCase() newbody[parsedkey] = "${"+parsedkey+"}" } + newaction["body"] = JSON.stringify(newbody, null, 2) } } } else if (methodvalue["requestBody"]["content"]["application/xml"] !== undefined) { console.log("METHOD XML: ", methodvalue) + newaction["headers"] = "Content-Type=application/xml\nAccept=application/xml" if (methodvalue["requestBody"]["content"]["application/xml"]["schema"] !== undefined && methodvalue["requestBody"]["content"]["application/xml"]["schema"] !== null) { if (methodvalue["requestBody"]["content"]["application/xml"]["schema"]["properties"] !== undefined) { var tmpobject = {} @@ -851,6 +854,9 @@ const AppCreator = (props) => { } if (securitySchemes !== undefined) { + // FIXME: Should add Oauth2 (Microsoft) and JWT (Wazuh) + //console.log("SECURITY: ", securitySchemes) + //if (Object.entries(securitySchemes) > 1 && for (const [key, value] of Object.entries(securitySchemes)) { if (value.scheme === "bearer") { setAuthenticationOption("Bearer auth") @@ -1240,7 +1246,7 @@ const AppCreator = (props) => { 'Content-Type': 'application/json', 'Accept': 'application/json', }, - body: JSON.stringify(data), + body: JSON.stringify(data, null, 4), credentials: "include", }) .then((response) => { @@ -2428,8 +2434,8 @@ const AppCreator = (props) => { Cancel diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index ddd62883..b06b8a7d 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -595,7 +595,7 @@ const Workflows = (props) => { for (var subkey in data.actions[key].parameters) { const param = data.actions[key].parameters[subkey] - if (param.name.includes("key") || param.name.includes("user") || param.name.includes("pass") || param.name.includes("api") || param.name.includes("auth") || param.name.includes("secret") || param.name.includes("domain") || param.name.includes("url")) { + if (param.name.includes("key") || param.name.includes("user") || param.name.includes("pass") || param.name.includes("api") || param.name.includes("auth") || param.name.includes("secret") || param.name.includes("domain") || param.name.includes("url") || param.name.includes("mail")) { // FIXME: This may be a vuln if api-keys are generated that start with $ if (param.value.startsWith("$")) { console.log("Skipping field, as it's referencing a variable") @@ -629,7 +629,7 @@ const Workflows = (props) => { if (data.workflow_variables !== null && data.workflow_variables !== undefined) { for (var key in data.workflow_variables) { const param = data.workflow_variables[key] - if (param.name.includes("key") || param.name.includes("user") || param.name.includes("pass") || param.name.includes("api") || param.name.includes("auth") || param.name.includes("secret")) { + if (param.name.includes("key") || param.name.includes("user") || param.name.includes("pass") || param.name.includes("api") || param.name.includes("auth") || param.name.includes("secret")|| param.name.includes("email")) { param.value = "" param.is_valid = false } From 8648daa95bc021478a8204a42b7d294eb99c119c Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 12 Apr 2021 21:07:44 +0200 Subject: [PATCH 13/96] BUGS: Fixed a bunch of bugs related to execution and visualization --- .env | 3 +- backend/app_sdk/app_base.py | 1 - backend/app_sdk/build.sh | 2 +- backend/go-app/go.mod | 2 +- backend/go-app/go.sum | 2 + backend/go-app/main.go | 12 +- backend/go-app/walkoff.go | 495 ++------------------- docker-compose.yml | 10 +- frontend/src/components/ParsedAction.jsx | 2 +- frontend/src/views/AngularWorkflow.jsx | 11 + frontend/src/views/Workflows.jsx | 12 +- functions/onprem/orborus/build.sh | 2 +- functions/onprem/orborus/go.mod | 3 +- functions/onprem/orborus/go.sum | 4 + functions/onprem/worker/build.sh | 2 +- functions/onprem/worker/go.mod | 9 +- functions/onprem/worker/worker.go | 532 ++--------------------- 17 files changed, 127 insertions(+), 977 deletions(-) diff --git a/.env b/.env index 4ffc4215..ac31f93a 100644 --- a/.env +++ b/.env @@ -29,8 +29,9 @@ BACKEND_HOSTNAME=shuffle-backend BACKEND_PORT=5001 FRONTEND_PORT=3001 FRONTEND_PORT_HTTPS=3443 +# CHANGE THIS IF YOU WANT GOOD LOCAL EXECUTIONS: OUTER_HOSTNAME=shuffle-backend -DB_LOCATION=./shuffle-database +DB_LOCATION=./shuffle-database-new # Proxy configurations. SHUFFLE_PASS_WORKER_PROXY must be FALSE to not pass the proxy information to sub-apps. # PS: It will skip proxy for diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 75d0acb4..29aa07cd 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -2371,7 +2371,6 @@ class AppBase: # self.action = cls app = cls(redis=None, logger=logger, console_logger=logger) - if isinstance(action, str): print("Normal execution. Action is a string.") elif isinstance(action, object): diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index ca82bf90..f760beee 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,6 +1,6 @@ #!/bin/bash NAME=shuffle-app_sdk -VERSION=0.8.72 +VERSION=0.8.73 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 58cb8459..c654792d 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -20,7 +20,7 @@ require ( github.com/docker/go-connections v0.4.0 github.com/docker/go-units v0.4.0 // indirect github.com/frikky/kin-openapi v0.38.0 - github.com/frikky/shuffle-shared v0.0.27 + github.com/frikky/shuffle-shared v0.0.28 github.com/ghodss/yaml v1.0.0 github.com/go-git/go-billy/v5 v5.0.0 github.com/go-git/go-git/v5 v5.0.0 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 8d8cf6ec..1360ecc7 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -100,6 +100,8 @@ github.com/frikky/shuffle-shared v0.0.22 h1:TFMcJCNmOOSneMMWbg5dNzp2z6m0aZLROuL+ github.com/frikky/shuffle-shared v0.0.22/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= github.com/frikky/shuffle-shared v0.0.23 h1:Pnlc2M6fHnFRLFd5K1iLTVv4/t4P04Ri1GJ5CMxzq0U= github.com/frikky/shuffle-shared v0.0.23/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= +github.com/frikky/shuffle-shared v0.0.27 h1:BbibbAv3a5GWR/DfaoSC4D9+fh2cwSEvn9H+EVfd7BM= +github.com/frikky/shuffle-shared v0.0.27/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= github.com/getkin/kin-openapi v0.8.0 h1:a6TQjTqwkyscC4/hShJX7WhCVE+4bi9lzw61XHQW5hE= github.com/getkin/kin-openapi v0.8.0/go.mod h1:zZQMFkVgRHCdhgb6ihCTIo9dyDZFvX0k/xAKqw1FhPw= github.com/getkin/kin-openapi v0.52.0 h1:6WqsF5d6PfJ8AscdD+9Rtb2RP2iBWyC7V6GcjssWg7M= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index dd437a88..6417dd55 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5023,7 +5023,9 @@ func runInit(ctx context.Context) { _, err = dbclient.GetAll(ctx, q, &users) if err == nil { setOrgBool := false + usernames := []string{} for _, user := range users { + usernames = append(usernames, user.Username) newUser := shuffle.User{ Username: user.Username, Id: user.Id, @@ -5048,6 +5050,8 @@ func runInit(ctx context.Context) { } } + log.Printf("Users found: %s", strings.Join(usernames, ", ")) + if setOrgBool { err = shuffle.SetOrg(ctx, activeOrg, activeOrg.Id) if err != nil { @@ -5076,9 +5080,11 @@ func runInit(ctx context.Context) { if err != nil && len(activeusers) == 0 { log.Printf("Error getting users during init: %s", err) } else { + log.Printf("Parsing all users and setting them to active.") q := datastore.NewQuery("Users") var users []shuffle.User _, err := dbclient.GetAll(ctx, q, &users) + //log.Printf("User ret: %s", err) if len(activeusers) == 0 && len(users) > 0 { log.Printf("No active users found - setting ALL to active") @@ -5136,7 +5142,7 @@ func runInit(ctx context.Context) { } } } else { - if len(users) < 5 && len(users) > 0 { + if len(users) < 10 && len(users) > 0 { for _, user := range users { log.Printf("[INFO] Username: %s, role: %s", user.Username, user.Role) } @@ -5414,7 +5420,7 @@ func runInit(ctx context.Context) { if workflowapp.Edited == 0 { err = shuffle.SetWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) if err == nil { - log.Printf("Updating time for workflowapp %s:%s", workflowapp.Name, workflowapp.AppVersion) + log.Printf("[INFO] Updating time for workflowapp %s:%s", workflowapp.Name, workflowapp.AppVersion) } } } @@ -6115,7 +6121,7 @@ func initHandlers() { // Make user related locations // Fix user changes with org - r.HandleFunc("/api/v1/users/login", handleLogin).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/users/login", shuffle.HandleLogin).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/users/register", handleRegister).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/users/checkusers", checkAdminLogin).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/users/getinfo", handleInfo).Methods("GET", "OPTIONS") diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 4952f40f..83ca6c12 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -881,61 +881,9 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { } -// Finds the child nodes of a node in execution and returns them -// Used if e.g. a node in a branch is exited, and all children have to be stopped -func findChildNodes(workflowExecution shuffle.WorkflowExecution, nodeId string) []string { - //log.Printf("\nNODE TO FIX: %s\n\n", nodeId) - allChildren := []string{nodeId} - - // 1. Find children of this specific node - // 2. Find the children of those nodes etc. - for _, branch := range workflowExecution.Workflow.Branches { - if branch.SourceID == nodeId { - //log.Printf("Children: %s", branch.DestinationID) - allChildren = append(allChildren, branch.DestinationID) - - childNodes := findChildNodes(workflowExecution, branch.DestinationID) - for _, bottomChild := range childNodes { - found := false - for _, topChild := range allChildren { - if topChild == bottomChild { - found = true - break - } - } - - if !found { - allChildren = append(allChildren, bottomChild) - } - } - } - } - - // Remove potential duplicates - newNodes := []string{} - for _, tmpnode := range allChildren { - found := false - for _, newnode := range newNodes { - if newnode == tmpnode { - found = true - break - } - } - - if !found { - newNodes = append(newNodes, tmpnode) - } - } - - return newNodes -} - // Checks if data is sent from Worker >0.8.51, which sends a full execution // instead of individial results func validateNewWorkerExecution(body []byte) error { - //type WorkflowExecution struct { - //} - ctx := context.Background() var execution shuffle.WorkflowExecution err := json.Unmarshal(body, &execution) @@ -981,7 +929,7 @@ func validateNewWorkerExecution(body []byte) error { execution.Status = "FINISHED" } - log.Printf("[INFO] BASEEXECUTION LENGTH: %d", len(baseExecution.Workflow.Actions)+extra) + //log.Printf("[INFO] BASEEXECUTION LENGTH: %d", len(baseExecution.Workflow.Actions)+extra) } // FIXME: Add extra here @@ -1035,6 +983,8 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { return } + //log.Printf("Received action: %#v", actionResult) + // 1. Get the WorkflowExecution(ExecutionId) from the database // 2. if ActionResult.Authentication != WorkflowExecution.Authentication -> exit // 3. Add to and update actionResult in workflowExecution @@ -1144,443 +1094,58 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl return } - //resultLength := len(workflowExecution.Results) - dbSave := false - setExecution := true - - if actionResult.Action.ID == "" { - //log.Printf("[ERROR] Failed handling EMPTY action %#v", actionResult) + //log.Printf("BASE LENGTH: %d", len(workflowExecution.Results)) + workflowExecution, dbSave, err := shuffle.ParsedExecutionResult(ctx, *workflowExecution, actionResult) + if err != nil { + log.Printf("[ERROR] Failed execution of parsedexecution: %s", err) resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't handle empty action"}`))) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting execution"}`))) return } - //tx, err := dbclient.NewTransaction(ctx) - //if err != nil { - // log.Printf("client.NewTransaction: %v", err) - // resp.WriteHeader(401) - // resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed creating transaction"}`))) - // return - //} - //key := datastore.NameKey("workflowexecution", workflowExecutionId, nil) - //workflowExecution := &WorkflowExecution{} - //if err := tx.Get(key, workflowExecution); err != nil { - // log.Printf("[ERROR] tx.Get bug: %v", err) - // tx.Rollback() - // resp.WriteHeader(401) - // resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting the workflow key"}`))) - // return - //} + //log.Printf("NEW LENGTH: %d", len(workflowExecution.Results)) - if actionResult.Status == "ABORTED" || actionResult.Status == "FAILURE" { - dbSave = true + _ = dbSave + //resultLength := len(workflowExecution.Results) + setExecution := true - newResults := []shuffle.ActionResult{} - childNodes := []string{} - if workflowExecution.Workflow.Configuration.ExitOnError { - log.Printf("[WARNING] Actionresult is %s for node %s in %s. Should set workflowExecution and exit all running functions", actionResult.Status, actionResult.Action.ID, workflowExecution.ExecutionId) - workflowExecution.Status = actionResult.Status - workflowExecution.LastNode = actionResult.Action.ID - // Find underlying nodes and add them - } else { - log.Printf("[WARNING] Actionresult is %s for node %s in %s. Continuing anyway because of workflow configuration.", actionResult.Status, actionResult.Action.ID, workflowExecution.ExecutionId) - - // Finds ALL childnodes to set them to SKIPPED - childNodes = findChildNodes(*workflowExecution, actionResult.Action.ID) - - // Remove duplicates - //log.Printf("CHILD NODES: %d", len(childNodes)) - for _, nodeId := range childNodes { - if nodeId == actionResult.Action.ID { - continue - } - - // 1. Find the action itself - // 2. Create an actionresult - curAction := shuffle.Action{ID: ""} - for _, action := range workflowExecution.Workflow.Actions { - if action.ID == nodeId { - curAction = action - break - } - } - - if len(curAction.ID) == 0 { - log.Printf("Couldn't find subnode %s", nodeId) - continue - } - - resultExists := false - for _, result := range workflowExecution.Results { - if result.Action.ID == curAction.ID { - resultExists = true - break - } - } - - if !resultExists { - // Check parents are done here. Only add it IF all parents are skipped - skipNodeAdd := false - for _, branch := range workflowExecution.Workflow.Branches { - if branch.DestinationID == nodeId { - // If the branch's source node is NOT in childNodes, it's not a skipped parent - sourceNodeFound := false - for _, item := range childNodes { - if item == branch.SourceID { - sourceNodeFound = true - break - } - } - - if !sourceNodeFound { - log.Printf("Not setting node %s to SKIPPED", nodeId) - skipNodeAdd = true - break - } - } - } - - if !skipNodeAdd { - newAction := shuffle.Action{ - AppName: curAction.AppName, - AppVersion: curAction.AppVersion, - Label: curAction.Label, - Name: curAction.Name, - ID: curAction.ID, - } - - newResult := shuffle.ActionResult{ - Action: newAction, - ExecutionId: actionResult.ExecutionId, - Authorization: actionResult.Authorization, - Result: "Skipped because of previous node", - StartedAt: 0, - CompletedAt: 0, - Status: "SKIPPED", - } - - newResults = append(newResults, newResult) - //increaseStatisticsField(ctx, "workflow_execution_actions_skipped", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) - } - } - } - } - - // Cleans up aborted, and always gives a result - lastResult := "" - // type ActionResult struct { - for _, result := range workflowExecution.Results { - if actionResult.Action.ID == result.Action.ID { - continue - } - - if result.Status == "EXECUTING" { - result.Status = actionResult.Status - result.Result = "Aborted because of error in another node (2)" - } - - if len(result.Result) > 0 { - lastResult = result.Result - } - - newResults = append(newResults, result) - } - - workflowExecution.Result = lastResult - workflowExecution.Results = newResults - - if workflowExecution.Status == "ABORTED" { - //err = increaseStatisticsField(ctx, "workflow_executions_aborted", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) - //if err != nil { - // log.Printf("Failed to increase aborted execution stats: %s", err) - //} - } else if workflowExecution.Status == "FAILURE" { - //err = increaseStatisticsField(ctx, "workflow_executions_failure", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) - //if err != nil { - // log.Printf("Failed to increase failure execution stats: %s", err) - //} - } - } - - // FIXME rebuild to be like this or something - // workflowExecution/ExecutionId/Nodes/NodeId - // Find the appropriate action - //log.Printf("[INFO] Setting value of %s in workflow %s to %s (1)", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status) - if len(workflowExecution.Results) > 0 { - // FIXME - skip := false - found := false - outerindex := 0 - for index, item := range workflowExecution.Results { - if item.Action.ID == actionResult.Action.ID { - found = true - if item.Status == actionResult.Status { - skip = true - } - - outerindex = index - break - } - } - - if skip { - //log.Printf("Both are %s. Skipping this node", item.Status) - } else if found { - // If result exists and execution variable exists, update execution value - //log.Printf("Exec var backend: %s", workflowExecution.Results[outerindex].Action.ExecutionVariable.Name) - actionVarName := workflowExecution.Results[outerindex].Action.ExecutionVariable.Name - // Finds potential execution arguments - if len(actionVarName) > 0 { - log.Printf("EXECUTION VARIABLE LOCAL: %s", actionVarName) - for index, execvar := range workflowExecution.ExecutionVariables { - if execvar.Name == actionVarName { - // Sets the value for the variable - workflowExecution.ExecutionVariables[index].Value = actionResult.Result - break - } - } - } - - log.Printf("[INFO] Updating %s in workflow %s from %s to %s (3)", actionResult.Action.ID, workflowExecution.ExecutionId, workflowExecution.Results[outerindex].Status, actionResult.Status) - workflowExecution.Results[outerindex] = actionResult - } else { - log.Printf("[INFO] Setting value of %s in workflow %s to %s (1)", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status) - workflowExecution.Results = append(workflowExecution.Results, actionResult) - } - } else { - log.Printf("[INFO] Setting value of %s in workflow %s to %s (2)", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status) - workflowExecution.Results = append(workflowExecution.Results, actionResult) - } - - // FIXME: Have a check for skippednodes and their parents - for resultIndex, result := range workflowExecution.Results { - if result.Status != "SKIPPED" { - continue - } - - // Checks if all parents are skipped or failed. Otherwise removes them from the results - for _, branch := range workflowExecution.Workflow.Branches { - if branch.DestinationID == result.Action.ID { - for _, subresult := range workflowExecution.Results { - if subresult.Action.ID == branch.SourceID { - if subresult.Status != "SKIPPED" && subresult.Status != "FAILURE" { - log.Printf("SUBRESULT PARENT STATUS: %s", subresult.Status) - log.Printf("Should remove resultIndex: %d", resultIndex) - - workflowExecution.Results = append(workflowExecution.Results[:resultIndex], workflowExecution.Results[resultIndex+1:]...) - - break - } - } - } - } - } - } - - extraInputs := 0 - for _, trigger := range workflowExecution.Workflow.Triggers { - if trigger.Name == "User Input" && trigger.AppName == "User Input" { - extraInputs += 1 - } else if trigger.Name == "Shuffle Workflow" && trigger.AppName == "Shuffle Workflow" { - extraInputs += 1 - } - } - - //log.Printf("EXTRA: %d", extraInputs) - //log.Printf("LENGTH: %d - %d", len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)+extraInputs) - - if len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions)+extraInputs { - //log.Printf("\nIN HERE WITH RESULTS %d vs %d\n", len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)+extraInputs) - finished := true - lastResult := "" - - // Doesn't have to be SUCCESS and FINISHED everywhere anymore. - skippedNodes := false - for _, result := range workflowExecution.Results { - if result.Status == "EXECUTING" { - finished = false - break - } - - // FIXME: Check if ALL parents are skipped or if its just one. Otherwise execute it - if result.Status == "SKIPPED" { - skippedNodes = true - - // Checks if all parents are skipped or failed. Otherwise removes them from the results - for _, branch := range workflowExecution.Workflow.Branches { - if branch.DestinationID == result.Action.ID { - for _, subresult := range workflowExecution.Results { - if subresult.Action.ID == branch.SourceID { - if subresult.Status != "SKIPPED" && subresult.Status != "FAILURE" { - //log.Printf("SUBRESULT PARENT STATUS: %s", subresult.Status) - //log.Printf("Should remove resultIndex: %d", resultIndex) - finished = false - break - } - } - } - } - - if !finished { - break - } - } - } - - lastResult = result.Result - } - - // FIXME: Handle skip nodes - change status? - _ = skippedNodes - - if finished { - dbSave = true - log.Printf("[INFO] Execution of %s finished.", workflowExecution.ExecutionId) - //log.Println("Might be finished based on length of results and everything being SUCCESS or FINISHED - VERIFY THIS. Setting status to finished.") - - workflowExecution.Result = lastResult - workflowExecution.Status = "FINISHED" - workflowExecution.CompletedAt = int64(time.Now().Unix()) - if workflowExecution.LastNode == "" { - workflowExecution.LastNode = actionResult.Action.ID - } - - //err = increaseStatisticsField(ctx, "workflow_executions_success", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) - //if err != nil { - // log.Printf("Failed to increase success execution stats: %s", err) - //} - - // Handles extra statistics stuff when it's done - // Does autocomplete magic with JSON - handleExecutionStatistics(*workflowExecution) - } - } - - // FIXME - why isn't this how it works otherwise, wtf? - //workflow, err := shuffle.GetWorkflow(workflowExecution.Workflow.ID) - //newActions := []Action{} - //for _, action := range workflowExecution.Workflow.Actions { - // log.Printf("Name: %s, Env: %s", action.Name, action.Environment) - //} - - tmpJson, err := json.Marshal(workflowExecution) - if err == nil { - if len(tmpJson) >= 1048487 { - dbSave = true - log.Printf("[ERROR] Result length is too long! Need to reduce result size") - - // Result string `json:"result" datastore:"result,noindex"` - // Arbitrary reduction size - maxSize := 500000 - newResults := []shuffle.ActionResult{} - for _, item := range workflowExecution.Results { - if len(item.Result) > maxSize { - item.Result = "[ERROR] Result too large to handle (https://github.com/frikky/shuffle/issues/171)" - } - - newResults = append(newResults, item) - } - - workflowExecution.Results = newResults - } - } - - // Validating that action results hasn't changed - // Handled using cachhing, so actually pretty fast - cacheKey := fmt.Sprintf("workflowexecution-%s", workflowExecution.ExecutionId) - cache, err := shuffle.GetCache(ctx, cacheKey) - if err == nil { - cacheData := []byte(cache.([]uint8)) - //log.Printf("CACHEDATA: %#v", cacheData) - err = json.Unmarshal(cacheData, &workflowExecution) - if err == nil { - if attempts > 5 { - //log.Printf("\n\nSkipping execution input - %d vs %d. Attempts: (%d)\n\n", len(parsedValue.Results), resultLength, attempts) - } - - attempts += 1 - if len(workflowExecution.Results) <= len(workflowExecution.Workflow.Actions) { - runWorkflowExecutionTransaction(ctx, attempts, workflowExecutionId, actionResult, resp) - return - } - } - } - - //if value, found := requestCache.Get(cacheKey); found { - // parsedValue := value.(*shuffle.WorkflowExecution) - // if len(parsedValue.Results) > 0 && len(parsedValue.Results) != resultLength { + //newExecution, err := shuffle.GetWorkflowExecution(ctx, workflowExecution.ExecutionId) + //if err == nil { + // //log.Printf("GOT GOOD EXECUTION CACHE FOR %s!", workflowExecution.ExecutionId) + // if len(newExecution.Results) > 0 && len(newExecution.Results) != resultLength { // setExecution = false // if attempts > 5 { // //log.Printf("\n\nSkipping execution input - %d vs %d. Attempts: (%d)\n\n", len(parsedValue.Results), resultLength, attempts) // } // attempts += 1 - // if len(workflowExecution.Results) <= len(workflowExecution.Workflow.Actions) { - // runWorkflowExecutionTransaction(ctx, attempts, workflowExecutionId, actionResult, resp) - // return - // } + // //if len(workflowExecution.Results) <= len(workflowExecution.Workflow.Actions) { + // // log.Printf("RUNNING AGAIN!!") + // // runWorkflowExecutionTransaction(ctx, attempts, workflowExecutionId, actionResult, resp) + // // return // } + //} else { + // log.Printf("[WARNING] Failed getting cache for %s: %s", workflowExecution.ExecutionId, err) //} if setExecution || workflowExecution.Status == "FINISHED" || workflowExecution.Status == "ABORTED" || workflowExecution.Status == "FAILURE" { - err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, dbSave) + err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, true) + //err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, dbSave) if err != nil { resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting workflowexecution actionresult: %s"}`, err))) return } + //handleExecutionResult(ctx, *workflowExecution) } else { log.Printf("Skipping setexec with status %s", workflowExecution.Status) } - //ExecutionId - // Transactions: https://cloud.google.com/datastore/docs/concepts/transactions#datastore-datastore-transactional-update-go - // Prevents timing issues - //if _, err := tx.Put(key, workflowExecution); err != nil { - // log.Printf("[ERROR] tx.Put error: %v", err) - // err = tx.Rollback() - // if err != nil { - // log.Printf("[ERROR] Rollback error (3): %s", err) - // } + if resp != nil { + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) + } - // resp.WriteHeader(401) - // resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting workflowexecution actionresult: %s"}`, err))) - // return - //} - - //if _, err = tx.Commit(); err != nil { - // err = tx.Rollback() - // if err != nil { - // log.Printf("[ERROR] Rollback error expected ? (1): %s", err) - // } - - // if attempts >= 7 { - // log.Printf("[ERROR] QUITTING: tx.Commit %d: %v", attempts, err) - - // workflowExecution.Status = "ABORTED" - // shuffle.SetWorkflowExecution(ctx, *workflowExecution, true) - - // resp.WriteHeader(401) - // resp.Write([]byte(`{"success": false}`)) - // return - // } - - // if attempts > 3 { - // log.Printf("[WARNING] tx.Commit %d: %v", attempts, err) - // } - - // attempts += 1 - // runWorkflowExecutionTransaction(ctx, attempts, workflowExecutionId, actionResult, resp) - // return - //} else { - // //if grpc.Code(err) == codes.Aborted { - // // return nil, ErrConcurrentTransaction - // //} - // //t.id = nil // mark the transaction as expired - //} - - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } func JSONCheck(str string) bool { @@ -2250,7 +1815,7 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request } //log.Printf("[INFO] New startnode: %s", workflowExecution.Start) - childNodes := findChildNodes(workflowExecution, workflowExecution.Start) + childNodes := shuffle.FindChildNodes(workflowExecution, workflowExecution.Start) topic := "workflows" startFound := false diff --git a/docker-compose.yml b/docker-compose.yml index f9399d8c..b2de1a33 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: frontend: #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.72 + image: ghcr.io/frikky/shuffle-frontend:0.8.73 container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -17,7 +17,7 @@ services: - backend backend: #build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.72 + image: ghcr.io/frikky/shuffle-backend:0.8.73 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: @@ -47,7 +47,7 @@ services: - database orborus: #build: ./functions/onprem/orborus - image: ghcr.io/frikky/shuffle-orborus:0.8.72 + image: ghcr.io/frikky/shuffle-orborus:0.8.73 container_name: shuffle-orborus hostname: shuffle-orborus networks: @@ -55,8 +55,8 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock environment: - - SHUFFLE_APP_SDK_VERSION=0.8.60 - - SHUFFLE_WORKER_VERSION=0.8.72 + - SHUFFLE_APP_SDK_VERSION=0.8.73 + - SHUFFLE_WORKER_VERSION=0.8.73 - ORG_ID=${ORG_ID} - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT} diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index 6b8e4938..40590a4a 100644 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -546,7 +546,7 @@ const ParsedAction = (props) => { var disabled = false var rows = "5" var openApiHelperText = "This is an OpenAPI specific field" - if (selectedApp.generated && selectedApp.activated && data.name === "body") { + if (selectedApp.generated && data.name === "body") { const regex = /\${(\w+)}/g const found = placeholder.match(regex) if (found === null) { diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index eb2cf843..9b018e62 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -5827,6 +5827,17 @@ const AngularWorkflow = (props) => { + {executionData.status === "EXECUTING" ? + + + + + + : null}
{executionData.status !== undefined && executionData.status.length > 0 ?
diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index b06b8a7d..bfca455f 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -561,7 +561,6 @@ const Workflows = (props) => { const newId = uuid.v4() if (trigger.trigger_type === "WEBHOOK") { - //"http://localhost:5002/api/v1/hooks/webhook_db179eb0-cb0c-4d2a-9c4b-53d2625a5008" const hookname = "webhook_"+newId if (trigger.parameters.length === 2) { trigger.parameters[0].value = referenceUrl+"webhook_"+trigger.id @@ -574,12 +573,10 @@ const Workflows = (props) => { for (var branchkey in data.branches) { const branch = data.branches[branchkey] if (branch.source_id === trigger.id) { - //console.log("CHANGING SOURCE ID") branch.source_id = newId } if (branch.destination_id === trigger.id) { - //console.log("CHANGING DESTINATION ID") branch.destination_id = newId } } @@ -610,16 +607,18 @@ const Workflows = (props) => { for (var branchkey in data.branches) { const branch = data.branches[branchkey] if (branch.source_id === data.actions[key].id) { - //console.log("CHANGING SOURCE ID IN ACTION") branch.source_id = newId } if (branch.destination_id === data.actions[key].id) { - //console.log("CHANGING DESTINATION ID IN ACTION") branch.destination_id = newId } } + if (data.actions[key].id === data.start) { + data.start = newId + } + //data.actions[key].environment = isCloud ? "cloud" : "Shuffle" data.actions[key].environment = "" data.actions[key].id = newId @@ -629,7 +628,7 @@ const Workflows = (props) => { if (data.workflow_variables !== null && data.workflow_variables !== undefined) { for (var key in data.workflow_variables) { const param = data.workflow_variables[key] - if (param.name.includes("key") || param.name.includes("user") || param.name.includes("pass") || param.name.includes("api") || param.name.includes("auth") || param.name.includes("secret")|| param.name.includes("email")) { + if (param.name.includes("key") || param.name.includes("user") || param.name.includes("pass") || param.name.includes("api") || param.name.includes("auth") || param.name.includes("secret") || param.name.includes("email")) { param.value = "" param.is_valid = false } @@ -659,6 +658,7 @@ const Workflows = (props) => { const exportWorkflow = (data) => { let exportFileDefaultName = data.name+'.json'; data = sanitizeWorkflow(data) + return // Add correct ID's for triggers // Add mag diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index 08b121c2..47d0510e 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-orborus -VERSION=0.8.72 +VERSION=0.8.73 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 2991cf7f..addb6aa1 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -3,13 +3,14 @@ module orborus go 1.13 require ( + github.com/Masterminds/semver v1.5.0 // indirect github.com/Microsoft/go-winio v0.4.16 // indirect github.com/containerd/containerd v1.4.3 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/docker v20.10.1+incompatible github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect - github.com/frikky/shuffle-shared v0.0.23 // indirect + github.com/frikky/shuffle-shared v0.0.28 github.com/gogo/protobuf v1.3.1 // indirect github.com/mackerelio/go-osstat v0.1.0 github.com/opencontainers/go-digest v1.0.0 // indirect diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index ada1bd1f..8a8e200b 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -41,6 +41,8 @@ cloud.google.com/go/storage v1.12.0/go.mod h1:fFLk2dp2oAhDz8QFKwqrjdJvxSp/W2g7ni dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk= github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -73,6 +75,8 @@ github.com/frikky/shuffle-shared v0.0.12 h1:+0EIfThmK47Po+LogPYZR4XjbS4Ds19WNMFu github.com/frikky/shuffle-shared v0.0.12/go.mod h1:SEY432/xs4oBkOUnGwxiYKCZWZLRaheGInhAoW7N8ww= github.com/frikky/shuffle-shared v0.0.23 h1:Pnlc2M6fHnFRLFd5K1iLTVv4/t4P04Ri1GJ5CMxzq0U= github.com/frikky/shuffle-shared v0.0.23/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= +github.com/frikky/shuffle-shared v0.0.28 h1:VQqL3+ePwKSUxCOiCC8DpOEgbb2GhXI8XzFB/YlHbps= +github.com/frikky/shuffle-shared v0.0.28/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh index 0c4d435e..4f9dc1df 100644 --- a/functions/onprem/worker/build.sh +++ b/functions/onprem/worker/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-worker -VERSION=0.8.72 +VERSION=0.8.73 echo "Running docker build with $NAME:$VERSION" #CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin . diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 0e54ab5a..73f41c23 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -3,16 +3,19 @@ module worker go 1.15 require ( + github.com/Masterminds/semver v1.5.0 // indirect + github.com/Microsoft/go-winio v0.4.16 // indirect github.com/containerd/containerd v1.4.4 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect - github.com/docker/docker v20.10.5+incompatible // indirect + github.com/docker/docker v20.10.5+incompatible github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect - github.com/frikky/shuffle-shared v0.0.24 // indirect + github.com/frikky/shuffle-shared v0.0.28 github.com/gogo/protobuf v1.3.2 // indirect - github.com/gorilla/mux v1.8.0 // indirect + github.com/gorilla/mux v1.8.0 github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect + github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 // indirect github.com/sirupsen/logrus v1.8.1 // indirect ) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 91e5b975..5272b052 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -94,9 +94,9 @@ func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason sleepDuration := 1 if handleResultSend && requestsSent < 2 { - data, err := json.Marshal(workflowExecution) + shutdownData, err := json.Marshal(workflowExecution) if err == nil { - sendResult(workflowExecution, data) + sendResult(workflowExecution, shutdownData) log.Printf("[WARNING] Sent shutdown update") } else { log.Printf("[WARNING] DIDNT send update") @@ -300,32 +300,25 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] ShowStdout: true, } + exit := true out, err := cli.ContainerLogs(ctx, cont.ID, logOptions) if err != nil { log.Printf("[INFO] Failed getting logs: %s", err) } else { - log.Printf("IN ELSE FOR DEPLOY") buf := new(strings.Builder) io.Copy(buf, out) logs := buf.String() - log.Printf("Logs: %s", logs) + log.Printf("Execution Logs: %s", logs) - //log.Printf(logs) - // check errors - /* - if strings.Contains(logs, "Error") { - log.Printf("ERROR IN %s?", cont.ID) - log.Println(logs) - //return errors.New(fmt.Sprintf("ERROR FROM CONTAINER %s", cont.ID)) - } else { - log.Printf("NORMAL EXEC OF %s?", cont.ID) - } - */ + if strings.Contains(logs, "Normal execution.") { + exit = false + } } - log.Printf("ERROR IN CONTAINER DEPLOYMENT - ITS EXITED!") - - return errors.New(fmt.Sprintf(`{"success": false, "reason": "Container %s exited prematurely.","debug": "docker logs -f %s"}`, cont.ID, cont.ID)) + if exit { + log.Printf("ERROR IN CONTAINER DEPLOYMENT - ITS EXITED!") + return errors.New(fmt.Sprintf(`{"success": false, "reason": "Container %s exited prematurely.","debug": "docker logs -f %s"}`, cont.ID, cont.ID)) + } } } } @@ -519,6 +512,7 @@ func removeIndex(s []string, i int) []string { } func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { + log.Printf("Inside execution results with %d / %d results", len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)) if len(startAction) == 0 { startAction = workflowExecution.Start if len(startAction) == 0 { @@ -1243,7 +1237,7 @@ func executionInit(workflowExecution shuffle.WorkflowExecution) error { return nil } -func handleExecution(client *http.Client, req *http.Request, workflowExecution shuffle.WorkflowExecution) error { +func handleDefaultExecution(client *http.Client, req *http.Request, workflowExecution shuffle.WorkflowExecution) error { // if no onprem runs (shouldn't happen, but extra check), exit // if there are some, load the images ASAP for the app @@ -1253,16 +1247,14 @@ func handleExecution(client *http.Client, req *http.Request, workflowExecution s shutdown(workflowExecution, "", "", true) } - log.Printf("Startaction: %s", startAction) + log.Printf("DEFAULT EXECUTION Startaction: %s", startAction) + + ctx := context.Background() + setWorkflowExecution(ctx, workflowExecution, false) - // source = parent node, dest = child node - // parent can have more children, child can have more parents - // Process the parents etc. How? for { - handleExecutionResult(workflowExecution) - //fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s/executions/%s/abort", baseUrl, workflowExecution.Workflow.ID, workflowExecution.ExecutionId) - fullUrl := fmt.Sprintf("%s/api/v1/streams", baseUrl) + fullUrl := fmt.Sprintf("%s/api/v1/streams/results", baseUrl) log.Printf("URL: %s", fullUrl) req, err := http.NewRequest( "POST", @@ -1313,6 +1305,8 @@ func handleExecution(client *http.Client, req *http.Request, workflowExecution s shutdown(workflowExecution, "", "", true) } + setWorkflowExecution(ctx, workflowExecution, false) + //handleExecutionResult(workflowExecution) } return nil @@ -1515,53 +1509,6 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { } -func findChildNodes(workflowExecution shuffle.WorkflowExecution, nodeId string) []string { - //log.Printf("\nNODE TO FIX: %s\n\n", nodeId) - allChildren := []string{nodeId} - - // 1. Find children of this specific node - // 2. Find the children of those nodes etc. - for _, branch := range workflowExecution.Workflow.Branches { - if branch.SourceID == nodeId { - //log.Printf("Children: %s", branch.DestinationID) - allChildren = append(allChildren, branch.DestinationID) - - childNodes := findChildNodes(workflowExecution, branch.DestinationID) - for _, bottomChild := range childNodes { - found := false - for _, topChild := range allChildren { - if topChild == bottomChild { - found = true - break - } - } - - if !found { - allChildren = append(allChildren, bottomChild) - } - } - } - } - - // Remove potential duplicates - newNodes := []string{} - for _, tmpnode := range allChildren { - found := false - for _, newnode := range newNodes { - if newnode == tmpnode { - found = true - break - } - } - - if !found { - newNodes = append(newNodes, tmpnode) - } - } - - return newNodes -} - // Will make sure transactions are always ran for an execution. This is recursive if it fails. Allowed to fail up to 5 times func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workflowExecutionId string, actionResult shuffle.ActionResult, resp http.ResponseWriter) { //log.Printf("IN WORKFLOWEXECUTION SUB!") @@ -1574,416 +1521,18 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl return } - log.Printf(`[INFO] Got result %s from %s`, actionResult.Status, actionResult.Action.ID) resultLength := len(workflowExecution.Results) - dbSave := false setExecution := true - if len(actionResult.Action.ExecutionVariable.Name) > 0 { - actionResult.Action.ExecutionVariable.Value = actionResult.Result - - foundIndex := -1 - for i, executionVariable := range workflowExecution.ExecutionVariables { - if executionVariable.Name == actionResult.Action.ExecutionVariable.Name { - foundIndex = i - break - } - } - - if foundIndex >= 0 { - workflowExecution.ExecutionVariables[foundIndex] = actionResult.Action.ExecutionVariable - } else { - workflowExecution.ExecutionVariables = append(workflowExecution.ExecutionVariables, actionResult.Action.ExecutionVariable) - } - } - - actionResult.Action = shuffle.Action{ - AppName: actionResult.Action.AppName, - AppVersion: actionResult.Action.AppVersion, - Label: actionResult.Action.Label, - Name: actionResult.Action.Name, - ID: actionResult.Action.ID, - Parameters: actionResult.Action.Parameters, - ExecutionVariable: actionResult.Action.ExecutionVariable, - } - - if actionResult.Status == "ABORTED" || actionResult.Status == "FAILURE" { - //dbSave = true - - newResults := []shuffle.ActionResult{} - childNodes := []string{} - if workflowExecution.Workflow.Configuration.ExitOnError { - log.Printf("[WARNING] shuffle.Actionresult is %s for node %s in %s. Should set workflowExecution and exit all running functions", actionResult.Status, actionResult.Action.ID, workflowExecution.ExecutionId) - workflowExecution.Status = actionResult.Status - workflowExecution.LastNode = actionResult.Action.ID - // Find underlying nodes and add them - } else { - log.Printf("[WARNING] shuffle.Actionresult is %s for node %s in %s. Continuing anyway because of workflow configuration.", actionResult.Status, actionResult.Action.ID, workflowExecution.ExecutionId) - // Finds ALL childnodes to set them to SKIPPED - // Remove duplicates - //log.Printf("CHILD NODES: %d", len(childNodes)) - childNodes = findChildNodes(*workflowExecution, actionResult.Action.ID) - for _, nodeId := range childNodes { - if nodeId == actionResult.Action.ID { - continue - } - - // 1. Find the action itself - // 2. Create an actionresult - curAction := shuffle.Action{ID: ""} - for _, action := range workflowExecution.Workflow.Actions { - if action.ID == nodeId { - curAction = action - break - } - } - - if len(curAction.ID) == 0 { - log.Printf("Couldn't find subnode %s", nodeId) - continue - } - - resultExists := false - for _, result := range workflowExecution.Results { - if result.Action.ID == curAction.ID { - resultExists = true - break - } - } - - if !resultExists { - // Check parents are done here. Only add it IF all parents are skipped - skipNodeAdd := false - for _, branch := range workflowExecution.Workflow.Branches { - if branch.DestinationID == nodeId { - // If the branch's source node is NOT in childNodes, it's not a skipped parent - sourceNodeFound := false - for _, item := range childNodes { - if item == branch.SourceID { - sourceNodeFound = true - break - } - } - - if !sourceNodeFound { - // FIXME: Shouldn't add skip for child nodes of these nodes. Check if this node is parent of upcoming nodes. - log.Printf("\n\n NOT setting node %s to SKIPPED", nodeId) - skipNodeAdd = true - - if !arrayContains(visited, nodeId) && !arrayContains(executed, nodeId) { - nextActions = append(nextActions, nodeId) - log.Printf("SHOULD EXECUTE NODE %s. Next actions: %s", nodeId, nextActions) - } - break - } - } - } - - if !skipNodeAdd { - newResult := shuffle.ActionResult{ - Action: curAction, - ExecutionId: actionResult.ExecutionId, - Authorization: actionResult.Authorization, - Result: "Skipped because of previous node", - StartedAt: 0, - CompletedAt: 0, - Status: "SKIPPED", - } - - newResults = append(newResults, newResult) - } else { - //log.Printf("\n\nNOT adding %s as skipaction - should add to execute?", nodeId) - //var visited []string - //var executed []string - //var nextActions []string - } - } - } - } - - // Cleans up aborted, and always gives a result - lastResult := "" - // type shuffle.ActionResult struct { - for _, result := range workflowExecution.Results { - if actionResult.Action.ID == result.Action.ID { - continue - } - - if result.Status == "EXECUTING" { - result.Status = actionResult.Status - result.Result = "Aborted because of error in another node (2)" - } - - if len(result.Result) > 0 { - lastResult = result.Result - } - - newResults = append(newResults, result) - } - - workflowExecution.Result = lastResult - workflowExecution.Results = newResults - } - - // FIXME rebuild to be like this or something - // workflowExecution/ExecutionId/Nodes/NodeId - // Find the appropriate action - if len(workflowExecution.Results) > 0 { - // FIXME - skip := false - found := false - outerindex := 0 - for index, item := range workflowExecution.Results { - if item.Action.ID == actionResult.Action.ID { - found = true - - if item.Status == actionResult.Status { - skip = true - } - - outerindex = index - break - } - } - - if skip { - //log.Printf("Both are %s. Skipping this node", item.Status) - } else if found { - // If result exists and execution variable exists, update execution value - //log.Printf("Exec var backend: %s", workflowExecution.Results[outerindex].Action.ExecutionVariable.Name) - // Finds potential execution arguments - actionVarName := workflowExecution.Results[outerindex].Action.ExecutionVariable.Name - if len(actionVarName) > 0 { - log.Printf("EXECUTION VARIABLE LOCAL: %s", actionVarName) - for index, execvar := range workflowExecution.ExecutionVariables { - if execvar.Name == actionVarName { - // Sets the value for the variable - workflowExecution.ExecutionVariables[index].Value = actionResult.Result - break - } - } - } - - log.Printf("[INFO] Updating %s in workflow %s from %s to %s", actionResult.Action.ID, workflowExecution.ExecutionId, workflowExecution.Results[outerindex].Status, actionResult.Status) - workflowExecution.Results[outerindex] = actionResult - } else { - workflowExecution.Results = append(workflowExecution.Results, actionResult) - log.Printf("[INFO] Setting value (1) of %s in execution %s to %s. New result length: %d", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status, len(workflowExecution.Results)) - } - } else { - workflowExecution.Results = append(workflowExecution.Results, actionResult) - log.Printf("[INFO] Setting value (2) of %s in execution %s to %s. New result length: %d", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status, len(workflowExecution.Results)) - } - - if actionResult.Status == "SKIPPED" { - log.Printf("\n\n[INFO] Handling special case for SKIPPED!\n\n") - childNodes := findChildNodes(*workflowExecution, actionResult.Action.ID) - for _, nodeId := range childNodes { - if nodeId == actionResult.Action.ID { - continue - } - - // 1. Find the action itself - // 2. Create an actionresult - curAction := shuffle.Action{ID: ""} - for _, action := range workflowExecution.Workflow.Actions { - if action.ID == nodeId { - curAction = action - break - } - } - - if len(curAction.ID) == 0 { - log.Printf("Couldn't find subnode %s", nodeId) - continue - } - - resultExists := false - for _, result := range workflowExecution.Results { - if result.Action.ID == curAction.ID { - resultExists = true - break - } - } - - if !resultExists { - // Check parents are done here. Only add it IF all parents are skipped - skipNodeAdd := false - for _, branch := range workflowExecution.Workflow.Branches { - if branch.DestinationID == nodeId { - // If the branch's source node is NOT in childNodes, it's not a skipped parent - sourceNodeFound := false - for _, item := range childNodes { - if item == branch.SourceID { - sourceNodeFound = true - break - } - } - - if !sourceNodeFound { - log.Printf("[INFO] Not setting node %s to SKIPPED", nodeId) - skipNodeAdd = true - break - } - } - } - - if !skipNodeAdd { - newAction := shuffle.Action{ - AppName: curAction.AppName, - AppVersion: curAction.AppVersion, - Label: curAction.Label, - Name: curAction.Name, - ID: curAction.ID, - } - - newResult := shuffle.ActionResult{ - Action: newAction, - ExecutionId: actionResult.ExecutionId, - Authorization: actionResult.Authorization, - Result: "Skipped because of previous node", - StartedAt: 0, - CompletedAt: 0, - Status: "SKIPPED", - } - - workflowExecution.Results = append(workflowExecution.Results, newResult) - } - } - } - } - - // FIXME: Have a check for skippednodes and their parents - /* - for resultIndex, result := range workflowExecution.Results { - if result.Status != "SKIPPED" { - continue - } - - // Checks if all parents are skipped or failed. - // Otherwise removes them from the results - for _, branch := range workflowExecution.Workflow.Branches { - if branch.DestinationID == result.Action.ID { - for _, subresult := range workflowExecution.Results { - if subresult.Action.ID == branch.SourceID { - if subresult.Status != "SKIPPED" && subresult.Status != "FAILURE" { - //log.Printf("SUBRESULT PARENT STATUS: %s", subresult.Status) - //log.Printf("Should remove resultIndex: %d", resultIndex) - - // FIXME: Reinstate this? - //workflowExecution.Results = append(workflowExecution.Results[:resultIndex], workflowExecution.Results[resultIndex+1:]...) - _ = resultIndex - - break - } - } - } - } - } - } - - log.Printf("NEW LENGTH: %d", len(workflowExecution.Results)) - */ - - extraInputs := 0 - for _, trigger := range workflowExecution.Workflow.Triggers { - if trigger.Name == "User Input" && trigger.AppName == "User Input" { - extraInputs += 1 - } else if trigger.Name == "Shuffle Workflow" && trigger.AppName == "Shuffle Workflow" { - extraInputs += 1 - } - } - - //log.Printf("EXTRA: %d", extraInputs) - //log.Printf("LENGTH: %d - %d", len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)+extraInputs) - - if len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions)+extraInputs { - //log.Printf("\nIN HERE WITH RESULTS %d vs %d\n", len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)+extraInputs) - finished := true - lastResult := "" - - // Doesn't have to be SUCCESS and FINISHED everywhere anymore. - skippedNodes := false - for _, result := range workflowExecution.Results { - if result.Status == "EXECUTING" { - finished = false - break - } - - // FIXME: Check if ALL parents are skipped or if its just one. Otherwise execute it - if result.Status == "SKIPPED" { - skippedNodes = true - - // Checks if all parents are skipped or failed. Otherwise removes them from the results - for _, branch := range workflowExecution.Workflow.Branches { - if branch.DestinationID == result.Action.ID { - for _, subresult := range workflowExecution.Results { - if subresult.Action.ID == branch.SourceID { - if subresult.Status != "SKIPPED" && subresult.Status != "FAILURE" { - //log.Printf("SUBRESULT PARENT STATUS: %s", subresult.Status) - //log.Printf("Should remove resultIndex: %d", resultIndex) - finished = false - break - } - } - } - } - - if !finished { - break - } - } - } - - lastResult = result.Result - } - - // FIXME: Handle skip nodes - change status? - _ = skippedNodes - - if finished { - dbSave = true - log.Printf("[INFO] Execution of %s finished.", workflowExecution.ExecutionId) - //log.Println("Might be finished based on length of results and everything being SUCCESS or FINISHED - VERIFY THIS. Setting status to finished.") - - workflowExecution.Result = lastResult - workflowExecution.Status = "FINISHED" - workflowExecution.CompletedAt = int64(time.Now().Unix()) - if workflowExecution.LastNode == "" { - workflowExecution.LastNode = actionResult.Action.ID - } - - } - } - - // FIXME - why isn't this how it works otherwise, wtf? - //workflow, err := getWorkflow(workflowExecution.Workflow.ID) - //newActions := []Action{} - //for _, action := range workflowExecution.Workflow.Actions { - // log.Printf("Name: %s, Env: %s", action.Name, action.Environment) - //} - - tmpJson, err := json.Marshal(workflowExecution) - if err == nil { - if len(tmpJson) >= 1048487 { - dbSave = true - log.Printf("[ERROR] Result length is too long! Need to reduce result size") - - // Result string `json:"result" datastore:"result,noindex"` - // Arbitrary reduction size - maxSize := 500000 - newResults := []shuffle.ActionResult{} - for _, item := range workflowExecution.Results { - if len(item.Result) > maxSize { - item.Result = "[ERROR] Result too large to handle (https://github.com/frikky/shuffle/issues/171)" - } - - newResults = append(newResults, item) - } - - workflowExecution.Results = newResults - } + workflowExecution, dbSave, err := shuffle.ParsedExecutionResult(ctx, *workflowExecution, actionResult) + if err != nil { + log.Printf("[ERROR] Failed execution of parsedexecution: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting execution"}`))) + return } + //log.Printf(`[INFO] Got result %s from %s`, actionResult.Status, actionResult.Action.ID) + //dbSave := false if len(results) != len(workflowExecution.Results) { log.Printf("\n\n[WARNING] There may have been an issue in transaction queue. Result lengths: %d vs %d. Should check which exists the base results, but not in entire execution, then append.\n\n", len(results), len(workflowExecution.Results)) @@ -2047,6 +1596,11 @@ func getWorkflowExecution(ctx context.Context, id string) (*shuffle.WorkflowExec } func sendResult(workflowExecution shuffle.WorkflowExecution, data []byte) { + if workflowExecution.ExecutionSource == "default" { + log.Printf("Not sending backend info since source is default") + return + } + fullUrl := fmt.Sprintf("%s/api/v1/streams", baseUrl) req, err := http.NewRequest( "POST", @@ -2083,13 +1637,13 @@ func validateFinished(workflowExecution shuffle.WorkflowExecution) { //log.Printf("[FINISHED] Should send full result to %s", baseUrl) //data = fmt.Sprintf(`{"execution_id": "%s", "authorization": "%s"}`, executionId, authorization) - data, err := json.Marshal(workflowExecution) + shutdownData, err := json.Marshal(workflowExecution) if err != nil { log.Printf("[ERROR] Failed to unmarshal data for backend") shutdown(workflowExecution, "", "", true) } - sendResult(workflowExecution, data) + sendResult(workflowExecution, shutdownData) } } @@ -2153,6 +1707,7 @@ func setWorkflowExecution(ctx context.Context, workflowExecution shuffle.Workflo handleExecutionResult(workflowExecution) validateFinished(workflowExecution) + if dbSave { shutdown(workflowExecution, "", "", false) } @@ -2339,8 +1894,8 @@ func main() { } } - log.Printf("Environments: %s. 1 = webserver, 0 or >1 = default", environments) - if len(environments) == 1 { //&& workflowExecution.ExecutionSource != "default" { + log.Printf("Environments: %s. Source: %s. 1 = webserver, 0 or >1 = default", environments, workflowExecution.ExecutionSource) + if len(environments) == 1 && workflowExecution.ExecutionSource != "default" { log.Printf("[INFO] Running OPTIMIZED execution (not manual)") listener := webserverSetup(workflowExecution) err := executionInit(workflowExecution) @@ -2360,10 +1915,13 @@ func main() { //wg.Add(1) //wg.Wait() } else { - log.Printf("[INFO] Running NON-OPTIMIZED execution for type %s with %d environments", workflowExecution.ExecutionSource, len(environments)) - + log.Printf("[INFO] Running NON-OPTIMIZED execution for type %s with %d environments. This only happens when ran manually. Status: %s", workflowExecution.ExecutionSource, len(environments), workflowExecution.Status) + //err := executionInit(workflowExecution) + //if err != nil { + // log.Printf("[INFO] Workflow setup failed: %s", workflowExecution.ExecutionId, err) + // shutdown(workflowExecution, "", "", true) + //} } - } if workflowExecution.Status == "FINISHED" || workflowExecution.Status == "SUCCESS" { @@ -2373,7 +1931,7 @@ func main() { if workflowExecution.Status == "EXECUTING" || workflowExecution.Status == "RUNNING" { //log.Printf("Status: %s", workflowExecution.Status) - err = handleExecution(client, req, workflowExecution) + err = handleDefaultExecution(client, req, workflowExecution) if err != nil { log.Printf("[INFO] Workflow %s is finished: %s", workflowExecution.ExecutionId, err) shutdown(workflowExecution, "", "", true) From 033124b6ecc3ba7795b61c7b8d41a19951be53cc Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 13 Apr 2021 21:52:32 +0200 Subject: [PATCH 14/96] Bugfixes for workflows and executions --- backend/app_sdk/app_base.py | 11 +++++++++-- backend/go-app/go.sum | 2 ++ frontend/src/components/ConfigureWorkflow.jsx | 2 +- frontend/src/views/Admin.jsx | 2 +- frontend/src/views/AngularWorkflow.jsx | 3 +-- frontend/src/views/Apps.jsx | 4 ++-- frontend/src/views/Workflows.jsx | 4 ++-- 7 files changed, 18 insertions(+), 10 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 29aa07cd..0fd4c9c2 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -1592,16 +1592,15 @@ class AppBase: "matches regex", ] - # FIXME - what should I do here? if not condition["condition"]["value"] in available_checks: self.logger.warning("Skipping %s %s %s because %s is invalid." % (sourcevalue, condition["condition"]["value"], destinationvalue, condition["condition"]["value"])) continue #print(destinationvalue) # NEGATE - validation = run_validation(sourcevalue, condition["condition"]["value"], destinationvalue) # Configuration = negated because of WorkflowAppActionParam.. + validation = run_validation(sourcevalue, condition["condition"]["value"], destinationvalue) try: if condition["condition"]["configuration"]: validation = not validation @@ -1622,6 +1621,14 @@ class AppBase: # THE START IS ACTUALLY RIGHT HERE :O # Checks whether conditions are met, otherwise set branchcheck, tmpresult = check_branch_conditions(action, fullexecution) + if isinstance(tmpresult, object) or isinstance(tmpresult, list): + print("Fixing branch return as object -> string") + try: + tmpresult = tmpresult.replace("'", "\"") + tmpresult = json.dumps(tmpresult) + except json.decoder.JSONDecodeError as e: + print(f"[WARNING] Failed condition parsing {tmpresult} to string") + if not branchcheck: self.logger.info("Failed one or more branch conditions.") action_result["result"] = tmpresult diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 1360ecc7..5890f5e5 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -102,6 +102,8 @@ github.com/frikky/shuffle-shared v0.0.23 h1:Pnlc2M6fHnFRLFd5K1iLTVv4/t4P04Ri1GJ5 github.com/frikky/shuffle-shared v0.0.23/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= github.com/frikky/shuffle-shared v0.0.27 h1:BbibbAv3a5GWR/DfaoSC4D9+fh2cwSEvn9H+EVfd7BM= github.com/frikky/shuffle-shared v0.0.27/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= +github.com/frikky/shuffle-shared v0.0.28 h1:VQqL3+ePwKSUxCOiCC8DpOEgbb2GhXI8XzFB/YlHbps= +github.com/frikky/shuffle-shared v0.0.28/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= github.com/getkin/kin-openapi v0.8.0 h1:a6TQjTqwkyscC4/hShJX7WhCVE+4bi9lzw61XHQW5hE= github.com/getkin/kin-openapi v0.8.0/go.mod h1:zZQMFkVgRHCdhgb6ihCTIo9dyDZFvX0k/xAKqw1FhPw= github.com/getkin/kin-openapi v0.52.0 h1:6WqsF5d6PfJ8AscdD+9Rtb2RP2iBWyC7V6GcjssWg7M= diff --git a/frontend/src/components/ConfigureWorkflow.jsx b/frontend/src/components/ConfigureWorkflow.jsx index 5dba1179..215617ab 100644 --- a/frontend/src/components/ConfigureWorkflow.jsx +++ b/frontend/src/components/ConfigureWorkflow.jsx @@ -81,7 +81,7 @@ const Workflow = (props) => { "app": {}, } - const app = apps.find(app => app.name === action.app_name && (app.app_version === action.app_version || app.loop_versions.includes(action.app_version))) + const app = apps.find(app => app.name === action.app_name && (app.app_version === action.app_version || (app.loop_versions !== null && app.loop_versions.includes(action.app_version)))) if (app === undefined || app === null) { console.log("App not found!") diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index 3749f4d0..20a13eda 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -1111,7 +1111,7 @@ const Admin = (props) => { const generateApikey = (user) => { - const userId = isCloud ? user.username : user.id + const userId = user.id const data = { "user_id": userId } fetch(globalUrl + "/api/v1/generateapikey", { diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 9b018e62..40d89fb4 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1326,8 +1326,7 @@ const AngularWorkflow = (props) => { return } - //console.log(apps) - const curapp = apps.find(a => a.name === curaction.app_name && (a.app_version === curaction.app_version || a.loop_versions.includes(curaction.app_version))) + const curapp = apps.find(a => a.name === curaction.app_name && ((a.app_version === curaction.app_version || (a.loop_versions !== null && a.loop_versions.includes(curaction.app_version))))) if (!curapp || curapp === undefined) { alert.error(`App ${curaction.app_name}:${curaction.app_version} not found. Is it activated?`) diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index d13c068a..bd344504 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -1461,7 +1461,7 @@ const Apps = (props) => { style={{backgroundColor: inputColor}} variant="outlined" margin="normal" - defaultValue={userdata.active_org.defaults.app_download_repo !== undefined && userdata.active_org.defaults.app_download_repo.length > 0 ? userdata.active_org.defaults.app_download_repo : "https://github.com/frikky/shuffle-apps"} + defaultValue={"https://github.com/frikky/shuffle-apps"} InputProps={{ style:{ color: "white", @@ -1479,7 +1479,7 @@ const Apps = (props) => { style={{backgroundColor: inputColor}} variant="outlined" margin="normal" - defaultValue={userdata.active_org.defaults.app_download_branch !== undefined && userdata.active_org.defaults.app_download_branch.length > 0 ? userdata.active_org.defaults.app_download_branch : downloadBranch} + defaultValue={downloadBranch} InputProps={{ style:{ color: "white", diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index bfca455f..d21af19a 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -1802,7 +1802,7 @@ const Workflows = (props) => { style={{backgroundColor: inputColor}} variant="outlined" margin="normal" - defaultValue={userdata.active_org.defaults.workflow_download_repo !== undefined && userdata.active_org.defaults.workflow_download_repo.length > 0 ? userdata.active_org.defaults.workflow_download_repo : downloadUrl} + defaultValue={downloadUrl} InputProps={{ style:{ color: "white", @@ -1821,7 +1821,7 @@ const Workflows = (props) => { style={{backgroundColor: inputColor}} variant="outlined" margin="normal" - defaultValue={userdata.active_org.defaults.workflow_download_branch !== undefined && userdata.active_org.defaults.workflow_download_branch.length > 0 ? userdata.active_org.defaults.workflow_download_branch : downloadBranch} + defaultValue={downloadBranch} InputProps={{ style:{ color: "white", From 00323689e2362a0397818d3d029252f4afbb06f7 Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 18 Apr 2021 20:48:11 +0200 Subject: [PATCH 15/96] Bugfixes to admin panel, workflow view and app creator --- .env | 2 +- backend/go-app/go.mod | 2 +- backend/go-app/go.sum | 2 + backend/go-app/walkoff.go | 15 +++-- docker-compose.yml | 4 +- frontend/src/views/Admin.jsx | 87 +++++++++++++------------- frontend/src/views/AngularWorkflow.jsx | 3 +- frontend/src/views/AppCreator.jsx | 11 +++- 8 files changed, 71 insertions(+), 55 deletions(-) diff --git a/.env b/.env index ac31f93a..1b646f6f 100644 --- a/.env +++ b/.env @@ -31,7 +31,7 @@ FRONTEND_PORT=3001 FRONTEND_PORT_HTTPS=3443 # CHANGE THIS IF YOU WANT GOOD LOCAL EXECUTIONS: OUTER_HOSTNAME=shuffle-backend -DB_LOCATION=./shuffle-database-new +DB_LOCATION=./shuffle-database # Proxy configurations. SHUFFLE_PASS_WORKER_PROXY must be FALSE to not pass the proxy information to sub-apps. # PS: It will skip proxy for diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index c654792d..eced0ea8 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -20,7 +20,7 @@ require ( github.com/docker/go-connections v0.4.0 github.com/docker/go-units v0.4.0 // indirect github.com/frikky/kin-openapi v0.38.0 - github.com/frikky/shuffle-shared v0.0.28 + github.com/frikky/shuffle-shared v0.0.32 github.com/ghodss/yaml v1.0.0 github.com/go-git/go-billy/v5 v5.0.0 github.com/go-git/go-git/v5 v5.0.0 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 5890f5e5..7e5d3932 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -104,6 +104,8 @@ github.com/frikky/shuffle-shared v0.0.27 h1:BbibbAv3a5GWR/DfaoSC4D9+fh2cwSEvn9H+ github.com/frikky/shuffle-shared v0.0.27/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= github.com/frikky/shuffle-shared v0.0.28 h1:VQqL3+ePwKSUxCOiCC8DpOEgbb2GhXI8XzFB/YlHbps= github.com/frikky/shuffle-shared v0.0.28/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= +github.com/frikky/shuffle-shared v0.0.32 h1:Uy/zcAetSVYtRr3HEkUb7aE7Ggm0oSFxVeUNsi6q4uc= +github.com/frikky/shuffle-shared v0.0.32/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= github.com/getkin/kin-openapi v0.8.0 h1:a6TQjTqwkyscC4/hShJX7WhCVE+4bi9lzw61XHQW5hE= github.com/getkin/kin-openapi v0.8.0/go.mod h1:zZQMFkVgRHCdhgb6ihCTIo9dyDZFvX0k/xAKqw1FhPw= github.com/getkin/kin-openapi v0.52.0 h1:6WqsF5d6PfJ8AscdD+9Rtb2RP2iBWyC7V6GcjssWg7M= diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 83ca6c12..ef777845 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -1366,12 +1366,15 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { return } - // FIXME - have a check for org etc too.. - if user.Id != workflow.Owner { - log.Printf("Wrong user (%s) for workflow %s", user.Username, workflow.ID) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return + if user.Id != workflow.Owner || len(user.Id) == 0 { + if workflow.OrgId == user.ActiveOrg.Id && user.Role == "admin" { + log.Printf("[INFO] User %s is deleting workflow %s as admin. Owner: %s", user.Username, workflow.ID, workflow.Owner) + } else { + log.Printf("Wrong user (%s) for workflow %s", user.Username, workflow.ID) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } } // Clean up triggers and executions diff --git a/docker-compose.yml b/docker-compose.yml index b2de1a33..55f7a079 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: frontend: #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.73 + image: ghcr.io/frikky/shuffle-frontend:0.8.74 container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -17,7 +17,7 @@ services: - backend backend: #build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.73 + image: ghcr.io/frikky/shuffle-backend:0.8.74 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index 20a13eda..60d7bb3b 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -373,7 +373,11 @@ const Admin = (props) => { .then(response => response.json().then(responseJson => { if (responseJson["success"] === false) { - alert.error("Failed setting new password") + if (responseJson.reason !== undefined) { + alert.error(responseJson.reason) + } else { + alert.error("Failed setting new password") + } } else { alert.success("Successfully updated password!") setSelectedUserModalOpen(false) @@ -387,7 +391,7 @@ const Admin = (props) => { const deleteUser = (data) => { // Just use this one? - const userId = isCloud ? data.username : data.id + const userId = data.id const url = globalUrl + '/api/v1/users/' + userId fetch(url, { @@ -1076,7 +1080,6 @@ const Admin = (props) => { const setUser = (userId, field, value) => { const data = { "user_id": userId } data[field] = value - console.log("DATA: ", data) fetch(globalUrl + "/api/v1/users/updateuser", { method: 'PUT', @@ -1238,37 +1241,41 @@ const Admin = (props) => { }, }} > - Edit user + Editing {selectedUser.username} -
- setNewPassword(e.target.value)} - /> - -
+ {isCloud ? + null + : +
+ setNewPassword(e.target.value)} + /> + +
+ }
: -
- - - Loading apps - -
+ apps.length > 0 ? +
+ + Couldn't find app. Is it active? + +
+ : +
+ + + Loading apps + +
}
@@ -5385,25 +5407,27 @@ const AngularWorkflow = (props) => {
{executionButton}
- - { - setExecutionText(e.target.value) - }} - /> - + {workflow.public ? null : + + { + setExecutionText(e.target.value) + }} + /> + + } - @@ -5411,7 +5435,7 @@ const AngularWorkflow = (props) => { {workflow.public ? - @@ -5627,16 +6082,22 @@ const AngularWorkflow = (props) => { ) } - const RightSideBar = () => { + const RightSideBar = (props) => { if (!rightSideBarOpen) { return null } if (Object.getOwnPropertyNames(selectedAction).length > 0) { - //console.time('ACTIONSTART') + if (Object.getOwnPropertyNames(selectedAction).length === 0) { + return null + } + + console.time('ACTIONSTART') return( -
- {appApiView} +
{ + console.log("LOADED RIGHTSIDE") + }}> + HELO
) } else if (Object.getOwnPropertyNames(selectedTrigger).length > 0) { @@ -5805,11 +6266,11 @@ const AngularWorkflow = (props) => { var copyText = document.getElementById(elementName); if (copyText !== null && copyText !== undefined) { navigator.clipboard.writeText(JSON.stringify(copy)) - copyText.select(); - copyText.setSelectionRange(0, 99999); /* For mobile devices */ + copyText.select() + copyText.setSelectionRange(0, 99999) /* For mobile devices */ /* Copy the text inside the text field */ - document.execCommand("copy"); + document.execCommand("copy") alert.success("Copied data") } } @@ -5849,10 +6310,10 @@ const AngularWorkflow = (props) => { to_be_copied.replaceAll(" ", "_") const elementName = "copy_element_shuffle" - var copyText = document.getElementById(elementName); + var copyText = document.getElementById(elementName) if (copyText !== null && copyText !== undefined) { navigator.clipboard.writeText(to_be_copied) - copyText.select(); + copyText.select() copyText.setSelectionRange(0, 99999); /* For mobile devices */ /* Copy the text inside the text field */ @@ -6423,7 +6884,45 @@ const AngularWorkflow = (props) => { />
{executionModal} - + + {rightSideBarOpen && Object.getOwnPropertyNames(selectedAction).length !== 0 ? +
+ +
+ : + + }
@@ -6435,6 +6934,7 @@ const AngularWorkflow = (props) => {
*/ + const executionVariableModal = executionVariablesModalOpen ? { {/*selectedApp.link.length > 0 ?
: null*/}
{selectedApp.authentication.parameters.map((data, index) => { - return ( -
- - {data.name} - { - authenticationOption.fields[data.name] = event.target.value - }} - /> -
- ) - })} + console.log("AUTH: ", data) + + return ( +
+ + {data.name} + + {data.schema !== undefined && data.schema !== null && data.schema.type === "bool" ? + + : + { + authenticationOption.fields[data.name] = event.target.value + }} + /> + } +
+ ) + })}
- const newView = //isLoggedIn ? + const newView =
{leftView} @@ -7152,45 +7226,40 @@ const AngularWorkflow = (props) => { />
{executionModal} + - -
- : - - } + setVariablesModalOpen={setVariablesModalOpen} + setLastSaved={setLastSaved} + setCodeModalOpen={setCodeModalOpen} + selectedNameChange={selectedNameChange} + rightsidebarStyle={rightsidebarStyle} + showEnvironment={showEnvironment} + selectedActionEnvironment={selectedActionEnvironment} + environments={environments} + setNewSelectedAction={setNewSelectedAction} + sortByKey={sortByKey} + + appApiViewStyle={appApiViewStyle} + globalUrl={globalUrl} + setSelectedActionEnvironment={setSelectedActionEnvironment} + requiresAuthentication={requiresAuthentication} + />
@@ -7723,13 +7792,54 @@ const AngularWorkflow = (props) => {
+ // Awful way of handling scroll + if (scrollConfig !== undefined && setScrollConfig !== undefined && Object.getOwnPropertyNames(selectedAction).length !== 0) { + //console.log("SET CONFIG: ", scrollConfig) + const rightSideActionView = document.getElementById("rightside_actions") + if (rightSideActionView !== undefined && rightSideActionView !== null) { + //console.log("FOUND RIGHTSIDE: ", rightSideActionView.scrollTop, scrollConfig) + if (scrollConfig.top !== 0 && scrollConfig.top !== undefined && scrollConfig.top !== 0) { + //console.log("SCROLL IS NOT 0: ", scrollConfig.top) + //rightSideActionView.scrollTop = scrollConfig.top + setTimeout(() => { + scroller.scrollTo('elements_wrapper', { + containerId: 'rightside_actions', + offset: scrollConfig.top, + }) + + if (scrollConfig.selected !== undefined && scrollConfig.selected !== null) { + const selectedField = document.getElementById(scrollConfig.selected) + if (selectedField !== undefined && selectedField !== null) { + selectedField.focus() + //const val = selectedField.value + //console.log("VAL: ", val) + //selectedField.value = '' + //selectedField.value = val + } + } + }, 5) + } else { + //console.log("SCROLL IS 0: ", scrollConfig.top, rightSideActionView.scrollTop) + if (rightSideActionView.scrollTop !== scrollConfig.top) { + setScrollConfig({ + top: rightSideActionView.scrollTop, + left: 0, + selected: "", + }) + } + } + } + } + return (
- - {loadedCheck} + + + {loadedCheck} +
) } diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index b00ce35a..38943545 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -84,7 +84,7 @@ export const GetIconInfo = (action) => { {"key": "download", "values": ["get", "download", "return", "hello_world", "curl",]}, {"key": "search", "values": ["search", "find"]}, {"key": "delete", "values": ["delete", "remove"]}, - {"key": "send", "values": ["send", "dispatch", "mail", "forward", "post", "submit"]}, + {"key": "send", "values": ["send", "dispatch", "mail", "forward", "post", "submit", "mark"]}, {"key": "repeat", "values": ["repeat", "retry", "pause",]}, {"key": "execute", "values": ["execute", "run", "play", "raise",]}, {"key": "extract", "values": ["extract", "unpack", "decompress"]}, From a39370a5fce66f9c6854f67766e7bc4c4e52f315 Mon Sep 17 00:00:00 2001 From: frikky Date: Wed, 12 May 2021 11:54:55 +0200 Subject: [PATCH 34/96] Added copy and delete buttons --- frontend/src/components/ParsedAction.jsx | 10 +- frontend/src/views/AngularWorkflow.jsx | 181 +++++++++++++++++++---- frontend/src/views/AppCreator.jsx | 7 +- 3 files changed, 167 insertions(+), 31 deletions(-) diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index 4b0120f3..d9052c6d 100644 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -312,10 +312,13 @@ const ParsedAction = (props) => { // PARAM FIX - Gonna use the ID field, even though it's a hack const paramcheck = selectedAction.parameters.find(param => param.name === "body") if (paramcheck !== undefined) { + // Escapes all double quotes + const toReplace = event.target.value.trim().replaceAll("\\\"", "\"").replaceAll("\"", "\\\"") + console.log("REPLACE WITH: ", toReplace) if (paramcheck["value_replace"] === undefined || paramcheck["value_replace"] === null) { paramcheck["value_replace"] = [{ "key": data.name, - "value": event.target.value, + "value": toReplace, }] console.log("IN IF: ", paramcheck) @@ -325,10 +328,10 @@ const ParsedAction = (props) => { if (subparamindex === -1) { paramcheck["value_replace"].push({ "key": data.name, - "value": event.target.value, + "value": toReplace, }) } else { - paramcheck["value_replace"][subparamindex]["value"] = event.target.value + paramcheck["value_replace"][subparamindex]["value"] = toReplace } console.log("IN ELSE: ", paramcheck) @@ -868,6 +871,7 @@ const ParsedAction = (props) => { paramcheck["value_replace"][subparamindex]["value"] += toComplete } } + selectedActionParameters[count]["value_replace"] = paramcheck selectedAction.parameters[count]["value_replace"] = paramcheck diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 5ae41a65..a755606a 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -156,6 +156,7 @@ const AngularWorkflow = (props) => { left: 0, selected: "", }) + const [history, setHistory] = React.useState([]) const [appAuthentication, setAppAuthentication] = React.useState([]); const [variablesModalOpen, setVariablesModalOpen] = React.useState(false); @@ -233,8 +234,6 @@ const AngularWorkflow = (props) => { const [workflowExecutions, setWorkflowExecutions] = React.useState([]); const [defaultEnvironmentIndex, setDefaultEnvironmentIndex] = React.useState(0) - useTraceUpdate(props) - // This should all be set once, not on every iteration // Use states and don't update lol const cloudSyncEnabled = props.userdata !== undefined && props.userdata.active_org !== null && props.userdata.active_org !== undefined ? props.userdata.active_org.cloud_sync === true : false @@ -250,18 +249,16 @@ const AngularWorkflow = (props) => { } }) - const [elements, setElements] = useState([]) // No point going as fast, as the nodes aren't realtime anymore, but bulk updated. // Set it from 2500 to 6000 to reduce overall load const { start, stop } = useInterval({ - duration: 6000, + duration: 3000, startImmediate: false, callback: () => { fetchUpdates() } }) - console.log("STATE UPDATE?") const getAvailableWorkflows = (trigger_index) => { fetch(globalUrl+"/api/v1/workflows", { @@ -1251,6 +1248,12 @@ const AngularWorkflow = (props) => { const onUnselect = (event) => { console.time("UNSELECT") + const nodedata = event.target.data() + if (nodedata.app_name === undefined && nodedata.source === undefined) { + return + } + + // Attempt at rewrite of name in other actions in following nodes. // Should probably be done in the onBlur for the textfield instead /* @@ -1321,7 +1324,7 @@ const AngularWorkflow = (props) => { const triggercheck = workflow.triggers.find(trigger => trigger.id === event.target.data()["source"]) if (triggercheck === undefined) { */ - console.log(event.target.data()) + //console.log(event.target.data()) if (event.target.data().decorator) { alert.info("This edge can't be edited.") } else { @@ -1398,6 +1401,7 @@ const AngularWorkflow = (props) => { nodedata.app_name !== "Testing" && nodedata.app_name !== "Shuffle Workflow" && nodedata.app_name !== "User Input" && + nodedata.app_name !== "Webhook" && nodedata.app_name !== "Schedule" && nodedata.app_name !== "Email") || nodedata.isStartNode) ) { @@ -1438,7 +1442,7 @@ const AngularWorkflow = (props) => { }, } - cy.add(decoratorNode) + cy.add(decoratorNode).unselectify() } else { console.log("Node already exists - don't add descriptor node") } @@ -1561,11 +1565,123 @@ const AngularWorkflow = (props) => { // https://stackoverflow.com/questions/16677856/cy-onselect-callback-only-once const onNodeSelect = (event, newAppAuth) => { const data = event.target.data() - console.log("NODE: ", data) setLastSaved(false) if (data.isButton) { console.log("BUTTON CLICKED: ", data) + if (data.buttonType === "delete") { + console.log("DELETE!") + const parentNode = cy.getElementById(data.attachedTo) + if (parentNode !== null && parentNode !== undefined) { + parentNode.remove() + } + + //for (var key in allNodes) { + // const currentNode = allNodes[key] + // if (currentNode.data.attachedTo === data.attachedTo) { + // cy.getElementById(currentNode.data.id).remove() + // } + //} + } else if (data.buttonType === "copy") { + console.log("COPY!") + // 1. Find parent + // 2. Find branches for parent + // 3. Make a new node that's moved a little bit + const parentNode = cy.getElementById(data.attachedTo) + if (parentNode !== null && parentNode !== undefined) { + //parentNode.data() + var newNodeData = JSON.parse(JSON.stringify(parentNode.data())) + newNodeData.id = uuid.v4() + newNodeData.position = { + "x": newNodeData.position.x+100, + "y": newNodeData.position.y+100, + } + newNodeData.isStartNode = false + newNodeData.errors = [] + newNodeData.is_valid = true + newNodeData.isValid = true + + cy.add({ + group: 'nodes', + data: newNodeData, + position: newNodeData.position, + }) + + // Readding the icon after moving the node + //console.log("Node wasn't found") + if (newNodeData.app_name !== "Testing" || newNodeData.app_name !== "Shuffle Workflow") { + } else { + const iconInfo = GetIconInfo(newNodeData) + const svg_pin = `` + const svgpin_Url = encodeURI("data:image/svg+xml;utf-8," + svg_pin) + + const offset = newNodeData.isStartNode ? 36 : 44 + //console.log(event.target.position()) + const decoratorNode = { + position: { + x: newNodeData.position.x+offset, + y: newNodeData.position.y+offset, + }, + locked: true, + data: { + "isDescriptor": true, + "isValid": true, + "is_valid": true, + "label": "", + "image": svgpin_Url, + "imageColor": iconInfo.iconBackgroundColor, + "attachedTo": newNodeData.id, + }, + } + + cy.add(decoratorNode).unselectify() + } + + workflow.actions.push(newNodeData) + + const sourcebranches = workflow.branches.filter(foundbranch => foundbranch.source_id === parentNode.data("id")) + const destinationbranches = workflow.branches.filter(foundbranch => foundbranch.destination_id === parentNode.data("id")) + + //for (var key in sourcebranches) { + // var newbranch = JSON.parse(JSON.stringify(sourcebranches[key])) + // newbranch.id = uuid.v4() + // newbranch.source_id = newNodeData.id + // cy.add({ + // group: "edges", + // data: newbranch, + // }) + //} + + for (var key in sourcebranches) { + var newbranch = JSON.parse(JSON.stringify(sourcebranches[key])) + newbranch.id = uuid.v4() + newbranch.source_id = newNodeData.id + + newbranch._id = newbranch.id + newbranch.source = newbranch.source_id + newbranch.target = newbranch.destination_id + cy.add({ + group: "edges", + data: newbranch, + }) + } + + for (var key in destinationbranches) { + var newbranch = JSON.parse(JSON.stringify(destinationbranches[key])) + newbranch.id = uuid.v4() + newbranch.destination_id = newNodeData.id + + newbranch._id = newbranch.id + newbranch.source = newbranch.source_id + newbranch.target = newbranch.destination_id + cy.add({ + group: "edges", + data: newbranch, + }) + } + } + } + event.target.unselect() return } else if (data.isDescriptor) { @@ -1574,7 +1690,7 @@ const AngularWorkflow = (props) => { return } - + console.log("NODE: ", data) //const node = cy.getElementById(data.id) //if (node.length > 0) { // node.addClass('shuffle-hover-highlight') @@ -1815,6 +1931,12 @@ const AngularWorkflow = (props) => { workflow.branches.push(newbranch) setWorkflow(workflow) } + + history.push({ + "type": "edge", + "action": "added", + "data": edge, + }) } const onNodeAdded = (event) => { @@ -1895,6 +2017,12 @@ const AngularWorkflow = (props) => { //} else { // //console.log("Shouldnt re-add info? ") //} + + history.push({ + "type": "node", + "action": "added", + "data": nodedata, + }) } const onEdgeRemoved = (event) => { @@ -2252,9 +2380,9 @@ const AngularWorkflow = (props) => { if (parentNode.data('isButton') || parentNode.data('buttonId')) return - parentNode.lock() + //parentNode.lock() const px = parentNode.position('x') - 65 - const py = parentNode.position('y') + 25 + const py = parentNode.position('y') - 25 const circleId = newNodeId = uuid.v4() parentNode.data('circleId', circleId) @@ -2282,7 +2410,8 @@ const AngularWorkflow = (props) => { }, position: { x: px, y: py }, locked: true - }).unselectify() + }) + //.unselectify() } const addDeleteButton = (event) => { @@ -2290,9 +2419,9 @@ const AngularWorkflow = (props) => { if (parentNode.data('isButton') || parentNode.data('buttonId')) return - parentNode.lock() + //parentNode.lock() const px = parentNode.position('x') - 65 - const py = parentNode.position('y') - 25 + const py = parentNode.position('y') + 25 const circleId = newNodeId = uuid.v4() parentNode.data('circleId', circleId) @@ -2320,7 +2449,8 @@ const AngularWorkflow = (props) => { }, position: { x: px, y: py }, locked: true - }).unselectify() + }) + //.unselectify() } const onNodeHover = (event) => { @@ -2346,10 +2476,10 @@ const AngularWorkflow = (props) => { } } - //if (!found) { - // addDeleteButton(event) - // addCopyButton(event) - //} + if (!found) { + addDeleteButton(event) + addCopyButton(event) + } } @@ -3197,7 +3327,7 @@ const AngularWorkflow = (props) => { if (data.name !== "User Input" && data.name !== "Shuffle Workflow") { //workflow.branches.push(newbranch) - cy.add(edgeToBeAdded) + //cy.add(edgeToBeAdded) } setWorkflow(workflow) @@ -3481,7 +3611,10 @@ const AngularWorkflow = (props) => { const runSearch = (value) => { if (value.length > 0) { - const newApps = allApps.filter(app => app.name.toLowerCase().includes(value.trim().toLowerCase() || app.description.toLowerCase().includes(value.trim().toLowerCase()))) + const newApps = allApps.filter(app => (app.name.toLowerCase().includes(value.trim().toLowerCase() || app.description.toLowerCase().includes(value.trim().toLowerCase()))) && !(!app.activated && app.generated)) + + //setFilteredApps(responseJson.filter(app => !internalIds.includes(app.name) && !(!app.activated && app.generated))) + //console.log("FOUND: ", newApps) setVisibleApps(newApps) } else { setVisibleApps(prioritizedApps.concat(filteredApps.filter(innerapp => !internalIds.includes(innerapp.id)))) @@ -6118,8 +6251,6 @@ const AngularWorkflow = (props) => { } const TopCytoscapeBar = (props) => { - useTraceUpdate(props) - return (
@@ -6280,8 +6411,6 @@ const AngularWorkflow = (props) => { const BottomCytoscapeBar = () => { const [anchorEl, setAnchorEl] = React.useState(null) - useTraceUpdate(props) - const boxSize = 100 const executionButton = executionRunning ? @@ -6391,8 +6520,6 @@ const AngularWorkflow = (props) => { const RightSideBar = (props) => { const {workflow, setWorkflow, setAction, setSelectedAction, setUpdate, appActionArguments, selectedApp, workflowExecutions, setSelectedResult, selectedAction, setSelectedApp, setSelectedTrigger, setSelectedEdge, setCurrentView, cy, setAuthenticationModalOpen,setVariablesModalOpen, setCodeModalOpen, selectedNameChange, rightsidebarStyle, showEnvironment, selectedActionEnvironment, environments, setNewSelectedAction, appApiViewStyle, globalUrl, setSelectedActionEnvironment, requiresAuthentication, hideExtraTypes, scrollConfig, setScrollConfig } = props - useTraceUpdate(props) - if (!rightSideBarOpen) { return null } diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 8bfc9537..e7ca5bd3 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -2516,12 +2516,17 @@ const AppCreator = (props) => { } if (e.target.value.length > 29) { - alert.error("Choose a shorter name.") + alert.error("Choose a shorter name (max 29).") return } + //e.target.value.trim() + setName(e.target.value) }} + onBlur={e => { + setName(e.target.value.trim()) + }} color="primary" InputProps={{ style:{ From 26b76ad3a100c5ca9dc15ceec9f0e7926d102072 Mon Sep 17 00:00:00 2001 From: frikky Date: Wed, 12 May 2021 12:26:30 +0200 Subject: [PATCH 35/96] Added BASIC undo system --- frontend/src/views/AngularWorkflow.jsx | 104 +++++++++++++++++++++---- 1 file changed, 89 insertions(+), 15 deletions(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index a755606a..b10875d0 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -11,7 +11,7 @@ import NestedMenuItem from "material-ui-nested-menu-item"; import {TextField, Drawer, Button, Paper, Grid, Tabs, InputAdornment, Tab, ButtonBase, Tooltip, Select, MenuItem, Divider, Dialog, Modal, DialogActions, DialogTitle, InputLabel, DialogContent, FormControl, IconButton, Menu, Input, FormGroup, FormControlLabel, Typography, Checkbox, Breadcrumbs, CircularProgress, Switch, Fade} from '@material-ui/core'; -import {FileCopy as FileCopyIcon, GetApp as GetAppIcon, Search as SearchIcon, ArrowUpward as ArrowUpwardIcon, Visibility as VisibilityIcon, Done as DoneIcon, Close as CloseIcon, Error as ErrorIcon, FindReplace as FindreplaceIcon, ArrowLeft as ArrowLeftIcon, Cached as CachedIcon, DirectionsRun as DirectionsRunIcon, Add as AddIcon, Polymer as PolymerIcon, FormatListNumbered as FormatListNumberedIcon, Create as CreateIcon, PlayArrow as PlayArrowIcon, AspectRatio as AspectRatioIcon, MoreVert as MoreVertIcon, Apps as AppsIcon, Schedule as ScheduleIcon, FavoriteBorder as FavoriteBorderIcon, Pause as PauseIcon, Delete as DeleteIcon, AddCircleOutline as AddCircleOutlineIcon, Save as SaveIcon, KeyboardArrowLeft as KeyboardArrowLeftIcon, KeyboardArrowRight as KeyboardArrowRightIcon, ArrowBack as ArrowBackIcon, Settings as SettingsIcon, LockOpen as LockOpenIcon, ExpandMore as ExpandMoreIcon, VpnKey as VpnKeyIcon} from '@material-ui/icons'; +import {Undo as UndoIcon, FileCopy as FileCopyIcon, GetApp as GetAppIcon, Search as SearchIcon, ArrowUpward as ArrowUpwardIcon, Visibility as VisibilityIcon, Done as DoneIcon, Close as CloseIcon, Error as ErrorIcon, FindReplace as FindreplaceIcon, ArrowLeft as ArrowLeftIcon, Cached as CachedIcon, DirectionsRun as DirectionsRunIcon, Add as AddIcon, Polymer as PolymerIcon, FormatListNumbered as FormatListNumberedIcon, Create as CreateIcon, PlayArrow as PlayArrowIcon, AspectRatio as AspectRatioIcon, MoreVert as MoreVertIcon, Apps as AppsIcon, Schedule as ScheduleIcon, FavoriteBorder as FavoriteBorderIcon, Pause as PauseIcon, Delete as DeleteIcon, AddCircleOutline as AddCircleOutlineIcon, Save as SaveIcon, KeyboardArrowLeft as KeyboardArrowLeftIcon, KeyboardArrowRight as KeyboardArrowRightIcon, ArrowBack as ArrowBackIcon, Settings as SettingsIcon, LockOpen as LockOpenIcon, ExpandMore as ExpandMoreIcon, VpnKey as VpnKeyIcon} from '@material-ui/icons'; import * as cytoscape from 'cytoscape'; import * as edgehandles from 'cytoscape-edgehandles'; @@ -156,7 +156,9 @@ const AngularWorkflow = (props) => { left: 0, selected: "", }) + const [history, setHistory] = React.useState([]) + const [historyIndex, setHistoryIndex] = React.useState(history.length) const [appAuthentication, setAppAuthentication] = React.useState([]); const [variablesModalOpen, setVariablesModalOpen] = React.useState(false); @@ -1937,6 +1939,8 @@ const AngularWorkflow = (props) => { "action": "added", "data": edge, }) + setHistory(history) + setHistoryIndex(history.length) } const onNodeAdded = (event) => { @@ -1965,6 +1969,16 @@ const AngularWorkflow = (props) => { setWorkflow(workflow) } + if (nodedata.app_name !== undefined) { + history.push({ + "type": "node", + "action": "added", + "data": nodedata, + }) + setHistory(history) + setHistoryIndex(history.length) + } + //if (nodedata.app_name !== undefined && (( // nodedata.app_name !== "Shuffle Tools" && // nodedata.app_name !== "Testing" && @@ -2018,11 +2032,6 @@ const AngularWorkflow = (props) => { // //console.log("Shouldnt re-add info? ") //} - history.push({ - "type": "node", - "action": "added", - "data": nodedata, - }) } const onEdgeRemoved = (event) => { @@ -2043,6 +2052,16 @@ const AngularWorkflow = (props) => { // data: newcybranch, //} } + + if (edge.data().source !== undefined) { + history.push({ + "type": "edge", + "action": "removed", + "data": edge.data().source, + }) + setHistory(history) + setHistoryIndex(history.length) + } } const onNodeRemoved = (event) => { @@ -2092,6 +2111,14 @@ const AngularWorkflow = (props) => { cy.getElementById(currentNode.data.id).remove() } } + + history.push({ + "type": "node", + "action": "removed", + "data": data, + }) + setHistory(history) + setHistoryIndex(history.length) } @@ -2116,42 +2143,44 @@ const AngularWorkflow = (props) => { break; case 38: console.log("UP") - break; + break; case 37: console.log("LEFT") - break; + break; case 40: console.log("DOWN") - break; + break; case 39: console.log("RIGHT") - break; + break; case 90: if (previouskey === 17) { console.log("CTRL+Z") + handleHistoryUndo() } - break; + + break; case 67: if (previouskey === 17) { console.log("CTRL+C") } - break; + break; case 86: if (previouskey === 17) { console.log("CTRL+V") } - break; + break; case 88: if (previouskey === 17) { console.log("CTRL+V") } - break; + break; case 83: if (previouskey === 17) { event.preventDefault() saveWorkflow() } - break; + break; case 70: //if (previouskey === 17) { // event.preventDefault() @@ -6408,6 +6437,40 @@ const AngularWorkflow = (props) => { ) } + const handleHistoryUndo = () => { + console.log("history: ", history, "index: ", historyIndex) + var item = history[historyIndex-1] + if (historyIndex === 0) { + item = history[historyIndex] + } + + if (item === undefined) { + console.log("Couldn't find the action you're looking for") + return + } + + console.log("HANDLE: ", item) + if (item.type === "node" && item.action === "removed") { + // Re-add the node + + cy.add({ + group: 'nodes', + data: item.data, + position: item.data.position, + }) + } else if (item.action === "added") { + console.log("Should remove item!") + const currentitem = cy.getElementById(item.data.id) + if (currentitem !== undefined && currentitem !== null) { + currentitem.remove() + } + } + + if (historyIndex > 0) { + setHistoryIndex(historyIndex-1) + } + } + const BottomCytoscapeBar = () => { const [anchorEl, setAnchorEl] = React.useState(null) @@ -6512,6 +6575,17 @@ const AngularWorkflow = (props) => { {/* */} {workflow.configuration !== null && workflow.configuration !== undefined && workflow.configuration.exit_on_error !== undefined ? : null} + {history.length > 0 ? + + + + + + : null}
) From 5dca18edf9affa3b9d67f004acf6c91cb563f995 Mon Sep 17 00:00:00 2001 From: frikky Date: Wed, 12 May 2021 12:40:45 +0200 Subject: [PATCH 36/96] Added extra authentication checker to workflow configuration popup --- frontend/src/components/ConfigureWorkflow.jsx | 21 ++++++++++++++++--- frontend/src/views/AngularWorkflow.jsx | 20 ++++++++---------- frontend/src/views/Apps.jsx | 7 +++++++ 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/frontend/src/components/ConfigureWorkflow.jsx b/frontend/src/components/ConfigureWorkflow.jsx index 215617ab..337c8649 100644 --- a/frontend/src/components/ConfigureWorkflow.jsx +++ b/frontend/src/components/ConfigureWorkflow.jsx @@ -2,6 +2,7 @@ import React, {useState} from 'react'; import { InputAdornment, Tooltip, TextField, CircularProgress, ButtonGroup, Button, Avatar, ListItemAvatar, Typography, List, ListItem, ListItemText} from '@material-ui/core'; import {FavoriteBorder as FavoriteBorderIcon} from '@material-ui/icons'; +import { FixName } from "../views/Apps.jsx"; // Handles workflow updates on first open to highlight the issues of the workflow // Variables @@ -88,8 +89,22 @@ const Workflow = (props) => { newaction.must_activate = true } else { if (action.authentication_id === "" && app.authentication.required === true) { - newaction.must_authenticate = true - newaction.action_ids.push(action.id) + // Check if configuration is filled or not + var filled = true + for (var key in action.parameters) { + if (action.parameters[key].configuration) { + console.log("Found config: ", action.parameters[key]) + if (action.parameters[key].value === null || action.parameters[key].value.length === 0) { + filled = false + break + } + } + } + + if (!filled) { + newaction.must_authenticate = true + newaction.action_ids.push(action.id) + } } newaction.app = app @@ -339,7 +354,7 @@ const Workflow = (props) => { diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index b10875d0..f8fdf9f5 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -6554,6 +6554,15 @@ const AngularWorkflow = (props) => { + + + + + - - - : null}
) diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index dd86648e..a2255cc6 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -20,6 +20,13 @@ const chipStyle = { backgroundColor: "#3d3f43", height: 30, marginRight: 5, paddingLeft: 5, paddingRight: 5, height: 28, cursor: "pointer", borderColor: "#3d3f43", color: "white", } +// Fixes names by making them uppercase and such +// Used for labels. A lot of places don't use this yet +export const FixName = (name) => { + const newAppname = (name.charAt(0).toUpperCase()+name.substring(1)).replaceAll("_", " ") + return newAppname +} + // Parses JSON data into keys that can be used everywhere :) export const GetParsedPaths = (inputdata, basekey) => { const splitkey = " > " From 8a750d0121a60d0e990fe5378255d8d78eb0cd77 Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 15 May 2021 18:10:24 +0200 Subject: [PATCH 37/96] Added API for Worker to download docker images --- backend/go-app/docker.go | 276 +++++++----------------------- backend/go-app/go.mod | 6 +- backend/go-app/go.sum | 88 ++++++++++ backend/go-app/main.go | 2 +- backend/tests/dockerpull.sh | 2 +- functions/onprem/worker/go.mod | 2 + functions/onprem/worker/worker.go | 94 ++++++++++ 7 files changed, 250 insertions(+), 220 deletions(-) diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index ac25f9e6..43168202 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -5,20 +5,25 @@ import ( "github.com/frikky/shuffle-shared" "archive/tar" + "bufio" "path/filepath" + "strconv" "bytes" "context" "encoding/json" "errors" "fmt" + //"github.com/docker/docker" "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" + //"github.com/docker/docker/api/types/container" "github.com/docker/docker/client" + newdockerclient "github.com/fsouza/go-dockerclient" "github.com/go-git/go-billy/v5" + "github.com/go-git/go-billy/v5/memfs" - network "github.com/docker/docker/api/types/network" - natting "github.com/docker/go-connections/nat" + //network "github.com/docker/docker/api/types/network" + //natting "github.com/docker/go-connections/nat" "io" "io/ioutil" @@ -409,89 +414,6 @@ func stopWebhook(image string, identifier string) error { return nil } -// FIXME - remember to set DOCKER_API_VERSION -// FIXME - remove github.com/docker/docker/vendor -// FIXME - Library dependencies for NAT is fucked.. -// https://docs.docker.com/develop/sdk/examples/ -func deployWebhook(image string, identifier string, path string, port string, callbackurl string, apikey string) error { - cli, err := client.NewEnvClient() - if err != nil { - fmt.Println("Unable to create docker client") - return err - } - - newport, err := natting.NewPort("tcp", port) - if err != nil { - fmt.Println("Unable to create docker port") - return err - } - - // FIXME - logging? - - hostConfig := &container.HostConfig{ - PortBindings: natting.PortMap{ - newport: []natting.PortBinding{ - { - HostIP: "0.0.0.0", - HostPort: port, - }, - }, - }, - RestartPolicy: container.RestartPolicy{ - Name: "always", - }, - LogConfig: container.LogConfig{ - Type: "json-file", - Config: map[string]string{}, - }, - } - - //networkConfig := &network.NetworkSettings{} - networkConfig := &network.NetworkingConfig{ - EndpointsConfig: map[string]*network.EndpointSettings{}, - } - - test := &network.EndpointSettings{ - Gateway: "helo", - } - - networkConfig.EndpointsConfig["bridge"] = test - - exposedPorts := map[natting.Port]struct{}{ - newport: struct{}{}, - } - - config := &container.Config{ - Image: image, - Env: []string{ - fmt.Sprintf("URIPATH=%s", path), - fmt.Sprintf("HOOKPORT=%s", port), - fmt.Sprintf("CALLBACKURL=%s", callbackurl), - fmt.Sprintf("APIKEY=%s", apikey), - fmt.Sprintf("HOOKID=%s", identifier), - }, - ExposedPorts: exposedPorts, - Hostname: fmt.Sprintf("%s-%s", image, identifier), - } - - cont, err := cli.ContainerCreate( - context.Background(), - config, - hostConfig, - networkConfig, - fmt.Sprintf("%s-%s", image, identifier), - ) - - if err != nil { - log.Println(err) - return err - } - - cli.ContainerStart(context.Background(), cont.ID, types.ContainerStartOptions{}) - log.Printf("Container %s is created", cont.ID) - return nil -} - // Starts a new webhook func handleStopHookDocker(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) @@ -624,121 +546,6 @@ func handleDeleteHookDocker(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(`{"success": true, "message": "Deleted webhook"}`)) } -// Starts a new webhook -func handleStartHookDocker(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - location := strings.Split(request.URL.String(), "/") - - var fileId string - if location[1] == "api" { - if len(location) <= 4 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - if len(fileId) != 32 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "message": "ID not valid"}`)) - return - } - - ctx := context.Background() - hook, err := getHook(ctx, fileId) - if err != nil { - log.Printf("Failed getting hook %s (start docker): %s", fileId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if len(hook.Info.Url) == 0 { - log.Printf("Hook url can't be empty.") - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - log.Printf("Status: %s", hook.Status) - log.Printf("Running: %t", hook.Running) - if hook.Running || hook.Status == "Running" { - message := fmt.Sprintf("Error: %s is already running", hook.Id) - log.Println(message) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "%s"}`, message))) - return - } - - // FIXME - verify? - // FIXME - static port? Generate from available range. - image := "webhook" - filepath := "/webhook" - baseUrl := "http://localhost" - callbackUrl := "http://localhost:8001" - - // This is here to force stop and remove the old webhook - err = stopWebhook(image, fileId) - if err != nil { - log.Printf("Container stop issue for %s-%s: %s", image, fileId, err) - } - - // Dynamic ish ports - var startPort int64 = 5001 - var endPort int64 = 5010 - port := findAvailablePorts(startPort, endPort) - if len(port) == 0 { - message := fmt.Sprintf("Not ports available in the range %d-%d", startPort, endPort) - log.Println(message) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "%s"}`, message))) - return - - } - - hook.Status = "running" - hook.Running = true - - // Set this for more than just hooks? - if hook.Type == "webhook" { - hook.Info.Url = fmt.Sprintf("%s:%s%s", baseUrl, port, filepath) - } - err = setHook(ctx, *hook) - if err != nil { - log.Printf("Failed setting hook: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - // Cloud run? Let's make a generic webhook that can be deployed easily - log.Printf("Should run a webhook with the following: \nUrl: %s\nId: %s\n", hook.Info.Url, hook.Id) - - // FIXME - set port based on what the user specified / what was generated - // FIXME - add nonstatic APIKEY - apiKey := "ASD" - - err = deployWebhook(image, fileId, filepath, port, callbackUrl, apiKey) - if err != nil { - log.Printf("Failed starting container %s-%s: %s", image, fileId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - // FIXME - get some real data? - log.Printf("[INFO] Successfully started %s-%s on port %s with filepath %s", image, fileId, port, filepath) - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true, "message": "Started webhook"}`)) - return -} - // Checks if an image exists func imageCheckBuilder(images []string) error { //log.Printf("[FIXME] ImageNames to check: %#v", images) @@ -819,7 +626,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { // Just here to verify that the user is logged in _, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { - log.Printf("Api authentication failed in validate swagger: %s", err) + log.Printf("[WARNING] Api authentication failed in DOWNLOAD IMAGE: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -857,7 +664,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("Image to load: %s", version.Name) + log.Printf("[DEBUG] Image to load: %s", version.Name) //cli, err := client.NewEnvClient() //if err != nil { // log.Println("Unable to create docker client") @@ -896,20 +703,59 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "Couldn't find image %s"}`, version.Name))) return } + _ = tagFound + log.Printf("Img (%s) found: %#v", tagFound, img) - /* - log.Printf("IMg: %#v", img) - pullOptions := types.ImagePullOptions{} - log.Printf("[INFO] Pulling image %s", image) - reader, err := dockercli.ImagePull(ctx, tag, pullOptions) - if err != nil { - log.Printf("[ERROR] Failed getting image %s: %s", image, err) - } + basepath := "base" + location := fmt.Sprintf("%s.tar.gz", tagFound) + fs := memfs.New() - io.Copy(os.Stdout, r) - */ + //Close after function return + f, err := fs.Create(fmt.Sprintf("%s/%s", basepath, location)) + if err != nil { + log.Printf("[WARNING] Failed making file: %s", err) + return + } - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "message": "Downloading image %s"}`, version.Name))) + newClient, err := newdockerclient.NewClientFromEnv() + if err != nil { + log.Printf("[WARNING] Failed setting up docker env: %s", newClient) + return + } + + //https://github.com/fsouza/go-dockerclient/issues/600 + defer f.Close() + w := bufio.NewWriter(f) + opts := newdockerclient.ExportImageOptions{ + Name: tagFound, + OutputStream: w, + } + + if err := newClient.ExportImage(opts); err != nil { + log.Printf("[WARNING] FAILED to save image to file: %s", err) + return + } + + w.Flush() + FileHeader := make([]byte, 512) + f.Read(FileHeader) + FileContentType := http.DetectContentType(FileHeader) + + //Get the file size + //FileStat, _ := f.Stat() //Get info from file + //FileSize := strconv.FormatInt(f.Size(), 10) //Get file size as a string + + //Send the headers + resp.Header().Set("Content-Disposition", "attachment; filename="+location) + resp.Header().Set("Content-Type", FileContentType) + resp.Header().Set("Content-Length", strconv.FormatInt(img.Size, 10)) + + //Send the file + //We read 512 bytes from the file already, so we reset the offset back to 0 + f.Seek(0, 0) + io.Copy(resp, f) //'Copy' the file to the client + + //resp.WriteHeader(200) + //resp.Write([]byte(fmt.Sprintf(`{"success": true, "message": "Downloading image %s"}`, version.Name))) } diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index ca52b271..05925447 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -3,6 +3,7 @@ module shuffle go 1.13 replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared + //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi require ( @@ -11,17 +12,17 @@ require ( cloud.google.com/go/pubsub v1.3.1 cloud.google.com/go/storage v1.12.0 github.com/Masterminds/semver v1.5.0 // indirect - github.com/Microsoft/go-winio v0.4.14 // indirect github.com/algolia/algoliasearch-client-go/v3 v3.18.1 // indirect github.com/basgys/goxml2json v1.1.0 github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 // indirect github.com/carlescere/scheduler v0.0.0-20170109141437-ee74d2f83d82 github.com/docker/distribution v2.7.1+incompatible // indirect - github.com/docker/docker v1.13.1 + github.com/docker/docker v20.10.3-0.20210216175712-646072ed6524+incompatible github.com/docker/go-connections v0.4.0 github.com/docker/go-units v0.4.0 // indirect github.com/frikky/kin-openapi v0.39.0 github.com/frikky/shuffle-shared v0.0.40 + github.com/fsouza/go-dockerclient v1.7.2 // indirect github.com/ghodss/yaml v1.0.0 github.com/go-git/go-billy/v5 v5.0.0 github.com/go-git/go-git/v5 v5.0.0 @@ -29,7 +30,6 @@ require ( github.com/gorilla/handlers v1.4.2 // indirect github.com/gorilla/mux v1.8.0 github.com/h2non/filetype v1.0.12 - github.com/opencontainers/go-digest v1.0.0-rc1 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible github.com/satori/go.uuid v1.2.0 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 11673d62..1cb3f682 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -1,3 +1,4 @@ +bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -45,6 +46,7 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 cloud.google.com/go/storage v1.12.0 h1:4y3gHptW1EHVtcPAVE0eBBlFuGqEejTTG3KdIE0lUX4= cloud.google.com/go/storage v1.12.0/go.mod h1:fFLk2dp2oAhDz8QFKwqrjdJvxSp/W2g7nillojlL5Ho= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -52,6 +54,9 @@ github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3Q github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= +github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= +github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= +github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= github.com/algolia/algoliasearch-client-go v2.25.0+incompatible h1:FGQr9l++u4uQPDXrW8jM5kNJm3Iw5SxEJJtYXSFmPRY= github.com/algolia/algoliasearch-client-go/v3 v3.18.1 h1:FP2Xtqqs/sefR5Qluygp+jVV+juXzEdJaPrZTCDLhDQ= @@ -68,21 +73,42 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= +github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= +github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.3 h1:ijQT13JedHSHrQGWFcGEwzcNKrAGIiZ+jSD5QQG07SY= +github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e h1:6JKvHHt396/qabvMhnhUZvWaHZzfVfldxE60TK8YLhg= +github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= +github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= +github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= +github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= +github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= +github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo= github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v20.10.3-0.20210216175712-646072ed6524+incompatible h1:Yu2uGErhwEoOT/OxAFe+/SiJCqRLs+pgcS5XKrDXnG4= +github.com/docker/docker v20.10.3-0.20210216175712-646072ed6524+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v20.10.6+incompatible h1:oXI3Vas8TI8Eu/EjH4srKHJBVqraSzJybhxY7Om9faQ= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -119,6 +145,8 @@ github.com/frikky/shuffle-shared v0.0.38 h1:OZSwU1HDOaPzdlG1s77svgXJKzlNewM6GjeH github.com/frikky/shuffle-shared v0.0.38/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= github.com/frikky/shuffle-shared v0.0.40 h1:H0au2np5xSy9mZEUWN+a29IORk+YP95FipENBD7iJRw= github.com/frikky/shuffle-shared v0.0.40/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= +github.com/fsouza/go-dockerclient v1.7.2 h1:bBEAcqLTkpq205jooP5RVroUKiVEWgGecHyeZc4OFjo= +github.com/fsouza/go-dockerclient v1.7.2/go.mod h1:+ugtMCVRwnPfY7d8/baCzZ3uwB0BrG5DB8OzbtxaRz8= github.com/getkin/kin-openapi v0.8.0 h1:a6TQjTqwkyscC4/hShJX7WhCVE+4bi9lzw61XHQW5hE= github.com/getkin/kin-openapi v0.8.0/go.mod h1:zZQMFkVgRHCdhgb6ihCTIo9dyDZFvX0k/xAKqw1FhPw= github.com/getkin/kin-openapi v0.52.0 h1:6WqsF5d6PfJ8AscdD+9Rtb2RP2iBWyC7V6GcjssWg7M= @@ -142,6 +170,10 @@ github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUe github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -183,6 +215,8 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= github.com/google/go-github/v28 v28.1.1 h1:kORf5ekX5qwXO2mGzXXOjMe/g6ap8ahVe0sBEulhSxo= github.com/google/go-github/v28 v28.1.1/go.mod h1:bsqJWQX05omyWVmc00nEUql9mhQyv38lDZ8kPZcQVoM= @@ -218,6 +252,7 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -226,8 +261,11 @@ github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfE github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= @@ -238,23 +276,51 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/moby/sys/mount v0.2.0 h1:WhCW5B355jtxndN5ovugJlMFJawbUODuW8fSnEH6SSM= +github.com/moby/sys/mount v0.2.0/go.mod h1:aAivFE2LB3W4bACsUXChRHQ0qKWsetY4Y9V7sxOougM= +github.com/moby/sys/mountinfo v0.4.0 h1:1KInV3Huv18akCu58V7lzNlt+jFmqlu1EaErnEHE/VM= +github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk= +github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= +github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJGY8Y= +github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -265,6 +331,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -327,6 +394,7 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -337,6 +405,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -385,15 +454,20 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -409,11 +483,17 @@ golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3 h1:kzM6+9dur93BcC2kVlYl34cHU+TYZLanmpSJHVMmL64= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210216224549-f992740a1bac h1:9glrpwtNjBYgRpb67AZJKHfzj1stG/8BL5H7In2oTC4= +golang.org/x/sys v0.0.0-20210216224549-f992740a1bac/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201113234701-d7a72108b828/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -427,6 +507,7 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -437,6 +518,7 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -466,6 +548,7 @@ golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= @@ -475,6 +558,7 @@ golang.org/x/tools v0.0.0-20200915173823-2db8f0ff891c/go.mod h1:z6u4i615ZeAfBE4X golang.org/x/tools v0.0.0-20200918232735-d647fc253266/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -560,6 +644,7 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -612,6 +697,9 @@ gopkg.in/yaml.v3 v3.0.0-20200506231410-2ff61e1afc86 h1:OfFoIUYv/me30yv7XlMy4F9RJ gopkg.in/yaml.v3 v3.0.0-20200506231410-2ff61e1afc86/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 39492255..20ec1d4d 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -6058,7 +6058,7 @@ func initHandlers() { //r.HandleFunc("/api/v1/orgs/{orgId}", handleEditOrg).Methods("POST", "OPTIONS") // Docker orborus specific - //r.HandleFunc("/api/v1/get_docker_image", getDockerImage).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/get_docker_image", getDockerImage).Methods("POST", "OPTIONS") // Important for email, IDS etc. Create this by: // PS: For cloud, this has to use cloud storage. diff --git a/backend/tests/dockerpull.sh b/backend/tests/dockerpull.sh index 3df0d1e5..7a256689 100644 --- a/backend/tests/dockerpull.sh +++ b/backend/tests/dockerpull.sh @@ -1,2 +1,2 @@ #!/bin/sh -curl -XPOST http://localhost:5001/api/v1/get_docker_image -d '{"name":"frikky/shuffle:Testing_1.0.0"}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" +curl -XPOST http://localhost:5001/api/v1/get_docker_image -d '{"name":"frikky/shuffle:Testing_1.0.0"}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4" --output tarball.tgz diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 9d718750..c18136a3 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -13,6 +13,8 @@ require ( github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/frikky/shuffle-shared v0.0.40 + github.com/fsouza/go-dockerclient v1.7.2 // indirect + github.com/go-git/go-billy/v5 v5.3.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/gorilla/mux v1.8.0 github.com/opencontainers/go-digest v1.0.0 // indirect diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index af893ab9..f48a3e5c 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -3,6 +3,7 @@ package main import ( "github.com/frikky/shuffle-shared" + //"bufio" "bytes" "context" "encoding/json" @@ -24,6 +25,9 @@ import ( //"github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/mount" dockerclient "github.com/docker/docker/client" + //"github.com/go-git/go-billy/v5/memfs" + + newdockerclient "github.com/fsouza/go-dockerclient" //"github.com/satori/go.uuid" "github.com/gorilla/mux" @@ -1780,8 +1784,94 @@ func runWebserver(listener net.Listener) { log.Fatal(http.Serve(listener, nil)) } +func downloadDockerImage(client *http.Client, imageName string) { + data := fmt.Sprintf(`{"name": "%s"}`, imageName) + fullUrl := fmt.Sprintf("%s/api/v1/get_docker_image", baseUrl) + + req, err := http.NewRequest( + "POST", + fullUrl, + bytes.NewBuffer([]byte(data)), + ) + + authorization := os.Getenv("AUTHORIZATION") + if len(authorization) > 0 { + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", authorization)) + } else { + log.Printf("[WARNING] No auth found.") + req.Header.Add("Authorization", fmt.Sprintf("Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4")) + //return + } + + newresp, err := client.Do(req) + if err != nil { + log.Printf("[ERROR] Failed request: %s", err) + return + } + + if newresp.StatusCode != 200 { + log.Printf("[ERROR] DOWNLOAD StatusCode (1): %d", newresp.StatusCode) + return + } + + // Write the body to file + + newClient, err := newdockerclient.NewClientFromEnv() + if err != nil { + log.Printf("[WARNING] Failed setting up docker env in download: %s", newClient) + return + } + + newImageName := strings.Replace(imageName, "/", "_", -1) + newFileName := newImageName + ".tar.gz" + //os.Create(newFileName) + + tar, err := os.Create(newFileName) + if err != nil { + log.Printf("[WARNING] Failed creating file: %s", err) + return + } + + //fs := memfs.New() + //if err != nil { + // log.Printf("[WARNING] Failed making memory file: %s", err) + // return + //} + + //imageName = strings.Replace(imageName, "/", "_", -1) + //tar, err := fs.Create(imageName + ".tar.gz") + //if err != nil { + // log.Printf("[WARNING] Failed making file: %s", err) + // return + //} + defer tar.Close() + _, err = io.Copy(tar, newresp.Body) + + //OutputStream: outFile, + //Context: context.Background(), + imageOptions := newdockerclient.LoadImageOptions{ + InputStream: tar, + } + + //log.Printf("BUF: %s", buf.String()) + err = newClient.LoadImage(imageOptions) + if err != nil { + log.Printf("[WARNING] Failed loading image %s: %s", imageName, err) + return + } + + log.Printf("[INFO] Successfully loaded image %s", imageName) + //err = os.Remove(newImageName) + //if err != nil { + // log.Printf("[WARNING] Failed removing file: %s", err) + //} + + return +} + // Initial loop etc func main() { + log.Printf("[INFO] Setting up worker environment") sleepTime := 5 @@ -1804,6 +1894,10 @@ func main() { } } + imageName := "frikky/shuffle:Testing_1.0.0" + downloadDockerImage(client, imageName) + os.Exit(3) + // WORKER_TESTING_WORKFLOW should be a workflow ID authorization := "" executionId := "" From 22098f72e0c26d1409c4063c8864827c3fd1f510 Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 15 May 2021 18:11:25 +0200 Subject: [PATCH 38/96] Removed worker download test code --- functions/onprem/worker/worker.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index f48a3e5c..cddf8c0a 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -1894,9 +1894,9 @@ func main() { } } - imageName := "frikky/shuffle:Testing_1.0.0" - downloadDockerImage(client, imageName) - os.Exit(3) + //imageName := "frikky/shuffle:Testing_1.0.0" + //downloadDockerImage(client, imageName) + //os.Exit(3) // WORKER_TESTING_WORKFLOW should be a workflow ID authorization := "" From ef0406456715aaad2b1069a95a90a14cfb5fb029 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 18 May 2021 05:47:11 +0200 Subject: [PATCH 39/96] Minor changes to action view --- backend/go-app/docker.go | 5 ++--- frontend/src/components/ParsedAction.jsx | 8 ++++++-- functions/extensions/wazuh/ossec.conf | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index 43168202..7cd34b7d 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -688,7 +688,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { tagFound := "" for _, image := range images { for _, tag := range image.RepoTags { - log.Printf("[INFO] Docker Image: %s", tag) + //log.Printf("[INFO] Docker Image: %s", tag) if strings.ToLower(tag) == strings.ToLower(version.Name) { img = image @@ -704,8 +704,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { return } - _ = tagFound - log.Printf("Img (%s) found: %#v", tagFound, img) + log.Printf("[INFO] Img found (%s): %#v", tagFound, img) basepath := "base" location := fmt.Sprintf("%s.tar.gz", tagFound) diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index d9052c6d..9302df5b 100644 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -53,11 +53,15 @@ const ParsedAction = (props) => { while(true) { for (var key in allkeys) { var currentnode = cy.getElementById(allkeys[key]) - if (handled.includes(currentnode.data().id)) { + if (currentnode === undefined) { + continue + } + + if (handled.includes(currentnode.data("id"))) { continue } else { // Get the name / label here too? - handled.push(currentnode.data().id) + handled.push(currentnode.data("id")) results.push(currentnode.data()) } diff --git a/functions/extensions/wazuh/ossec.conf b/functions/extensions/wazuh/ossec.conf index 5b55f5d3..6c281103 100644 --- a/functions/extensions/wazuh/ossec.conf +++ b/functions/extensions/wazuh/ossec.conf @@ -1,5 +1,5 @@ custom-shuffle - http://:3001/api/v1/hooks/webhook_ + http://:/api/v1/hooks/webhook_ json From a807f4c2e8bc332b40062577d6e1eb1241a683a6 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 18 May 2021 10:51:12 +0200 Subject: [PATCH 40/96] Fixed dragging trigger bug --- frontend/src/views/AngularWorkflow.jsx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index f8fdf9f5..ee44b45b 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1475,7 +1475,7 @@ const AngularWorkflow = (props) => { // 3. If it is, then hide text } - if (nodedata.app_id !== undefined) { + if (nodedata.app_name !== undefined) { //console.log("Trying to remove friendly nodes") const allNodes = cy.nodes().jsons() for (var key in allNodes) { @@ -1484,6 +1484,8 @@ const AngularWorkflow = (props) => { cy.getElementById(currentNode.data.id).remove() } } + } else { + console.log("No appid? ", nodedata) } if (nodedata.id === selectedAction.id) { @@ -2101,7 +2103,7 @@ const AngularWorkflow = (props) => { //}) } - if (data.app_id !== undefined) { + if (data.app_name !== undefined) { //console.log("Trying to remove friendly nodes") const allNodes = cy.nodes().jsons() //console.log("NOT UNDEFINED IN HOVEROUT!", allNodes) From 0696febd524c3aa31ee79c744aaa15fc0f7aecb3 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 18 May 2021 10:51:28 +0200 Subject: [PATCH 41/96] Added another new workflow button to grid --- frontend/src/views/Workflows.jsx | 61 +++++++++++++++++++++++++++----- 1 file changed, 52 insertions(+), 9 deletions(-) diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 38943545..5064691e 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -3,7 +3,7 @@ import { useInterval } from 'react-powerhooks'; import { makeStyles } from '@material-ui/core/styles'; import {Avatar, Grid, Paper, Tooltip, Divider, Button, TextField, FormControl, IconButton, Menu, MenuItem, FormControlLabel, Chip, Switch, Typography, Zoom, CircularProgress, Dialog, DialogTitle, DialogActions, DialogContent} from '@material-ui/core'; -import {Compare as CompareIcon, Maximize as MaximizeIcon, Minimize as MinimizeIcon, Toc as TocIcon, Send as SendIcon, Search as SearchIcon, FileCopy as FileCopyIcon, Delete as DeleteIcon, BubbleChart as BubbleChartIcon, Restore as RestoreIcon, Cached as CachedIcon, GetApp as GetAppIcon, Apps as AppsIcon, Edit as EditIcon, MoreVert as MoreVertIcon, PlayArrow as PlayArrowIcon, Add as AddIcon, Publish as PublishIcon, CloudUpload as CloudUploadIcon, CloudDownload as CloudDownloadIcon} from '@material-ui/icons'; +import {Compare as CompareIcon, Maximize as MaximizeIcon, Minimize as MinimizeIcon, AddCircle as AddCircleIcon, Toc as TocIcon, Send as SendIcon, Search as SearchIcon, FileCopy as FileCopyIcon, Delete as DeleteIcon, BubbleChart as BubbleChartIcon, Restore as RestoreIcon, Cached as CachedIcon, GetApp as GetAppIcon, Apps as AppsIcon, Edit as EditIcon, MoreVert as MoreVertIcon, PlayArrow as PlayArrowIcon, Add as AddIcon, Publish as PublishIcon, CloudUpload as CloudUploadIcon, CloudDownload as CloudDownloadIcon} from '@material-ui/icons'; //import {Search as SearchIcon, ArrowUpward as ArrowUpwardIcon, Visibility as VisibilityIcon, Done as DoneIcon, Close as CloseIcon, Error as ErrorIcon, FindReplace as FindreplaceIcon, ArrowLeft as ArrowLeftIcon, Cached as CachedIcon, DirectionsRun as DirectionsRunIcon, Add as AddIcon, Polymer as PolymerIcon, FormatListNumbered as FormatListNumberedIcon, Create as CreateIcon, PlayArrow as PlayArrowIcon, AspectRatio as AspectRatioIcon, MoreVert as MoreVertIcon, Apps as AppsIcon, Schedule as ScheduleIcon, FavoriteBorder as FavoriteBorderIcon, Pause as PauseIcon, Delete as DeleteIcon, AddCircleOutline as AddCircleOutlineIcon, Save as SaveIcon, KeyboardArrowLeft as KeyboardArrowLeftIcon, KeyboardArrowRight as KeyboardArrowRightIcon, ArrowBack as ArrowBackIcon, Settings as SettingsIcon, LockOpen as LockOpenIcon, ExpandMore as ExpandMoreIcon, VpnKey as VpnKeyIcon} from '@material-ui/icons'; import {DataGrid, GridToolbarContainer, GridDensitySelector, GridToolbar} from '@material-ui/data-grid'; @@ -614,6 +614,7 @@ const Workflows = (props) => { position: "relative", } + const gridContainer = { height: "auto", color: "white", @@ -898,6 +899,39 @@ const Workflows = (props) => { addFilter(e.target.innerHTML) } + + const NewWorkflowPaper = (props) => { + const [hover, setHover] = React.useState(false) + + const innerColor = "rgba(255,255,255,0.3)" + const setupPaperStyle = { + minHeight: paperAppStyle.minHeight, + width: paperAppStyle.width, + color: innerColor, + padding: paperAppStyle.padding, + borderRadius: paperAppStyle.borderRadius, + display: "flex", + boxSizing: "border-box", + position: "relative", + border: `2px solid ${innerColor}`, + cursor: "pointer", + backgroundColor: hover ? "rgba(39,41,45,0.5)" : "rgba(39,41,45,1)", + } + + return( + + setModalOpen(true)} onMouseOver={() => {setHover(true)}} onMouseOut={() => {setHover(false)}}> + + + + + + + + ) + } + + const WorkflowPaper = (props) => { const { data } = props; const [open, setOpen] = React.useState(false); @@ -1469,7 +1503,8 @@ const Workflows = (props) => { const data = params.row.record; let [triggers, schedules, webhooks, subflows] = getWorkflowMeta(data); - return + return + { disableClickEventBubbling: true, renderCell: (params) => { const data = params.row.record; - return + return + {data.tags !== undefined ? data.tags.map((tag, index) => { if (index >= 3) { @@ -1525,12 +1561,18 @@ const Workflows = (props) => { ]; let rows = []; rows = workflows.map((data, index) => { - let obj = {"id":index+1, "title":data.name, "record":data,}; + let obj = { + "id":index+1, + "title":data.name, + "record":data, + } + return obj; }); - workflowData = + workflowData = + } return (
@@ -1820,6 +1862,7 @@ const Workflows = (props) => {
{view === "grid" && ( + {filteredWorkflows.map((data, index) => { return ( @@ -1828,9 +1871,9 @@ const Workflows = (props) => { )} - {view === "list" && ( + {/*view === "list" && ( - )} + )*/}
From 4c6c6ba9c5396454e0ff77e3a5912ec6fe737430 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 18 May 2021 10:52:10 +0200 Subject: [PATCH 42/96] #373: Added basic fix to the Worker --- functions/extensions/wazuh/ossec.conf | 1 + functions/onprem/worker/Dockerfile | 4 ++- functions/onprem/worker/go.mod | 3 +- functions/onprem/worker/worker.go | 43 +++++++++++++-------------- 4 files changed, 27 insertions(+), 24 deletions(-) diff --git a/functions/extensions/wazuh/ossec.conf b/functions/extensions/wazuh/ossec.conf index 6c281103..45674065 100644 --- a/functions/extensions/wazuh/ossec.conf +++ b/functions/extensions/wazuh/ossec.conf @@ -1,5 +1,6 @@ custom-shuffle + 9 http://:/api/v1/hooks/webhook_ json diff --git a/functions/onprem/worker/Dockerfile b/functions/onprem/worker/Dockerfile index 826ef4d3..6ef340fa 100644 --- a/functions/onprem/worker/Dockerfile +++ b/functions/onprem/worker/Dockerfile @@ -12,7 +12,9 @@ RUN go get github.com/docker/docker/api/types && \ go get github.com/gorilla/mux && \ go get github.com/patrickmn/go-cache && \ go get github.com/frikky/shuffle-shared && \ - go get github.com/satori/go.uuid + go get github.com/satori/go.uuid && \ + go get github.com/fsouza/go-dockerclient && \ + go get google.golang.org/grpc/balancer/grpclb@v1.37.1 RUN go build RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker . diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index c18136a3..651b1514 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -13,7 +13,7 @@ require ( github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/frikky/shuffle-shared v0.0.40 - github.com/fsouza/go-dockerclient v1.7.2 // indirect + github.com/fsouza/go-dockerclient v1.7.2 github.com/go-git/go-billy/v5 v5.3.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/gorilla/mux v1.8.0 @@ -22,4 +22,5 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 // indirect github.com/sirupsen/logrus v1.8.1 // indirect + google.golang.org/grpc v1.37.1 // indirect ) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index cddf8c0a..e981804e 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -853,7 +853,8 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { appname = strings.Replace(appname, ".", "-", -1) appversion = strings.Replace(appversion, ".", "-", -1) - image := fmt.Sprintf("%s:%s_%s", baseimagename, strings.ToLower(action.AppName), action.AppVersion) + parsedAppname := strings.Replace(strings.ToLower(action.AppName), " ", "-", -1) + image := fmt.Sprintf("%s:%s_%s", baseimagename, parsedAppname, action.AppVersion) if strings.Contains(image, " ") { image = strings.ReplaceAll(image, " ", "-") } @@ -957,14 +958,12 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { // Uses a few ways of getting / checking if an app is available // 1. Try original with lowercase - // 2. Go to original + // 2. Go to original (no spaces) // 3. Add remote repo location - // 4. Actually download last repo - images := []string{ image, - fmt.Sprintf("%s:%s_%s", baseimagename, action.AppName, action.AppVersion), - fmt.Sprintf("%s/%s:%s_%s", registryName, baseimagename, strings.ToLower(action.AppName), action.AppVersion), + fmt.Sprintf("%s:%s_%s", baseimagename, strings.Replace(action.AppName, " ", "-", -1), action.AppVersion), + fmt.Sprintf("%s/%s:%s_%s", registryName, baseimagename, parsedAppname, action.AppVersion), } // If cleanup is set, it should run for efficiency @@ -973,26 +972,26 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { err = deployApp(dockercli, images[0], identifier, env, workflowExecution) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { if strings.Contains(err.Error(), "exited prematurely") { - shutdown(workflowExecution, action.ID, err.Error(), true) + shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) } log.Printf("[WARNING] Failed CLEANUP execution. Downloading image remotely.") image = images[2] reader, err := dockercli.ImagePull(context.Background(), image, pullOptions) if err != nil { - log.Printf("[ERROR] Failed getting %s. The couldn't be find locally, AND is missing.", image) - shutdown(workflowExecution, action.ID, err.Error(), true) + log.Printf("[ERROR] Failed getting %s. Couldn't be find locally, AND is missing.", image) + shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) } buildBuf := new(strings.Builder) _, err = io.Copy(buildBuf, reader) - if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { + if err != nil && !strings.Contains(fmt.Sprintf("Docker error: %s", err.Error()), "Conflict. The container name") { log.Printf("[ERROR] Error in IO copy: %s", err) - shutdown(workflowExecution, action.ID, err.Error(), true) + shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) } else { if strings.Contains(buildBuf.String(), "errorDetail") { log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), image) - shutdown(workflowExecution, action.ID, err.Error(), true) + shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) } log.Printf("[INFO] Successfully downloaded %s", image) @@ -1003,13 +1002,13 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { log.Printf("[ERROR] Failed deploying image for the FOURTH time. Aborting if the image doesn't exist") if strings.Contains(err.Error(), "exited prematurely") { - shutdown(workflowExecution, action.ID, err.Error(), true) + shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) } if strings.Contains(err.Error(), "No such image") { //log.Printf("[WARNING] Failed deploying %s from image %s: %s", identifier, image, err) log.Printf("[ERROR] Image doesn't exist. Shutting down") - shutdown(workflowExecution, action.ID, err.Error(), true) + shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) } } } @@ -1018,7 +1017,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { err = deployApp(dockercli, images[0], identifier, env, workflowExecution) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { if strings.Contains(err.Error(), "exited prematurely") { - shutdown(workflowExecution, action.ID, err.Error(), true) + shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) } // Trying to replace with lowercase to deploy again. This seems to work with Dockerhub well. @@ -1031,7 +1030,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { err = deployApp(dockercli, image, identifier, env, workflowExecution) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { if strings.Contains(err.Error(), "exited prematurely") { - shutdown(workflowExecution, action.ID, err.Error(), true) + shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) } image = images[2] @@ -1042,25 +1041,25 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { err = deployApp(dockercli, image, identifier, env, workflowExecution) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { if strings.Contains(err.Error(), "exited prematurely") { - shutdown(workflowExecution, action.ID, err.Error(), true) + shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) } log.Printf("[WARNING] Failed deploying image THREE TIMES. Attempting to download the latter as last resort.") reader, err := dockercli.ImagePull(context.Background(), image, pullOptions) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { log.Printf("[ERROR] Failed getting %s. The couldn't be find locally, AND is missing.", image) - shutdown(workflowExecution, action.ID, err.Error(), true) + shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) } buildBuf := new(strings.Builder) _, err = io.Copy(buildBuf, reader) if err != nil { log.Printf("[ERROR] Error in IO copy: %s", err) - shutdown(workflowExecution, action.ID, err.Error(), true) + shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) } else { if strings.Contains(buildBuf.String(), "errorDetail") { log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), image) - shutdown(workflowExecution, action.ID, err.Error(), true) + shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) } log.Printf("[INFO] Successfully downloaded %s", image) @@ -1070,13 +1069,13 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { log.Printf("[ERROR] Failed deploying image for the FOURTH time. Aborting if the image doesn't exist") if strings.Contains(err.Error(), "exited prematurely") { - shutdown(workflowExecution, action.ID, err.Error(), true) + shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) } if strings.Contains(err.Error(), "No such image") { //log.Printf("[WARNING] Failed deploying %s from image %s: %s", identifier, image, err) log.Printf("[ERROR] Image doesn't exist. Shutting down") - shutdown(workflowExecution, action.ID, err.Error(), true) + shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) } } } From b8bcd50aadba913ba2a0b05b00a0c0902f677a89 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 18 May 2021 19:44:09 +0200 Subject: [PATCH 43/96] #223: Added basic resultgrabbing from subworkflow if NOT loop --- backend/go-app/walkoff.go | 123 ++++++++++++++++++++++++- frontend/src/views/AngularWorkflow.jsx | 19 ++-- frontend/src/views/Apps.jsx | 15 ++- frontend/src/views/Workflows.jsx | 37 ++++++-- functions/onprem/worker/worker.go | 8 -- 5 files changed, 177 insertions(+), 25 deletions(-) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 4e04cd71..4b100d48 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -1936,7 +1936,7 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request } } - for _, trigger := range workflowExecution.Workflow.Triggers { + for triggerIndex, trigger := range workflowExecution.Workflow.Triggers { //log.Printf("[INFO] ID: %s vs %s", trigger.ID, workflowExecution.Start) if trigger.ID == workflowExecution.Start { if trigger.AppName == "User Input" { @@ -1975,7 +1975,78 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request Status: "SKIPPED", }) } else { - //log.Printf("SHOULD KEEP TRIGGER %s", trigger.ID) + // Replaces trigger with the subflow + if trigger.AppName == "Shuffle Workflow" { + replaceActions := false + workflowAction := "" + for _, param := range trigger.Parameters { + if param.Name == "argument" && !strings.Contains(param.Value, ".#") { + replaceActions = true + } + + if param.Name == "startnode" { + workflowAction = param.Value + } + } + + if replaceActions { + replacementNodes, newBranches, lastnode := shuffle.GetReplacementNodes(ctx, workflowExecution, trigger) + log.Printf("REPLACEMENTS: %d, %d", len(replacementNodes), len(newBranches)) + if len(replacementNodes) > 0 { + //workflowExecution.Workflow.Actions = append(workflowExecution.Workflow.Actions, action) + + //lastnode = replacementNodes[0] + // Have to validate in case it's the same workflow and such + for _, action := range replacementNodes { + found := false + for subActionIndex, subaction := range newActions { + if subaction.ID == action.ID { + found = true + //newActions[subActionIndex].Name = action.Name + newActions[subActionIndex].Label = action.Label + break + } + } + + if !found { + newActions = append(newActions, action) + } + + // Check if it's already set to have a value + for resultIndex, result := range defaultResults { + if result.Action.ID == action.ID { + defaultResults = append(defaultResults[:resultIndex], defaultResults[resultIndex+1:]...) + break + } + } + } + + for _, branch := range newBranches { + workflowExecution.Workflow.Branches = append(workflowExecution.Workflow.Branches, branch) + } + + // Append branches: + // parent -> new inner node (FIRST one) + for branchIndex, branch := range workflowExecution.Workflow.Branches { + if branch.DestinationID == trigger.ID { + log.Printf("REPLACE DESTINATION WITH %s!!", workflowAction) + workflowExecution.Workflow.Branches[branchIndex].DestinationID = workflowAction + } + + if branch.SourceID == trigger.ID { + log.Printf("REPLACE SOURCE WITH LASTNODE %s!!", lastnode) + workflowExecution.Workflow.Branches[branchIndex].SourceID = lastnode + } + } + + // Remove the trigger + workflowExecution.Workflow.Triggers = append(workflowExecution.Workflow.Triggers[:triggerIndex], workflowExecution.Workflow.Triggers[triggerIndex+1:]...) + workflow.Triggers = append(workflow.Triggers[:triggerIndex], workflow.Triggers[triggerIndex+1:]...) + } + + log.Printf("NEW ACTION LENGTH %d, RESULT: %d, Triggers: %d", len(newActions), len(defaultResults), len(workflowExecution.Workflow.Triggers)) + } + } } } } @@ -4334,6 +4405,54 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { // workflowapp.Environment = baseEnvironment //} + // Fixes (appends) authentication parameters if they're required + if workflowapp.Authentication.Required { + log.Printf("[INFO] Checking authentication fields and appending for %s!", workflowapp.Name) + // FIXME: + // Might require reflection into the python code to append the fields as well + for index, action := range workflowapp.Actions { + if action.AuthNotRequired { + log.Printf("Skipping auth setup: %s", action.Name) + continue + } + + // 1. Check if authentication params exists at all + // 2. Check if they're present in the action + // 3. Add them IF they DONT exist + // 4. Fix python code with reflection (FIXME) + appendParams := []shuffle.WorkflowAppActionParameter{} + for _, fieldname := range workflowapp.Authentication.Parameters { + found := false + for index, param := range action.Parameters { + if param.Name == fieldname.Name { + found = true + + action.Parameters[index].Configuration = true + //log.Printf("Set config to true for field %s!", param.Name) + break + } + } + + if !found { + appendParams = append(appendParams, shuffle.WorkflowAppActionParameter{ + Name: fieldname.Name, + Description: fieldname.Description, + Example: fieldname.Example, + Required: fieldname.Required, + Configuration: true, + Schema: fieldname.Schema, + }) + } + } + + if len(appendParams) > 0 { + log.Printf("[AUTH] Appending %d params to the START of %s", len(appendParams), action.Name) + workflowapp.Actions[index].Parameters = append(appendParams, workflowapp.Actions[index].Parameters...) + } + + } + } + workflowapp.ID = uuid.NewV4().String() workflowapp.IsValid = true workflowapp.Generated = false diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index ee44b45b..3b11d6dd 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -563,7 +563,7 @@ const AngularWorkflow = (props) => { } } } else { - console.log("JSON is same") + //console.log("JSON is same") } //console.log("PRE LOOPING RESULTS: !", responseJson.execution_id, executionRequest.execution_id) @@ -732,7 +732,7 @@ const AngularWorkflow = (props) => { getWorkflowExecution(props.match.params.key, "") setUpdate(Math.random()) } else { - console.log("Nothing to update") + //console.log("Nothing to update") } } @@ -1596,10 +1596,13 @@ const AngularWorkflow = (props) => { //parentNode.data() var newNodeData = JSON.parse(JSON.stringify(parentNode.data())) newNodeData.id = uuid.v4() - newNodeData.position = { - "x": newNodeData.position.x+100, - "y": newNodeData.position.y+100, + if (newNodeData.position !== undefined) { + newNodeData.position = { + "x": newNodeData.position.x+100, + "y": newNodeData.position.y+100, + } } + newNodeData.isStartNode = false newNodeData.errors = [] newNodeData.is_valid = true @@ -1713,6 +1716,7 @@ const AngularWorkflow = (props) => { } const curapp = apps.find(a => a.name === curaction.app_name && ((a.app_version === curaction.app_version || (a.loop_versions !== null && a.loop_versions.includes(curaction.app_version))))) + console.log("APP: ", curapp) if (!curapp || curapp === undefined) { alert.error(`App ${curaction.app_name}:${curaction.app_version} not found. Is it activated?`) @@ -1734,6 +1738,7 @@ const AngularWorkflow = (props) => { //console.log("AUTHENTICATION: ", curapp.authentication) setRequiresAuthentication(curapp.authentication.required && curapp.authentication.parameters !== undefined && curapp.authentication.parameters !== null) if (curapp.authentication.required) { + console.log("App requires auth.") // Setup auth here :) const authenticationOptions = [] var findAuthId = "" @@ -2793,7 +2798,9 @@ const AngularWorkflow = (props) => { .then((responseJson) => { // No matter what, it's being stopped. if (!responseJson.success) { - alert.WARNING("Failed to stop schedule: " + responseJson.reason) + if (responseJson.reason !== undefined) { + alert.error("Failed to stop schedule: " + responseJson.reason) + } } else { alert.success("Successfully stopped schedule") } diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index a2255cc6..8beddedb 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -3,7 +3,7 @@ import React, { useEffect } from 'react'; import { useInterval } from 'react-powerhooks'; import {IconButton, Typography, Grid, Select, Paper, Divider, ButtonBase, Button, TextField, FormControl, MenuItem, Tooltip, FormControlLabel, Switch, Input, Breadcrumbs, Chip, Dialog, DialogTitle, DialogActions, DialogContent, CircularProgress} from '@material-ui/core'; -import {OpenInNew as OpenInNewIcon,Apps as AppsIcon, Cached as CachedIcon, Publish as PublishIcon, CloudDownload as CloudDownloadIcon, Edit as EditIcon, Delete as DeleteIcon} from '@material-ui/icons'; +import {LockOpen as LockOpenIcon, OpenInNew as OpenInNewIcon,Apps as AppsIcon, Cached as CachedIcon, Publish as PublishIcon, CloudDownload as CloudDownloadIcon, Edit as EditIcon, Delete as DeleteIcon} from '@material-ui/icons'; import { useTheme } from '@material-ui/core/styles'; @@ -333,6 +333,10 @@ const Apps = (props) => { // dropdown with copy etc I guess const appPaper = (data) => { + if (data.name === "" && data.id === "") { + return null + } + var boxWidth = "2px" if (selectedApp.id === data.id) { boxWidth = "4px" @@ -801,9 +805,14 @@ const Apps = (props) => { const circleSize = 10 return ( -
+ {data.configuration === true ? + + + + : +
+ } {data.name} - ) })} diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 5064691e..b009de75 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -311,8 +311,29 @@ const Workflows = (props) => { if (curWorkflow.tags === undefined || curWorkflow.tags === null) { found = filters.map(filter => curWorkflow.name.toLowerCase().includes(filter)) } else { - found = filters.map(filter => curWorkflow.name.toLowerCase().includes(filter.toLowerCase()) || curWorkflow.tags.includes(filter)) + found = filters.map(filter => { + if (filter === undefined) { + return false + } + + if (curWorkflow.name.toLowerCase().includes(filter.toLowerCase())) { + return true + } else if (curWorkflow.tags.includes(filter)) { + return true + } else if (curWorkflow.actions !== null && curWorkflow.actions !== undefined) { + const newfilter = filter.toLowerCase() + for (var key in curWorkflow.actions) { + const action = curWorkflow.actions[key] + if (action.app_name.toLowerCase().includes(newfilter)) { + return true + } + } + } + + return false + }) } + //console.log("FOUND: ", found) //if (found) { if (found.every(v => v === true)) { @@ -604,6 +625,8 @@ const Workflows = (props) => { const paperAppStyle = { minHeight: 130, + maxHeight: 130, + overflow: "hidden", width: "100%", color: "white", backgroundColor: surfaceColor, @@ -1503,7 +1526,7 @@ const Workflows = (props) => { const data = params.row.record; let [triggers, schedules, webhooks, subflows] = getWorkflowMeta(data); - return + return ( { : null} - } + ) + } }, { field: 'tags', headerName: 'Tags', width: 390, sortable: false, disableClickEventBubbling: true, renderCell: (params) => { const data = params.row.record; - return + return ( {data.tags !== undefined ? data.tags.map((tag, index) => { @@ -1556,6 +1580,7 @@ const Workflows = (props) => { }) : null} + ) } }, ]; @@ -1707,11 +1732,11 @@ const Workflows = (props) => { const workflowButtons = - {workflows.length > 0 ? + {/*workflows.length > 0 ? - : null} + : null*/} {importLoading ? + : } + + ) + })} +
+ const apiKey = authenticationOption === "API key" ?
API key authentication @@ -1447,7 +1545,7 @@ const AppCreator = (props) => { setParameterLocation(e.target.value) }} value={parameterLocation} - style={{backgroundColor: inputColor, paddingLeft: "10px", color: "white", height: "50px"}} + style={{borderRadius: 5, backgroundColor: inputColor, paddingLeft: "10px", color: "white", height: "50px"}} inputProps={{ name: 'age', id: 'outlined-age-simple', @@ -2628,6 +2726,7 @@ const AppCreator = (props) => { {basicAuth} {bearerAuth} {apiKey} + {extraKeys} {/*authenticationOption === "No authentication" ? null : Date: Mon, 24 May 2021 19:26:11 +0200 Subject: [PATCH 53/96] Added global headers, queries and more control app creator --- frontend/src/views/AppCreator.jsx | 104 +++++++++++++++++++++--------- functions/onprem/worker/build.sh | 2 +- functions/onprem/worker/worker.go | 32 ++++----- 3 files changed, 91 insertions(+), 47 deletions(-) diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 26e7c3ba..873ec947 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -3,8 +3,7 @@ import { makeStyles } from '@material-ui/styles'; import {BrowserView, MobileView} from "react-device-detect"; import {Paper, Typography, FormControlLabel, Button, Divider, Select, MenuItem, FormControl, Switch, Dialog, DialogTitle, DialogContent, DialogActions, TextField, Tooltip, Breadcrumbs, CircularProgress, Chip} from '@material-ui/core'; -import {Add as AddIcon, CheckCircle as CheckCircleIcon, AttachFile as AttachFileIcon, Apps as AppsIcon, ErrorOutline as ErrorOutlineIcon} from '@material-ui/icons'; - +import {FileCopy as FileCopyIcon, Delete as DeleteIcon, Remove as RemoveIcon, Add as AddIcon, CheckCircle as CheckCircleIcon, AttachFile as AttachFileIcon, Apps as AppsIcon, ErrorOutline as ErrorOutlineIcon} from '@material-ui/icons'; import {Link} from 'react-router-dom'; import YAML from 'yaml' @@ -198,7 +197,8 @@ const AppCreator = (props) => { const increaseAmount = 50 const actionNonBodyRequest = ["GET", "HEAD", "DELETE", "CONNECT"] const actionBodyRequest = ["POST", "PUT", "PATCH",] - const authenticationOptions = ["No authentication", "API key", "Bearer auth", "Basic auth", ] + //const authenticationOptions = ["No authentication", "API key", "Bearer auth", "Basic auth", "Oauth2"] + const authenticationOptions = ["No authentication", "API key", "Bearer auth", "Basic auth"] const apikeySelection = ["Header", "Query",] const [name, setName] = useState(""); @@ -233,9 +233,9 @@ const AppCreator = (props) => { const defaultAuth = { "name": "", "type": "header", - "example": "hello", + "example": "", } - const [extraAuth, setExtraAuth] = useState([defaultAuth]) + const [extraAuth, setExtraAuth] = useState([]) //const [actions, setActions] = useState([{ // "name": "Get workflows", @@ -898,6 +898,10 @@ const AppCreator = (props) => { setAuthenticationOption("Basic auth") setAuthenticationRequired(true) break + } else if (value.scheme === "oauth2") { + setAuthenticationOption("Oauth2") + setAuthenticationRequired(true) + break } } } @@ -1256,6 +1260,26 @@ const AppCreator = (props) => { "type": "http", "scheme": "basic", } + } else if (authenticationOption === "Oauth2") { + data.components.securitySchemes["Oauth2"] = { + "type": "oauth2", + "flow": { + "authorizationCode": { + + }, + }, + } + } + + if (setExtraAuth.length > 0) { + for (var key in extraAuth) { + const curauth = extraAuth[key] + data.components.securitySchemes[curauth.name] = { + "type": "apiKey", + "in": curauth.type, + "name": curauth.name, + } + } } fetch(globalUrl+"/api/v1/verify_openapi", { @@ -1420,13 +1444,24 @@ const AppCreator = (props) => { //console.log("Location: ", parameterLocation) //console.log("Name: ", parameterName) const extraKeys = -
+
+ Extra headers or queries + {extraAuth.length === 0 ? + + : } {extraAuth.map((value, index) => { return ( - + { setExtraAuth(extraAuth) }} InputProps={{ - //classes: { - // notchedOutline: classes.notchedOutline, - //}, + classes: { + notchedOutline: classes.notchedOutline, + }, style:{ color: "white", minHeight: 50, @@ -1452,9 +1487,9 @@ const AppCreator = (props) => { /> { - {index === extraAuth.length-1 ? - - : } + : } + +
) })}
+ const oauth2Auth = authenticationOption === "Oauth2" ? +
+ Oauth2 authentication + + Add the URL to redirect to + +
+ : null + const apiKey = authenticationOption === "API key" ?
API key authentication @@ -1682,12 +1734,12 @@ const AppCreator = (props) => {
{ duplicateAction(index) }}> - Duplicate +
{deleteAction(index)}}> - Delete +
@@ -1777,9 +1829,6 @@ const AppCreator = (props) => { } key={currentAction} InputProps={{ - classes: { - notchedOutline: classes.notchedOutline, - }, style:{ color: "white", }, @@ -2015,9 +2064,6 @@ const AppCreator = (props) => { defaultValue={currentAction["description"]} onChange={e => setActionField("description", e.target.value)} InputProps={{ - classes: { - notchedOutline: classes.notchedOutline, - }, style:{ color: "white", }, @@ -2220,9 +2266,6 @@ const AppCreator = (props) => { onChange={e => setActionField("headers", e.target.value)} helperText={Headers that are part of the request. Default: EMPTY} InputProps={{ - classes: { - notchedOutline: classes.notchedOutline, - }, style:{ color: "white", }, @@ -2726,6 +2769,7 @@ const AppCreator = (props) => { {basicAuth} {bearerAuth} {apiKey} + {oauth2Auth} {extraKeys} {/*authenticationOption === "No authentication" ? null : diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh index a0c83665..d3c6b70a 100644 --- a/functions/onprem/worker/build.sh +++ b/functions/onprem/worker/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-worker -VERSION=0.8.90 +VERSION=0.8.92 echo "Running docker build with $NAME:$VERSION" #CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin . diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index e5b79140..84b4c578 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -974,31 +974,31 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { err = deployApp(dockercli, images[0], identifier, env, workflowExecution) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { if strings.Contains(err.Error(), "exited prematurely") { - shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) + shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } image = images[2] err = deployApp(dockercli, image, identifier, env, workflowExecution) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { if strings.Contains(err.Error(), "exited prematurely") { - shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) + shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } log.Printf("[WARNING] Failed CLEANUP execution. Downloading image remotely.") reader, err := dockercli.ImagePull(context.Background(), image, pullOptions) if err != nil { log.Printf("[ERROR] Failed getting %s. Couldn't be find locally, AND is missing.", image) - shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) + shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } buildBuf := new(strings.Builder) _, err = io.Copy(buildBuf, reader) - if err != nil && !strings.Contains(fmt.Sprintf("Docker error: %s", err.Error()), "Conflict. The container name") { + if err != nil && !strings.Contains(fmt.Sprintf("%s", err.Error()), "Conflict. The container name") { log.Printf("[ERROR] Error in IO copy: %s", err) - shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) + shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } else { if strings.Contains(buildBuf.String(), "errorDetail") { log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), image) - shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) + shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } log.Printf("[INFO] Successfully downloaded %s", image) @@ -1009,13 +1009,13 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { log.Printf("[ERROR] Failed deploying image for the FOURTH time. Aborting if the image doesn't exist") if strings.Contains(err.Error(), "exited prematurely") { - shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) + shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } if strings.Contains(err.Error(), "No such image") { //log.Printf("[WARNING] Failed deploying %s from image %s: %s", identifier, image, err) log.Printf("[ERROR] Image doesn't exist. Shutting down") - shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) + shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } } } @@ -1025,7 +1025,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { err = deployApp(dockercli, images[0], identifier, env, workflowExecution) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { if strings.Contains(err.Error(), "exited prematurely") { - shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) + shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } // Trying to replace with lowercase to deploy again. This seems to work with Dockerhub well. @@ -1034,32 +1034,32 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { err = deployApp(dockercli, image, identifier, env, workflowExecution) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { if strings.Contains(err.Error(), "exited prematurely") { - shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) + shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } image = images[2] err = deployApp(dockercli, image, identifier, env, workflowExecution) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { if strings.Contains(err.Error(), "exited prematurely") { - shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) + shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } log.Printf("[WARNING] Failed deploying image THREE TIMES. Attempting to download the latter as last resort.") reader, err := dockercli.ImagePull(context.Background(), image, pullOptions) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { log.Printf("[ERROR] Failed getting %s. The couldn't be find locally, AND is missing.", image) - shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) + shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } buildBuf := new(strings.Builder) _, err = io.Copy(buildBuf, reader) if err != nil { log.Printf("[ERROR] Error in IO copy: %s", err) - shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) + shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } else { if strings.Contains(buildBuf.String(), "errorDetail") { log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), image) - shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) + shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } log.Printf("[INFO] Successfully downloaded %s", image) @@ -1069,13 +1069,13 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { log.Printf("[ERROR] Failed deploying image for the FOURTH time. Aborting if the image doesn't exist") if strings.Contains(err.Error(), "exited prematurely") { - shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) + shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } if strings.Contains(err.Error(), "No such image") { //log.Printf("[WARNING] Failed deploying %s from image %s: %s", identifier, image, err) log.Printf("[ERROR] Image doesn't exist. Shutting down") - shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) + shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } } } From d3834f52d5e02e5ad8cfad2d7917e6ef1959e094 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 25 May 2021 18:36:43 +0200 Subject: [PATCH 54/96] Added workflow storage set and get --- backend/go-app/main.go | 25 +++++------ backend/tests/cache.sh | 15 +++++++ frontend/src/views/AppCreator.jsx | 45 ++++++++++++-------- frontend/src/views/Workflows.jsx | 2 +- functions/extensions/wazuh/custom-shuffle.py | 11 +++-- 5 files changed, 62 insertions(+), 36 deletions(-) create mode 100644 backend/tests/cache.sh diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 7d6f8455..72fa2ae8 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -4888,7 +4888,7 @@ func runInit(ctx context.Context) { cloneOptions.ReferenceName = plumbing.ReferenceName(branch) } - log.Printf("Getting apps from %s", url) + log.Printf("[DEBUG] Getting apps from %s", url) r, err := git.Clone(storer, fs, cloneOptions) @@ -5793,20 +5793,28 @@ func initHandlers() { r.HandleFunc("/api/v1/validate_openapi", shuffle.ValidateSwagger).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/get_openapi/{key}", getOpenapi).Methods("GET", "OPTIONS") - // NEW for 0.8.0 + // Specific triggers + r.HandleFunc("/api/v1/triggers/outlook/register", handleNewOutlookRegister).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/triggers/outlook/getFolders", shuffle.HandleGetOutlookFolders).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/triggers/outlook/{key}", handleGetSpecificTrigger).Methods("GET", "OPTIONS") + //r.HandleFunc("/api/v1/triggers/outlook/{key}/callback", handleOutlookCallback).Methods("POST", "OPTIONS") + //r.HandleFunc("/api/v1/stats/{key}", handleGetSpecificStats).Methods("GET", "OPTIONS") + + // EVERYTHING below here is NEW for 0.8.0 (written 25.05.2021) r.HandleFunc("/api/v1/workflows/{key}/publish", makeWorkflowPublic).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/cloud/setup", handleCloudSetup).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/orgs", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}", shuffle.HandleGetOrg).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}", shuffle.HandleEditOrg).Methods("POST", "OPTIONS") + // This is a new API that validates if a key has been seen before. // Not sure what the best course of action is for it. r.HandleFunc("/api/v1/orgs/{orgId}/validate_app_values", shuffle.HandleKeyValueCheck).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/orgs/{orgId}/get_cache", shuffle.HandleGetCacheKey).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/orgs/{orgId}/set_cache", shuffle.HandleSetCacheKey).Methods("POST", "OPTIONS") - //r.HandleFunc("/api/v1/orgs/{orgId}", handleEditOrg).Methods("POST", "OPTIONS") - - // Docker orborus specific + // Docker orborus specific - downloads an image r.HandleFunc("/api/v1/get_docker_image", getDockerImage).Methods("POST", "OPTIONS") //r.HandleFunc("/api/v1/migrate_database", migrateDatabase).Methods("POST", "OPTIONS") @@ -5821,13 +5829,6 @@ func initHandlers() { r.HandleFunc("/api/v1/files/{fileId}", shuffle.HandleDeleteFile).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/files", shuffle.HandleGetFiles).Methods("GET", "OPTIONS") - // Trigger hmm - r.HandleFunc("/api/v1/triggers/outlook/register", handleNewOutlookRegister).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/triggers/outlook/getFolders", shuffle.HandleGetOutlookFolders).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/triggers/outlook/{key}", handleGetSpecificTrigger).Methods("GET", "OPTIONS") - //r.HandleFunc("/api/v1/triggers/outlook/{key}/callback", handleOutlookCallback).Methods("POST", "OPTIONS") - //r.HandleFunc("/api/v1/stats/{key}", handleGetSpecificStats).Methods("GET", "OPTIONS") - http.Handle("/", r) } diff --git a/backend/tests/cache.sh b/backend/tests/cache.sh new file mode 100644 index 00000000..401d9ce1 --- /dev/null +++ b/backend/tests/cache.sh @@ -0,0 +1,15 @@ +curl -XPOST http://192.168.3.8:5001/api/v1/orgs/6a6a99f5-6630-4f91-88ff-571c9f030ea0/set_cache -H "Authorization: Bearer e663cf93-7f10-4560-bef0-303f14aad982" -d '{ + "workflow_id": "61825389-a125-43a5-9119-97c401e9934b", + "execution_id": "2c8ca1b7-6658-4742-86a1-105c9467702d", + "org_id": "6a6a99f5-6630-4f91-88ff-571c9f030ea0", + "key": "test", + "value": "THIS IS SOME DATA HELLO" +}' + +curl -XPOST http://192.168.3.8:5001/api/v1/orgs/6a6a99f5-6630-4f91-88ff-571c9f030ea0/get_cache -H "Authorization: Bearer e663cf93-7f10-4560-bef0-303f14aad982" -d '{ + "workflow_id": "61825389-a125-43a5-9119-97c401e9934b", + "execution_id": "2c8ca1b7-6658-4742-86a1-105c9467702d", + "org_id": "6a6a99f5-6630-4f91-88ff-571c9f030ea0", + "key": "test" +}' + diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 873ec947..0f0981d0 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -871,16 +871,18 @@ const AppCreator = (props) => { } } + + console.log("SECURITYSCHEMES: ", securitySchemes) if (securitySchemes !== undefined) { // FIXME: Should add Oauth2 (Microsoft) and JWT (Wazuh) //console.log("SECURITY: ", securitySchemes) //if (Object.entries(securitySchemes) > 1 && + var newauth = [] for (const [key, value] of Object.entries(securitySchemes)) { if (value.scheme === "bearer") { setAuthenticationOption("Bearer auth") setAuthenticationRequired(true) - break - } else if (value.type === "apiKey") { + } else if (key === "ApiKeyAuth") { setAuthenticationOption("API key") value.in = value.in.charAt(0).toUpperCase() + value.in.slice(1); @@ -893,17 +895,24 @@ const AppCreator = (props) => { console.log("PARAM NAME: ", value.name) setParameterName(value.name) setAuthenticationRequired(true) - break } else if (value.scheme === "basic") { setAuthenticationOption("Basic auth") setAuthenticationRequired(true) - break } else if (value.scheme === "oauth2") { setAuthenticationOption("Oauth2") setAuthenticationRequired(true) - break + } else { + newauth.push({ + "name": key, + "type": value.in, + "example": "", + }) } } + + if (newauth.length > 0) { + setExtraAuth(newauth) + } } if (newActions.length > increaseAmount-1) { @@ -1445,20 +1454,22 @@ const AppCreator = (props) => { //console.log("Name: ", parameterName) const extraKeys =
- Extra headers or queries - {extraAuth.length === 0 ? - - : } +
+ Extra authentication options + {extraAuth.length === 0 ? + + : } +
{extraAuth.map((value, index) => { return ( - + { {"key": "download", "values": ["get", "download", "return", "hello_world", "curl",]}, {"key": "search", "values": ["search", "find"]}, {"key": "delete", "values": ["delete", "remove"]}, - {"key": "send", "values": ["send", "dispatch", "mail", "forward", "post", "submit", "mark"]}, + {"key": "send", "values": ["send", "dispatch", "mail", "forward", "post", "submit", "mark", "set"]}, {"key": "repeat", "values": ["repeat", "retry", "pause",]}, {"key": "execute", "values": ["execute", "run", "play", "raise",]}, {"key": "extract", "values": ["extract", "unpack", "decompress"]}, diff --git a/functions/extensions/wazuh/custom-shuffle.py b/functions/extensions/wazuh/custom-shuffle.py index 06fa4c7d..e7900a4d 100644 --- a/functions/extensions/wazuh/custom-shuffle.py +++ b/functions/extensions/wazuh/custom-shuffle.py @@ -74,8 +74,7 @@ def debug(msg): # Skips container kills to stop self-recursion def filter_msg(alert): # These are things that recursively happen because Shuffle starts Docker containers - # Docker integration rules: https://github.com/wazuh/wazuh-ruleset/blob/ae36745db1d3f312db0392f5925c2f2b0ec009a9/rules/0560-docker_integration_rules.xml - skip = ["87924", "87900", "87901", "87902", "87903", "87904", "86001", "86002", "86003", "87932", "80710", "87929", "87928",] + skip = ["87924", "87900", "87901", "87902", "87903", "87904", "86001", "86002", "86003", "87932", "80710", "87929", "87928", "5710"] if alert["rule"]["id"] in skip: return False @@ -96,14 +95,14 @@ def generate_msg(alert): level = alert['rule']['level'] if (level <= 4): - color = "good" + severity = 1 elif (level >= 5 and level <= 7): - color = "warning" + severity = 2 else: - color = "danger" + severity = 3 msg = {} - msg['color'] = color + msg['severity'] = severity msg['pretext'] = "WAZUH Alert" msg['title'] = alert['rule']['description'] if 'description' in alert['rule'] else "N/A" msg['text'] = alert.get('full_log') From c64e24aca99c8305d8545da438dedfccb0f4c0b0 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 25 May 2021 20:31:25 +0200 Subject: [PATCH 55/96] BUFIX: Dragged apps spawn in correct location --- backend/go-app/go.mod | 4 +- backend/go-app/go.sum | 2 + docker-compose.yml | 4 +- frontend/src/views/AngularWorkflow.jsx | 406 +++++++++++++------------ 4 files changed, 218 insertions(+), 198 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 5b32f667..74d4f4f7 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -2,7 +2,7 @@ module shuffle go 1.13 -replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared +//replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi @@ -24,7 +24,7 @@ require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 // indirect github.com/elastic/go-elasticsearch/v8 v8.0.0-20210519083322-55daf7425ecb // indirect github.com/frikky/kin-openapi v0.39.0 - github.com/frikky/shuffle-shared v0.0.49 + github.com/frikky/shuffle-shared v0.0.50 github.com/fsouza/go-dockerclient v1.7.2 // indirect github.com/ghodss/yaml v1.0.0 github.com/go-git/go-billy/v5 v5.0.0 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 8699383f..759c5d66 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -157,6 +157,8 @@ github.com/frikky/shuffle-shared v0.0.47 h1:fywEmbJGbD/VT9LdkHvLWPGh9HEt8ipFDzoG github.com/frikky/shuffle-shared v0.0.47/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= github.com/frikky/shuffle-shared v0.0.49 h1:fChF0Nh/bMuXZg67Pt9XXn9+mH4IlKgB3dAzhqwQF5o= github.com/frikky/shuffle-shared v0.0.49/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= +github.com/frikky/shuffle-shared v0.0.50 h1:dQIXf4mwUHuEVsXiMtZaSznz6vWt+C0KjyTAsAgMs3s= +github.com/frikky/shuffle-shared v0.0.50/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= github.com/fsouza/go-dockerclient v1.7.2 h1:bBEAcqLTkpq205jooP5RVroUKiVEWgGecHyeZc4OFjo= github.com/fsouza/go-dockerclient v1.7.2/go.mod h1:+ugtMCVRwnPfY7d8/baCzZ3uwB0BrG5DB8OzbtxaRz8= github.com/getkin/kin-openapi v0.8.0 h1:a6TQjTqwkyscC4/hShJX7WhCVE+4bi9lzw61XHQW5hE= diff --git a/docker-compose.yml b/docker-compose.yml index eac89280..05b999e3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: frontend: #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.91 + image: ghcr.io/frikky/shuffle-frontend:0.8.92 container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -17,7 +17,7 @@ services: - backend backend: #build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.91 + image: ghcr.io/frikky/shuffle-backend:0.8.92 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 5e95de26..af5d7437 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -3379,200 +3379,6 @@ const AngularWorkflow = (props) => { } } - const handleAppDrag = (e, app) => { - const cycontainer = cy.container() - - // Chrome lol - //if (e.srcElement !== undefined && e.srcElement.localName === "canvas") { - if (e.pageX > cycontainer.offsetLeft && e.pageX < cycontainer.offsetLeft+cycontainer.offsetWidth && e.pageY > cycontainer.offsetTop && e.pageY < cycontainer.offsetTop+cycontainer.offsetHeight) { - if (newNodeId.length > 0) { - var currentnode = cy.getElementById(newNodeId) - if (currentnode.length === 0) { - return - } - - currentnode[0].renderedPosition("x", e.pageX-cycontainer.offsetLeft) - currentnode[0].renderedPosition("y", e.pageY-cycontainer.offsetTop) - } else{ - if (workflow.public) { - return - } - - if (app.actions === undefined || app.actions === null || app.actions.length === 0) { - alert.error("App "+app.name+" currently has no actions to perform. Please go to https://shuffler.io/apps to edit it.") - return - } - - newNodeId = uuid.v4() - const actionType = "ACTION" - const actionLabel = getNextActionName(app.name) - var parameters = null - var example = "" - - if (app.actions[0].parameters !== null && app.actions[0].parameters.length > 0) { - parameters = app.actions[0].parameters - } - if (app.actions[0].returns.example !== undefined && app.actions[0].returns.example !== null && app.actions[0].returns.example.length > 0) { - example = app.actions[0].returns.example - } - - var newAppPopup = false - - /* - FIXME: Add auth. - selectedAction.selectedAuthentication = e.target.value - selectedAction.authentication_id = e.target.value.id - setSelectedAction(selectedAction) - setUpdate(Math.random()) - */ - - console.log("ENVS: ", environments) - const parsedEnvironments = environments === null || environments === [] ? "cloud" : environments[defaultEnvironmentIndex] === undefined ? "cloud" : environments[defaultEnvironmentIndex].Name - const newAppData = { - app_name: app.name, - app_version: app.app_version, - app_id: app.id, - sharing: app.sharing, - private_id: app.private_id, - environment: parsedEnvironments, - errors: [], - id_: newNodeId, - _id_: newNodeId, - id: newNodeId, - is_valid: true, - label: actionLabel, - type: actionType, - name: app.actions[0].name, - parameters: parameters, - isStartNode: false, - large_image: app.large_image, - authentication: [], - execution_variable: undefined, - example: example, - category: app.categories !== null && app.categories !== undefined && app.categories.length > 0 ? app.categories[0] : "", - authentication_id: "", - } - - // FIXME: overwrite category if the ACTION chosen has a different category - - // const image = "url("+app.large_image+")" - - // FIXME - find the cytoscape offset position - // Can this be done with zoom calculations? - const nodeToBeAdded = { - group: "nodes", - data: newAppData, - renderedPosition: { - x: e.layerX, - y: e.layerY, - } - } - - cy.add(nodeToBeAdded) - - if (workflow.actions === undefined || workflow.actions.length === 0) { - workflow.start = newAppData.id - workflow.actions = [] - newAppData.isStartNode = true - //setStartNode(newAppData.id) - } - - if (workflow.actions.length > 0 && elements.length === 0) { - const actions = workflow.actions.map(action => { - const node = {} - node.position = action.position - node.data = action - - node.data._id = action["id"] - node.data.type = "ACTION" - node.isStartNode = action["id"] === workflow.start - - return node - }) - - const tmpelements = [].concat(actions) - setElements(tmpelements) - } - - if (workflow.actions.length === 1 && workflow.actions[0].id === workflow.start) { - const newEdgeUuid = uuid.v4() - const newcybranch = { - "source": workflow.start, - "target": newNodeId, - "_id": newEdgeUuid, - "id": newEdgeUuid, - "hasErrors": false, - } - - const edgeToBeAdded = { - group: "edges", - data: newcybranch, - } - console.log("SHOULD STITCH WITH STARTNODE") - cy.add(edgeToBeAdded) - } - - // AUTHENTICATION - if (app.authentication.required) { - // Setup auth here :) - const authenticationOptions = [] - var findAuthId = "" - if (newAppData.authentication_id !== null && newAppData.authentication_id !== undefined && newAppData.authentication_id.length > 0) { - findAuthId = newAppData.authentication_id - } - - var tmpAuth = JSON.parse(JSON.stringify(appAuthentication)) - for (var key in tmpAuth) { - var item = tmpAuth[key] - - const newfields = {} - for (var filterkey in item.fields) { - newfields[item.fields[filterkey].key] = item.fields[filterkey].value - } - - item.fields = newfields - if (item.app.name === app.name) { - authenticationOptions.push(item) - if (item.id === findAuthId) { - newAppData.selectedAuthentication = item - } - } - } - - if (authenticationOptions !== undefined && authenticationOptions !== null && authenticationOptions.length > 0) { - for (var key in authenticationOptions) { - const option = authenticationOptions[key] - if (option.active) { - newAppData.selectedAuthentication = option - newAppData.authentication_id = option.id - break - } - } - } - - //newAppData.authentication = authenticationOptions - //if (newAppData.selectedAuthentication === null || newAppData.selectedAuthentication === undefined || newAppData.selectedAuthentication.length === "") { - // newAppData.selectedAuthentication = {} - //} else { - // console.log("CAN WE SELECT AUTH?: ", authenticationOptions) - //} - } else { - newAppData.authentication = [] - newAppData.authentication_id = "" - newAppData.selectedAuthentication = {} - } - - workflow.actions.push(newAppData) - setWorkflow(workflow) - - if (newAppPopup) { - //alert.error("SHOULD MAKE USER AUTHENTICATE THE APP OR SET hasError") - //alert.info("Remember: set the authentication for the user itself, not the app") - } - } - } - } - const handleDragStop = (e, app) => { newNodeId = "" console.log("STOP!: ", e) @@ -3592,6 +3398,212 @@ const AngularWorkflow = (props) => { const { allApps, prioritizedApps, filteredApps } = props; const [visibleApps, setVisibleApps] = React.useState(prioritizedApps.concat(filteredApps.filter(innerapp => !internalIds.includes(innerapp.id)))) + const handleAppDrag = (e, app) => { + const cycontainer = cy.container() + console.log("X: ", e.pageX) + console.log("Y: ", e.pageY) + console.log("Height: ", cycontainer.offsetHeight) + + // Chrome lol + //if (e.srcElement !== undefined && e.srcElement.localName === "canvas") { + if (e.pageX > cycontainer.offsetLeft && e.pageX < cycontainer.offsetLeft+cycontainer.offsetWidth && e.pageY > cycontainer.offsetTop && e.pageY < cycontainer.offsetTop+cycontainer.offsetHeight) { + if (newNodeId.length > 0) { + var currentnode = cy.getElementById(newNodeId) + if (currentnode.length === 0) { + return + } + + console.log("RENDEREDX: ", e.pageX-cycontainer.offsetLeft) + console.log("RENDEREDY: ", e.pageY-cycontainer.offsetTop) + currentnode[0].renderedPosition("x", e.pageX-cycontainer.offsetLeft) + currentnode[0].renderedPosition("y", e.pageY-cycontainer.offsetTop) + } else{ + if (workflow.public) { + return + } + + if (app.actions === undefined || app.actions === null || app.actions.length === 0) { + alert.error("App "+app.name+" currently has no actions to perform. Please go to https://shuffler.io/apps to edit it.") + return + } + + newNodeId = uuid.v4() + const actionType = "ACTION" + const actionLabel = getNextActionName(app.name) + var parameters = null + var example = "" + + if (app.actions[0].parameters !== null && app.actions[0].parameters.length > 0) { + parameters = app.actions[0].parameters + } + if (app.actions[0].returns.example !== undefined && app.actions[0].returns.example !== null && app.actions[0].returns.example.length > 0) { + example = app.actions[0].returns.example + } + + var newAppPopup = false + + /* + FIXME: Add auth. + selectedAction.selectedAuthentication = e.target.value + selectedAction.authentication_id = e.target.value.id + setSelectedAction(selectedAction) + setUpdate(Math.random()) + */ + + console.log("ENVS: ", environments) + const parsedEnvironments = environments === null || environments === [] ? "cloud" : environments[defaultEnvironmentIndex] === undefined ? "cloud" : environments[defaultEnvironmentIndex].Name + const newAppData = { + app_name: app.name, + app_version: app.app_version, + app_id: app.id, + sharing: app.sharing, + private_id: app.private_id, + environment: parsedEnvironments, + errors: [], + id_: newNodeId, + _id_: newNodeId, + id: newNodeId, + is_valid: true, + label: actionLabel, + type: actionType, + name: app.actions[0].name, + parameters: parameters, + isStartNode: false, + large_image: app.large_image, + authentication: [], + execution_variable: undefined, + example: example, + category: app.categories !== null && app.categories !== undefined && app.categories.length > 0 ? app.categories[0] : "", + authentication_id: "", + } + + // FIXME: overwrite category if the ACTION chosen has a different category + + // const image = "url("+app.large_image+")" + + // FIXME - find the cytoscape offset position + // Can this be done with zoom calculations? + console.log("LAYERX: ", e.layerX) + console.log("LAYERY: ", e.layerY) + console.log("RENDEREDX: ", e.pageX-cycontainer.offsetLeft) + console.log("RENDEREDY: ", e.pageY-cycontainer.offsetTop) + console.log("E: ", e) + const nodeToBeAdded = { + group: "nodes", + data: newAppData, + renderedPosition: { + //x: e.layerX, + //y: e.layerY, + x: e.pageX-cycontainer.offsetLeft, + y: e.pageY-cycontainer.offsetTop, + } + } + + cy.add(nodeToBeAdded) + + if (workflow.actions === undefined || workflow.actions.length === 0) { + workflow.start = newAppData.id + workflow.actions = [] + newAppData.isStartNode = true + //setStartNode(newAppData.id) + } + + if (workflow.actions.length > 0 && elements.length === 0) { + const actions = workflow.actions.map(action => { + const node = {} + node.position = action.position + node.data = action + + node.data._id = action["id"] + node.data.type = "ACTION" + node.isStartNode = action["id"] === workflow.start + + return node + }) + + const tmpelements = [].concat(actions) + setElements(tmpelements) + } + + if (workflow.actions.length === 1 && workflow.actions[0].id === workflow.start) { + const newEdgeUuid = uuid.v4() + const newcybranch = { + "source": workflow.start, + "target": newNodeId, + "_id": newEdgeUuid, + "id": newEdgeUuid, + "hasErrors": false, + } + + const edgeToBeAdded = { + group: "edges", + data: newcybranch, + } + console.log("SHOULD STITCH WITH STARTNODE") + cy.add(edgeToBeAdded) + } + + // AUTHENTICATION + if (app.authentication.required) { + // Setup auth here :) + const authenticationOptions = [] + var findAuthId = "" + if (newAppData.authentication_id !== null && newAppData.authentication_id !== undefined && newAppData.authentication_id.length > 0) { + findAuthId = newAppData.authentication_id + } + + var tmpAuth = JSON.parse(JSON.stringify(appAuthentication)) + for (var key in tmpAuth) { + var item = tmpAuth[key] + + const newfields = {} + for (var filterkey in item.fields) { + newfields[item.fields[filterkey].key] = item.fields[filterkey].value + } + + item.fields = newfields + if (item.app.name === app.name) { + authenticationOptions.push(item) + if (item.id === findAuthId) { + newAppData.selectedAuthentication = item + } + } + } + + if (authenticationOptions !== undefined && authenticationOptions !== null && authenticationOptions.length > 0) { + for (var key in authenticationOptions) { + const option = authenticationOptions[key] + if (option.active) { + newAppData.selectedAuthentication = option + newAppData.authentication_id = option.id + break + } + } + } + + //newAppData.authentication = authenticationOptions + //if (newAppData.selectedAuthentication === null || newAppData.selectedAuthentication === undefined || newAppData.selectedAuthentication.length === "") { + // newAppData.selectedAuthentication = {} + //} else { + // console.log("CAN WE SELECT AUTH?: ", authenticationOptions) + //} + } else { + newAppData.authentication = [] + newAppData.authentication_id = "" + newAppData.selectedAuthentication = {} + } + + workflow.actions.push(newAppData) + setWorkflow(workflow) + + if (newAppPopup) { + //alert.error("SHOULD MAKE USER AUTHENTICATE THE APP OR SET hasError") + //alert.info("Remember: set the authentication for the user itself, not the app") + } + } + } + } + const ParsedAppPaper = (props) => { const app = props.app const [hover, setHover] = React.useState(false) @@ -6845,6 +6857,12 @@ const AngularWorkflow = (props) => { const elementName = "copy_element_shuffle" var copyText = document.getElementById(elementName); if (copyText !== null && copyText !== undefined) { + if (copy.namespace !== undefined && copy.name !== undefined && copy.src !== undefined) { + copy = copy.src + } + + console.log("NEW: ", copy) + navigator.clipboard.writeText(JSON.stringify(copy)) copyText.select() copyText.setSelectionRange(0, 99999) /* For mobile devices */ From f0065eaa11e912ebd8e70b307c163fc55a64dbe3 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 25 May 2021 21:22:42 +0200 Subject: [PATCH 56/96] Minor app drag calculation changes --- docker-compose.yml | 10 ---------- frontend/package.json | 4 ++-- frontend/src/views/AngularWorkflow.jsx | 15 ++------------- 3 files changed, 4 insertions(+), 25 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 05b999e3..a503ac8a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -100,10 +100,6 @@ services: - 9200:9200 networks: - shuffle - #volumes: - # #- ${DB_LOCATION}/opensearch:/usr/share/opensearch/data - # #- tmp/opensearch:/usr/share/opensearch/data - # - ${DB_LOCATION}:/usr/share/opensearch/data #database: # #build: ./backend/database # image: frikky/shuffle:database @@ -122,9 +118,3 @@ services: networks: shuffle: driver: bridge -#volumes: -# opensearch-data1: -# driver: local -# driver_opts: -# type: nfs -# device: ${DB_LOCATION} diff --git a/frontend/package.json b/frontend/package.json index c9372113..4bfb693d 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "shuffler", "homepage": "https://shuffler.io", - "version": "0.8.76", + "version": "0.8.92", "private": true, "dependencies": { "@babel/helper-regex": "^7.10.5", @@ -46,7 +46,7 @@ "react-cytoscapejs": "^1.2.0", "react-device-detect": "^1.9.10", "react-dom": "^16.14.0", - "react-draggable": "^3.3.2", + "react-draggable": "4.4.3", "react-dropzone": "^10.1.10", "react-ga": "^2.7.0", "react-iframe": "^1.8.0", diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index af5d7437..6e8f8050 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -3400,9 +3400,6 @@ const AngularWorkflow = (props) => { const handleAppDrag = (e, app) => { const cycontainer = cy.container() - console.log("X: ", e.pageX) - console.log("Y: ", e.pageY) - console.log("Height: ", cycontainer.offsetHeight) // Chrome lol //if (e.srcElement !== undefined && e.srcElement.localName === "canvas") { @@ -3413,8 +3410,6 @@ const AngularWorkflow = (props) => { return } - console.log("RENDEREDX: ", e.pageX-cycontainer.offsetLeft) - console.log("RENDEREDY: ", e.pageY-cycontainer.offsetTop) currentnode[0].renderedPosition("x", e.pageX-cycontainer.offsetLeft) currentnode[0].renderedPosition("y", e.pageY-cycontainer.offsetTop) } else{ @@ -3480,14 +3475,8 @@ const AngularWorkflow = (props) => { // FIXME: overwrite category if the ACTION chosen has a different category // const image = "url("+app.large_image+")" - // FIXME - find the cytoscape offset position // Can this be done with zoom calculations? - console.log("LAYERX: ", e.layerX) - console.log("LAYERY: ", e.layerY) - console.log("RENDEREDX: ", e.pageX-cycontainer.offsetLeft) - console.log("RENDEREDY: ", e.pageY-cycontainer.offsetTop) - console.log("E: ", e) const nodeToBeAdded = { group: "nodes", data: newAppData, @@ -3593,8 +3582,8 @@ const AngularWorkflow = (props) => { newAppData.selectedAuthentication = {} } - workflow.actions.push(newAppData) - setWorkflow(workflow) + //workflow.actions.push(newAppData) + //setWorkflow(workflow) if (newAppPopup) { //alert.error("SHOULD MAKE USER AUTHENTICATE THE APP OR SET hasError") From 878f796dd55f267a2bbf3df0c6602c4af47ed842 Mon Sep 17 00:00:00 2001 From: frikky Date: Wed, 26 May 2021 10:56:10 +0200 Subject: [PATCH 57/96] Fix app dragging and data mapping issues in frontend --- frontend/src/components/ParsedAction.jsx | 2 - frontend/src/views/AngularWorkflow.jsx | 600 ++++++++++++----------- frontend/src/views/Workflows.jsx | 4 +- 3 files changed, 329 insertions(+), 277 deletions(-) diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index b8abf9fd..be5588cc 100644 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -1485,8 +1485,6 @@ const ParsedAction = (props) => { const iconInfo = GetIconInfo({"name": data.name}) const useIcon = iconInfo.originalIcon - - // ROFL FIXME - loop newActionname = (newActionname.charAt(0).toUpperCase()+newActionname.substring(1)).replaceAll("_", " ") return ( diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 6e8f8050..276dc752 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1141,7 +1141,6 @@ const AngularWorkflow = (props) => { // Remove the old listener for select, run with new one cy.removeListener('select') - cy.on('select', 'node', (e) => onNodeSelect(e, newauth)) cy.on('select', 'edge', (e) => onEdgeSelect(e)) } @@ -1355,11 +1354,18 @@ const AngularWorkflow = (props) => { var hiddenNodes = [] const onNodeDragStop = (event, selectedAction) => { const nodedata = event.target.data() - + console.log("IN NODE DRAG STOP: ", nodedata) if (nodedata.id === selectedAction.id) { return } + if (nodedata.finished === false) { + return + } + + //console.log("Drag: ", nodedata) + //return + //console.log("DRAGGED NODE: ", nodedata) //console.log("TARGET NODE: ", selectedAction) if (styledElements.length === 1) { @@ -1462,8 +1468,12 @@ const AngularWorkflow = (props) => { //if (Object.getOwnPropertyNames(selectedAction).length === 0) { // return //} - const nodedata = event.target.data() + //console.log("Dragging: ", nodedata) + if (nodedata.finished === false) { + return + } + //console.log("Dragging node!!") if (nodedata.app_name == "Shuffle Tools" || nodedata.app_name == "Testing") { //console.log("NODE: ", @@ -1569,8 +1579,6 @@ const AngularWorkflow = (props) => { // https://stackoverflow.com/questions/16677856/cy-onselect-callback-only-once const onNodeSelect = (event, newAppAuth) => { const data = event.target.data() - - setLastSaved(false) if (data.isButton) { console.log("BUTTON CLICKED: ", data) if (data.buttonType === "delete") { @@ -1697,7 +1705,6 @@ const AngularWorkflow = (props) => { return } - console.log("NODE: ", data) //const node = cy.getElementById(data.id) //if (node.length > 0) { // node.addClass('shuffle-hover-highlight') @@ -1709,6 +1716,7 @@ const AngularWorkflow = (props) => { if (data.type === "ACTION") { var curaction = workflow.actions.find(a => a.id === data.id) + console.log("INSIDE CURACTION: ", curaction) if (!curaction || curaction === undefined) { //event.target.unselect() //alert.error("Action not found. Please remake it.") @@ -1781,7 +1789,11 @@ const AngularWorkflow = (props) => { } //setSelectedAction(JSON.parse(JSON.stringify(curaction))) - setSelectedApp(curapp) + console.log("CURAPP: ", curapp, selectedApp) + if (curapp.id !== selectedApp.id) { + setSelectedApp(curapp) + } + setSelectedAction(curaction) cy.removeListener('drag') @@ -1799,7 +1811,6 @@ const AngularWorkflow = (props) => { setSelectedActionEnvironment(env) } - setRightSideBarOpen(true) } else if (data.type === "TRIGGER") { //console.log("Should handle trigger "+data.triggertype) //console.log(data) @@ -1812,11 +1823,19 @@ const AngularWorkflow = (props) => { getAvailableWorkflows(trigger_index) getSettings() } - setRightSideBarOpen(true) } else { alert.error("Can't handle "+data.type) } + //console.log("BAR: ", rightSideBarOpen, "SAVE: ", lastSaved) + setRightSideBarOpen(true) + setLastSaved(false) + + // Refresh listeners + //cy.removeListener('select') + //cy.on('select', 'node', (e) => onNodeSelect(e, newAppAuth, curapp, rightSideBarOpen, lastSaved)) + //cy.on('select', 'edge', (e) => onEdgeSelect(e)) + setScrollConfig({ top: 0, left: 0, @@ -1840,28 +1859,38 @@ const AngularWorkflow = (props) => { targetnode = -1 var sourcenode = workflow.triggers.findIndex(data => data.id === edge.source) - //console.log("SOURCENODE: ", sourcenode) + console.log("SOURCENODE: ", sourcenode) if (sourcenode !== -1) { if (workflow.triggers[sourcenode].app_name === "User Input" || workflow.triggers[sourcenode].app_name === "Shuffle Workflow") { //console.log("NORMAL TRIGGER") } else { - var currentnode = cy.getElementById(workflow.triggers[sourcenode].id) - if (currentnode !== null && currentnode !== undefined) { - console.log("SHOULD CHECK IF TRIGGER HAS MULTIPLE EDGES: ", currentnode) - // https://js.cytoscape.org/#edges.connectedNodes - //console.log("CURRENTNODE: ", currentnode) - //console.log("EDGES: ", currentnode.connectedEdges(`node[id=${workflow.triggers[sourcenode].id}]`)) - //console.log("EDGES2: ", currentnode.connectedEdges()) - //currentnode.connectedEdges().animate({style: {lineColor: "red"}}) - //console.log("OUTGOERS: ", currentnode.outgoers()) + //var currentnode = cy.getElementById(workflow.triggers[sourcenode].id) + //console.log("NODE: ", currentnode) + //if (currentnode !== null && currentnode !== undefined) { + // console.log("SHOULD CHECK IF TRIGGER HAS MULTIPLE EDGES: ", currentnode) + //if (workflow.branches !== undefined && workflow.branches !== null) { + // const found_branches = workflow.branches.filter(branch => branch.source == workflow.triggers[sourcenode].id) + // console.log("FOUND BRANCHES: ", found_branches) + // if (found_branches.length > 0) { + // alert.error("Can't have multiple branches from this trigger") + // event.target.remove() + // } + //} - //console.log("LEN2: ", currentnode.edges().length) - //if (currentnode.connectedNodes().length > 0) { - // alert.error("Can't have multiple branches from this trigger") - // event.target.remove() - //} - } - } + //if (cy.edges().size() === 1) { + // https://js.cytoscape.org/#edges.connectedNodes + //console.log("CURRENTNODE: ", currentnode) + //console.log("EDGES: ", currentnode.connectedEdges(`node[id=${workflow.triggers[sourcenode].id}]`)) + //console.log("EDGES2: ", currentnode.connectedEdges()) + //currentnode.connectedEdges().animate({style: {lineColor: "red"}}) + //console.log("OUTGOERS: ", currentnode.outgoers()) + + //console.log("LEN2: ", currentnode.edges().length) + //if (currentnode.connectedNodes().length > 0) { + // alert.error("Can't have multiple branches from this trigger") + // event.target.remove() + //} + } } //console.log(workflow.branches) @@ -1953,14 +1982,18 @@ const AngularWorkflow = (props) => { } const onNodeAdded = (event) => { - setLastSaved(false) const node = event.target const nodedata = event.target.data() + if (nodedata.finished === false || (nodedata.id !== undefined && nodedata.is_valid === undefined)) { + console.log("RETURNING (NOT ADDING) NODE ADD FOR: ", nodedata) + return + } + //console.log("IS IT ADDED TO THE WORKFLOW?: ", nodedata) if (node.isNode() && cy.nodes().size() === 1) { //setStartNode(node.data('id')) workflow.start = node.data('id') - setWorkflow(workflow) + nodedata.isStartNode = true } else { if (workflow.actions === null) { return @@ -1974,10 +2007,75 @@ const AngularWorkflow = (props) => { action.isStartNode = false } } + } + + if (nodedata.type === "ACTION") { + if (workflow.actions.length === 1 && workflow.actions[0].id === workflow.start) { + const newEdgeUuid = uuid.v4() + const newcybranch = { + "source": workflow.start, + "target": nodedata.id, + "_id": newEdgeUuid, + "id": newEdgeUuid, + "hasErrors": false, + } + + const edgeToBeAdded = { + group: "edges", + data: newcybranch, + } + + console.log("SHOULD STITCH WITH STARTNODE") + cy.add(edgeToBeAdded) + } + + if (workflow.actions === undefined || workflow.actions === null) { + workflow.actions = [nodedata] + } else { + workflow.actions.push(nodedata) + } + + setWorkflow(workflow) + } else if (nodedata.type === "TRIGGER") { + if (workflow.triggers === undefined) { + workflow.triggers = [nodedata] + } else { + workflow.triggers.push(nodedata) + } + + const newEdgeUuid = uuid.v4() + const newcybranch = { + "source": nodedata.id, + "target": workflow.start, + "source_id": nodedata.id, + "destination_id": workflow.start, + "_id": newEdgeUuid, + "id": newEdgeUuid, + "hasErrors": false, + "decorator": false, + } + + const edgeToBeAdded = { + group: "edges", + data: newcybranch, + } + + if (nodedata.name !== "User Input" && nodedata.name !== "Shuffle Workflow") { + //workflow.branches.push(newbranch) + if (workflow.actions !== undefined && workflow.actions !== null && workflow.actions.length > 0) { + cy.add(edgeToBeAdded) + } + } + + //if (data.trigger_type === "WEBHOOK") { + // newWebhook(newAppData) + // saveWorkflow(workflow) + //} setWorkflow(workflow) } + if (nodedata.app_name !== undefined) { history.push({ "type": "node", @@ -2076,7 +2174,12 @@ const AngularWorkflow = (props) => { const onNodeRemoved = (event) => { const node = event.target const data = node.data() - setLastSaved(false) + + if (data.finished === false) { + return + } + + //setLastSaved(false) workflow.actions = workflow.actions.filter(a => a.id !== data.id) workflow.triggers = workflow.triggers.filter(a => a.id !== data.id) @@ -2366,8 +2469,6 @@ const AngularWorkflow = (props) => { //cy.on('cxtdragover', 'node', (e) => edgeHandler.preview(e.target)) //cy.on('cxtdragout', 'node', (e) => edgeHandler.unpreview(e.target)) - // RIGHT HERE..? - // This is wrong sometimes.. I'm mad document.title = "Workflow - "+workflow.name registerKeys() //setStartNode(workflow.start) @@ -2493,13 +2594,12 @@ const AngularWorkflow = (props) => { const onNodeHover = (event) => { //console.log("TAR: ", event.target) - + const nodedata = event.target.data() var parentNode = cy.$('#' + event.target.data("id")); if (parentNode.data('isButton') || parentNode.data('buttonId')) return - const nodedata = event.target.data() - if (event.target.data().app_name !== undefined) { + if (nodedata.app_name !== undefined) { const allNodes = cy.nodes().jsons() var found = false @@ -3239,7 +3339,9 @@ const AngularWorkflow = (props) => { {handleTriggerDrag(e, trigger)}} - onStop={(e) => {handleDragStop(e)}} + onStop={(e) => { + handleDragStop(e) + }} dragging={false} position={{ x: 0, @@ -3274,6 +3376,7 @@ const AngularWorkflow = (props) => { } var newNodeId = "" + var parsedApp = {} const handleTriggerDrag = (e, data) => { const cycontainer = cy.container() // Chrome lol @@ -3288,7 +3391,6 @@ const AngularWorkflow = (props) => { currentnode[0].renderedPosition("x", e.pageX-cycontainer.offsetLeft) currentnode[0].renderedPosition("y", e.pageY-cycontainer.offsetTop) } else{ - console.log(workflow) if (workflow.start === "" || workflow.start === undefined) { alert.error("Define a starting action first.") return @@ -3307,7 +3409,6 @@ const AngularWorkflow = (props) => { "y": e.pageY-cycontainer.offsetTop, } - console.log(data) const newAppData = { app_name: data.name, app_version: "1.0.0", @@ -3318,6 +3419,7 @@ const AngularWorkflow = (props) => { id_: newNodeId, _id_: newNodeId, id: newNodeId, + finished: false, label: triggerLabel, type: data.type, is_valid: true, @@ -3341,48 +3443,97 @@ const AngularWorkflow = (props) => { } cy.add(nodeToBeAdded) - - if (workflow.triggers === undefined) { - workflow.triggers = [newAppData] - } else { - workflow.triggers.push(newAppData) - } - - const newEdgeUuid = uuid.v4() - const newcybranch = { - "source": newNodeId, - "target": workflow.start, - "source_id": newNodeId, - "destination_id": workflow.start, - "_id": newEdgeUuid, - "id": newEdgeUuid, - "hasErrors": false, - "decorator": false, - } - - const edgeToBeAdded = { - group: "edges", - data: newcybranch, - } - - if (data.name !== "User Input" && data.name !== "Shuffle Workflow") { - //workflow.branches.push(newbranch) - cy.add(edgeToBeAdded) - } - - setWorkflow(workflow) - //if (data.trigger_type === "WEBHOOK") { - // newWebhook(newAppData) - // saveWorkflow(workflow) - //} + parsedApp = nodeToBeAdded + return } } } const handleDragStop = (e, app) => { - newNodeId = "" console.log("STOP!: ", e) - console.log("APP!: ", app) + console.log("APP!: ", parsedApp) + //const onNodeAdded = (event) => { + //const node = event.target + //const nodedata = event.target.data() + var currentnode = cy.getElementById(newNodeId) + if (currentnode === undefined || currentnode === null || currentnode.length === 0) { + return + } + + // Using remove & replace, as this triggers the function + // onNodeAdded() with this node after it's added + + currentnode.remove() + parsedApp.data.finished = true + parsedApp.data.position = currentnode.renderedPosition() + parsedApp.position = currentnode.renderedPosition() + parsedApp.renderedPosition = currentnode.renderedPosition() + + var newAppData = parsedApp.data + if (newAppData.type === "ACTION") { + // AUTHENTICATION + if (app.authentication.required) { + // Setup auth here :) + const authenticationOptions = [] + var findAuthId = "" + if (newAppData.authentication_id !== null && newAppData.authentication_id !== undefined && newAppData.authentication_id.length > 0) { + findAuthId = newAppData.authentication_id + } + + var tmpAuth = JSON.parse(JSON.stringify(appAuthentication)) + for (var key in tmpAuth) { + var item = tmpAuth[key] + + const newfields = {} + for (var filterkey in item.fields) { + newfields[item.fields[filterkey].key] = item.fields[filterkey].value + } + + item.fields = newfields + if (item.app.name === app.name) { + authenticationOptions.push(item) + if (item.id === findAuthId) { + newAppData.selectedAuthentication = item + } + } + } + + if (authenticationOptions !== undefined && authenticationOptions !== null && authenticationOptions.length > 0) { + for (var key in authenticationOptions) { + const option = authenticationOptions[key] + if (option.active) { + newAppData.selectedAuthentication = option + newAppData.authentication_id = option.id + break + } + } + } + + //newAppData.authentication = authenticationOptions + //if (newAppData.selectedAuthentication === null || newAppData.selectedAuthentication === undefined || newAppData.selectedAuthentication.length === "") { + // newAppData.selectedAuthentication = {} + //} else { + // console.log("CAN WE SELECT AUTH?: ", authenticationOptions) + //} + // + // + + //console.log(parsedApp) + } else { + newAppData.authentication = [] + newAppData.authentication_id = "" + newAppData.selectedAuthentication = {} + } + + parsedApp.data = newAppData + cy.add(parsedApp) + } else if (newAppData.type === "TRIGGER") { + cy.add(parsedApp) + + } + + newNodeId = "" + parsedApp = {} } const appScrollStyle = { @@ -3394,205 +3545,103 @@ const AngularWorkflow = (props) => { overflowX: "hidden", } - const AppView = (props) => { - const { allApps, prioritizedApps, filteredApps } = props; - const [visibleApps, setVisibleApps] = React.useState(prioritizedApps.concat(filteredApps.filter(innerapp => !internalIds.includes(innerapp.id)))) + const handleAppDrag = (e, app) => { + const cycontainer = cy.container() - const handleAppDrag = (e, app) => { - const cycontainer = cy.container() + // Chrome lol + //if (e.srcElement !== undefined && e.srcElement.localName === "canvas") { + if (e.pageX > cycontainer.offsetLeft && e.pageX < cycontainer.offsetLeft+cycontainer.offsetWidth && e.pageY > cycontainer.offsetTop && e.pageY < cycontainer.offsetTop+cycontainer.offsetHeight) { + console.log("NODEID: ", newNodeId) + if (newNodeId.length > 0) { + var currentnode = cy.getElementById(newNodeId) + if (currentnode === undefined || currentnode === null || currentnode.length === 0) { + return + } - // Chrome lol - //if (e.srcElement !== undefined && e.srcElement.localName === "canvas") { - if (e.pageX > cycontainer.offsetLeft && e.pageX < cycontainer.offsetLeft+cycontainer.offsetWidth && e.pageY > cycontainer.offsetTop && e.pageY < cycontainer.offsetTop+cycontainer.offsetHeight) { - if (newNodeId.length > 0) { - var currentnode = cy.getElementById(newNodeId) - if (currentnode.length === 0) { - return - } + currentnode[0].renderedPosition("x", e.pageX-cycontainer.offsetLeft) + currentnode[0].renderedPosition("y", e.pageY-cycontainer.offsetTop) + } else { + console.log("IN NEW NODE!") + if (workflow.public) { + console.log("workflow is public - not adding") + return + } - currentnode[0].renderedPosition("x", e.pageX-cycontainer.offsetLeft) - currentnode[0].renderedPosition("y", e.pageY-cycontainer.offsetTop) - } else{ - if (workflow.public) { - return - } + console.log("IN NEW NODE2!") + if (app.actions === undefined || app.actions === null || app.actions.length === 0) { + alert.error("App "+app.name+" currently has no actions to perform. Please go to https://shuffler.io/apps to edit it.") + return + } - if (app.actions === undefined || app.actions === null || app.actions.length === 0) { - alert.error("App "+app.name+" currently has no actions to perform. Please go to https://shuffler.io/apps to edit it.") - return - } + console.log("IN NEW NODE3!") + newNodeId = uuid.v4() + const actionType = "ACTION" + const actionLabel = getNextActionName(app.name) + var parameters = null + var example = "" - newNodeId = uuid.v4() - const actionType = "ACTION" - const actionLabel = getNextActionName(app.name) - var parameters = null - var example = "" + if (app.actions[0].parameters !== null && app.actions[0].parameters.length > 0) { + parameters = app.actions[0].parameters + } + if (app.actions[0].returns.example !== undefined && app.actions[0].returns.example !== null && app.actions[0].returns.example.length > 0) { + example = app.actions[0].returns.example + } + + const parsedEnvironments = environments === null || environments === [] ? "cloud" : environments[defaultEnvironmentIndex] === undefined ? "cloud" : environments[defaultEnvironmentIndex].Name + const newAppData = { + app_name: app.name, + app_version: app.app_version, + app_id: app.id, + sharing: app.sharing, + private_id: app.private_id, + environment: parsedEnvironments, + errors: [], + id_: newNodeId, + _id_: newNodeId, + id: newNodeId, + is_valid: true, + label: actionLabel, + type: actionType, + name: app.actions[0].name, + parameters: parameters, + isStartNode: false, + large_image: app.large_image, + authentication: [], + execution_variable: undefined, + example: example, + category: app.categories !== null && app.categories !== undefined && app.categories.length > 0 ? app.categories[0] : "", + authentication_id: "", + finished: false, + } - if (app.actions[0].parameters !== null && app.actions[0].parameters.length > 0) { - parameters = app.actions[0].parameters - } - if (app.actions[0].returns.example !== undefined && app.actions[0].returns.example !== null && app.actions[0].returns.example.length > 0) { - example = app.actions[0].returns.example - } - - var newAppPopup = false + // FIXME: overwrite category if the ACTION chosen has a different category - /* - FIXME: Add auth. - selectedAction.selectedAuthentication = e.target.value - selectedAction.authentication_id = e.target.value.id - setSelectedAction(selectedAction) - setUpdate(Math.random()) - */ - - console.log("ENVS: ", environments) - const parsedEnvironments = environments === null || environments === [] ? "cloud" : environments[defaultEnvironmentIndex] === undefined ? "cloud" : environments[defaultEnvironmentIndex].Name - const newAppData = { - app_name: app.name, - app_version: app.app_version, - app_id: app.id, - sharing: app.sharing, - private_id: app.private_id, - environment: parsedEnvironments, - errors: [], - id_: newNodeId, - _id_: newNodeId, - id: newNodeId, - is_valid: true, - label: actionLabel, - type: actionType, - name: app.actions[0].name, - parameters: parameters, - isStartNode: false, - large_image: app.large_image, - authentication: [], - execution_variable: undefined, - example: example, - category: app.categories !== null && app.categories !== undefined && app.categories.length > 0 ? app.categories[0] : "", - authentication_id: "", - } - - // FIXME: overwrite category if the ACTION chosen has a different category - - // const image = "url("+app.large_image+")" - // FIXME - find the cytoscape offset position - // Can this be done with zoom calculations? - const nodeToBeAdded = { - group: "nodes", - data: newAppData, - renderedPosition: { - //x: e.layerX, - //y: e.layerY, - x: e.pageX-cycontainer.offsetLeft, - y: e.pageY-cycontainer.offsetTop, - } - } - - cy.add(nodeToBeAdded) - - if (workflow.actions === undefined || workflow.actions.length === 0) { - workflow.start = newAppData.id - workflow.actions = [] - newAppData.isStartNode = true - //setStartNode(newAppData.id) - } - - if (workflow.actions.length > 0 && elements.length === 0) { - const actions = workflow.actions.map(action => { - const node = {} - node.position = action.position - node.data = action - - node.data._id = action["id"] - node.data.type = "ACTION" - node.isStartNode = action["id"] === workflow.start - - return node - }) - - const tmpelements = [].concat(actions) - setElements(tmpelements) - } - - if (workflow.actions.length === 1 && workflow.actions[0].id === workflow.start) { - const newEdgeUuid = uuid.v4() - const newcybranch = { - "source": workflow.start, - "target": newNodeId, - "_id": newEdgeUuid, - "id": newEdgeUuid, - "hasErrors": false, - } - - const edgeToBeAdded = { - group: "edges", - data: newcybranch, - } - console.log("SHOULD STITCH WITH STARTNODE") - cy.add(edgeToBeAdded) - } - - // AUTHENTICATION - if (app.authentication.required) { - // Setup auth here :) - const authenticationOptions = [] - var findAuthId = "" - if (newAppData.authentication_id !== null && newAppData.authentication_id !== undefined && newAppData.authentication_id.length > 0) { - findAuthId = newAppData.authentication_id - } - - var tmpAuth = JSON.parse(JSON.stringify(appAuthentication)) - for (var key in tmpAuth) { - var item = tmpAuth[key] - - const newfields = {} - for (var filterkey in item.fields) { - newfields[item.fields[filterkey].key] = item.fields[filterkey].value - } - - item.fields = newfields - if (item.app.name === app.name) { - authenticationOptions.push(item) - if (item.id === findAuthId) { - newAppData.selectedAuthentication = item - } - } - } - - if (authenticationOptions !== undefined && authenticationOptions !== null && authenticationOptions.length > 0) { - for (var key in authenticationOptions) { - const option = authenticationOptions[key] - if (option.active) { - newAppData.selectedAuthentication = option - newAppData.authentication_id = option.id - break - } - } - } - - //newAppData.authentication = authenticationOptions - //if (newAppData.selectedAuthentication === null || newAppData.selectedAuthentication === undefined || newAppData.selectedAuthentication.length === "") { - // newAppData.selectedAuthentication = {} - //} else { - // console.log("CAN WE SELECT AUTH?: ", authenticationOptions) - //} - } else { - newAppData.authentication = [] - newAppData.authentication_id = "" - newAppData.selectedAuthentication = {} - } - - //workflow.actions.push(newAppData) - //setWorkflow(workflow) - - if (newAppPopup) { - //alert.error("SHOULD MAKE USER AUTHENTICATE THE APP OR SET hasError") - //alert.info("Remember: set the authentication for the user itself, not the app") - } + // const image = "url("+app.large_image+")" + // FIXME - find the cytoscape offset position + // Can this be done with zoom calculations? + const nodeToBeAdded = { + group: "nodes", + data: newAppData, + renderedPosition: { + //x: e.layerX, + //y: e.layerY, + x: e.pageX-cycontainer.offsetLeft, + y: e.pageY-cycontainer.offsetTop, } } + + console.log("IN NEW NODE4: !", nodeToBeAdded) + parsedApp = nodeToBeAdded + cy.add(nodeToBeAdded) + return + } + } } + const AppView = (props) => { + const { allApps, prioritizedApps, filteredApps } = props; + const [visibleApps, setVisibleApps] = React.useState(prioritizedApps.concat(filteredApps.filter(innerapp => !internalIds.includes(innerapp.id)))) + const ParsedAppPaper = (props) => { const app = props.app const [hover, setHover] = React.useState(false) @@ -3614,7 +3663,9 @@ const AngularWorkflow = (props) => { return ( {handleAppDrag(e, app)}} - onStop={(e) => {handleDragStop(e, app)}} + onStop={(e) => { + handleDragStop(e, app) + }} key={app.id} dragging={false} position={{ @@ -4813,7 +4864,7 @@ const AngularWorkflow = (props) => { native rows="10" value={selectedTrigger.parameters[0].value.split(splitter)} - style={{backgroundColor: inputColor, color: "white"}} + style={{backgroundColor: inputColor, color: "white", height: 50,}} disabled={selectedTrigger.status === "running"} SelectDisplayProps={{ style: { @@ -5343,11 +5394,12 @@ const AngularWorkflow = (props) => { onChange={(e) => { setUpdate(Math.random()) workflow.triggers[selectedTriggerIndex].parameters[0].value = e.target.value.id - setWorkflow(workflow) setSubworkflowStartnode(e.target.value.start) // Sets the startnode if (e.target.value.id !== workflow.id) { + console.log("WORKFLOW: ", e.target.value) + setSubworkflow(e.target.value) const startnode = e.target.value.actions.find(action => action.id === e.target.value.start) if (startnode !== undefined && startnode !== null) { @@ -5365,17 +5417,19 @@ const AngularWorkflow = (props) => { } console.log("STARTNODE: ", startnode) } else { - setSubworkflow(workflow) + console.log("WORKFLOW: ", workflow) + setSubworkflow(e.target.value) } + + setWorkflow(workflow) }} - style={{backgroundColor: inputColor, color: "white", height: "50px"}} + style={{backgroundColor: inputColor, color: "white", height: 50}} > {workflows.map((data, index) => { - /* if (data.id === workflow.id) { - return null + //return null + data = workflow } - */ return ( @@ -5467,7 +5521,7 @@ const AngularWorkflow = (props) => { setWorkflow(workflow) //setUpdate(Math.random()) }} - style={{backgroundColor: inputColor, color: "white", height: "50px"}} + style={{backgroundColor: inputColor, color: "white", height: 50}} > {subworkflow.actions.map((action, index) => { //console.log(action) @@ -5635,7 +5689,7 @@ const AngularWorkflow = (props) => { setWorkflow(workflow) setUpdate(Math.random()) }} - style={{backgroundColor: inputColor, color: "white", height: "50px"}} + style={{backgroundColor: inputColor, color: "white", height: 50}} > {triggerEnvironments.map(data => { if (data.archived) { @@ -6200,7 +6254,7 @@ const AngularWorkflow = (props) => { setWorkflow(workflow) setUpdate(Math.random()) }} - style={{backgroundColor: inputColor, color: "white", height: "50px"}} + style={{backgroundColor: inputColor, color: "white", height: 50}} > {triggerEnvironments.map(data => { if (data.archived) { @@ -7875,7 +7929,7 @@ const AngularWorkflow = (props) => { onChange={(e) => { authenticationOption.fields[data.name] = e.target.value }} - style={{backgroundColor: theme.palette.surfaceColor, color: "white", height: "50px"}} + style={{backgroundColor: theme.palette.surfaceColor, color: "white", height: 50}} > false diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 34545944..18945796 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -208,7 +208,7 @@ const counterStyle = {fontSize: "36px",fontWeight:"bold"} const blockRightStyle = {textAlign: "right",padding: "20px 20px 0px 0px",width:"100%"} const chipStyle = { - backgroundColor: "#3d3f43", height: 30, marginRight: 5, paddingLeft: 5, paddingRight: 5, height: 28, cursor: "pointer", borderColor: "#3d3f43", color: "white", + backgroundColor: "#3d3f43", height: 30, marginRight: 5, paddingLeft: 5, paddingRight: 5, height: 28, cursor: "pointer", borderColor: "#3d3f43", color: "white", } const flexContentStyle = { @@ -1556,7 +1556,7 @@ const Workflows = (props) => { ) } }, - { field: 'tags', headerName: 'Tags', width: 390, sortable: false, + { field: 'tags', headerName: 'Tags', maxHeight: 15, width: 390, sortable: false, disableClickEventBubbling: true, renderCell: (params) => { const data = params.row.record; From cfab53012e9e3b6b822a301c39be2d9aa1dbbdff Mon Sep 17 00:00:00 2001 From: frikky Date: Wed, 26 May 2021 11:40:21 +0200 Subject: [PATCH 58/96] Updated compose file --- docker-compose.yml | 4 +-- frontend/package.json | 1 + frontend/src/views/AngularWorkflow.jsx | 45 ++++++++++++++++++++++++-- frontend/src/views/AppCreator.jsx | 9 ++++-- 4 files changed, 53 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index a503ac8a..d20a6d8c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: frontend: #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.92 + image: ghcr.io/frikky/shuffle-frontend:0.8.93 container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -17,7 +17,7 @@ services: - backend backend: #build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.92 + image: ghcr.io/frikky/shuffle-backend:0.8.93 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: diff --git a/frontend/package.json b/frontend/package.json index 4bfb693d..6295f47f 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -22,6 +22,7 @@ "cytoscape-grid-guide": "~2.1.2", "cytoscape-node-html-label": "^1.1.5", "cytoscape-panzoom": "^2.5.2", + "cytoscape-popper": "^2.0.0", "cytoscape-undo-redo": "^1.3.2", "d3": "~4.10.0", "dotenv": "^6.1.0", diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 276dc752..121ee0f1 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -39,7 +39,12 @@ const inputColor = "#383B40" cytoscape.use(edgehandles); cytoscape.use(clipboard); cytoscape.use(undoRedo); -cytoscape.use( cxtmenu ); +cytoscape.use(cxtmenu); + +// Adds specific text to items +//import popper from 'cytoscape-popper'; +//cytoscape.use(popper); + // https://stackoverflow.com/questions/19014250/rerender-view-on-browser-resize-with-react function useWindowSize() { @@ -2462,6 +2467,42 @@ const AngularWorkflow = (props) => { cy.on('drag', 'node', (e) => onNodeDrag(e, selectedAction)) cy.on('free', 'node', (e) => onNodeDragStop(e, selectedAction)) + //let popper2 = cy.popper({ + // content: () => { + // let div = document.createElement('div'); + // + // div.innerHTML = 'Popper content' + // + // document.body.appendChild(div) + // + // return div + // }, + // renderedPosition: () => ({ x: 100, y: 200 }), + // popper: {} // my popper options here + //}); + + //let popper1 = cy.nodes()[0].popper({ + // content: () => { + // let div = document.createElement('div') + // + // div.innerHTML = 'Popper content' + // document.body.appendChild(div) + // + // return div + // }, + // popper: {} // my popper options here + //}) + //let update = () => { + // popper.update() + //} + + //let node = cy.nodes().first() + //node.on('position', update) + + + + + //cy.on('mouseover', 'node', () => $(targetElement).addClass('mouseover')); //cy.on('cxttapstart', 'node', (e) => edgeHandler.start(e.target)) @@ -3397,7 +3438,7 @@ const AngularWorkflow = (props) => { } if (data.is_valid === false) { - alert.error(data.name+" trigger isn't available yet") + alert.error(data.name+" requires hybrid version of Shuffle") return } diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 0f0981d0..abd711ef 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -198,7 +198,7 @@ const AppCreator = (props) => { const actionNonBodyRequest = ["GET", "HEAD", "DELETE", "CONNECT"] const actionBodyRequest = ["POST", "PUT", "PATCH",] //const authenticationOptions = ["No authentication", "API key", "Bearer auth", "Basic auth", "Oauth2"] - const authenticationOptions = ["No authentication", "API key", "Bearer auth", "Basic auth"] + const authenticationOptions = ["No authentication", "API key", "Bearer auth", "Basic auth", "JWT"] const apikeySelection = ["Header", "Query",] const [name, setName] = useState(""); @@ -1263,7 +1263,12 @@ const AppCreator = (props) => { "scheme": "bearer", "bearerFormat": "UUID", } - + } else if (authenticationOption === "JWT") { + data.components.securitySchemes["BearerAuth"] = { + "type": "http", + "scheme": "bearer", + "bearerFormat": "JWT", + } } else if (authenticationOption === "Basic auth") { data.components.securitySchemes["BasicAuth"] = { "type": "http", From 06b74cf4da0ea180914b13db263db0c4df662ab6 Mon Sep 17 00:00:00 2001 From: frikky Date: Wed, 26 May 2021 12:39:07 +0200 Subject: [PATCH 59/96] Added loading to login and admin user creation --- docker-compose.yml | 1 + frontend/src/views/AdminSetup.jsx | 14 +++++--- frontend/src/views/AngularWorkflow.jsx | 17 ++++------ frontend/src/views/Apps.jsx | 46 +++++++++++++++----------- frontend/src/views/LoginPage.jsx | 15 +++++---- 5 files changed, 52 insertions(+), 41 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index d20a6d8c..9fcdbcf5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -100,6 +100,7 @@ services: - 9200:9200 networks: - shuffle + # OLD DATABASE: #database: # #build: ./backend/database # image: frikky/shuffle:database diff --git a/frontend/src/views/AdminSetup.jsx b/frontend/src/views/AdminSetup.jsx index 275cd5b3..7f0e3a9a 100644 --- a/frontend/src/views/AdminSetup.jsx +++ b/frontend/src/views/AdminSetup.jsx @@ -2,9 +2,7 @@ import React, {useState} from 'react'; import { makeStyles } from '@material-ui/styles'; -import TextField from '@material-ui/core/TextField'; -import Button from '@material-ui/core/Button'; -import Paper from '@material-ui/core/Paper'; +import {CircularProgress, TextField, Button, Paper, Typography} from '@material-ui/core' const bodyDivStyle = { margin: "auto", @@ -35,6 +33,7 @@ const AdminAccount = props => { const [username, setUsername] = useState(""); const [password, setPassword] = useState(""); const [firstRequest, setFirstRequest] = useState(true); + const [loginLoading, setLoginLoading] = useState(false); // Used to swap from login to register. True = login, false = register const register = true @@ -81,6 +80,7 @@ const AdminAccount = props => { } const onSubmit = (e) => { + setLoginLoading(true) e.preventDefault() // FIXME - add some check here ROFL @@ -97,6 +97,7 @@ const AdminAccount = props => { }) .then(response => response.json().then(responseJson => { + setLoginLoading(false) if (responseJson["success"] === false) { setLoginInfo(responseJson["reason"]) } else { @@ -106,6 +107,7 @@ const AdminAccount = props => { }), ) .catch(error => { + setLoginLoading(false) setLoginInfo("Error in userdata: ", error) }); } @@ -161,7 +163,7 @@ const AdminAccount = props => { id="emailfield" margin="normal" variant="outlined" - onChange={onChangeUser} + onChange={onChangeUser} />
Password @@ -191,7 +193,9 @@ const AdminAccount = props => { />
- +
diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 121ee0f1..f79df0e0 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1721,7 +1721,7 @@ const AngularWorkflow = (props) => { if (data.type === "ACTION") { var curaction = workflow.actions.find(a => a.id === data.id) - console.log("INSIDE CURACTION: ", curaction) + //console.log("INSIDE CURACTION: ", curaction) if (!curaction || curaction === undefined) { //event.target.unselect() //alert.error("Action not found. Please remake it.") @@ -1729,7 +1729,7 @@ const AngularWorkflow = (props) => { } const curapp = apps.find(a => a.name === curaction.app_name && ((a.app_version === curaction.app_version || (a.loop_versions !== null && a.loop_versions.includes(curaction.app_version))))) - console.log("APP: ", curapp) + //console.log("APP: ", curapp) if (!curapp || curapp === undefined) { alert.error(`App ${curaction.app_name}:${curaction.app_version} not found. Is it activated?`) @@ -1752,7 +1752,7 @@ const AngularWorkflow = (props) => { //console.log("AUTHENTICATION: ", curapp.authentication) setRequiresAuthentication(curapp.authentication.required && curapp.authentication.parameters !== undefined && curapp.authentication.parameters !== null) if (curapp.authentication.required) { - console.log("App requires auth.") + //console.log("App requires auth.") // Setup auth here :) const authenticationOptions = [] var findAuthId = "" @@ -1761,7 +1761,7 @@ const AngularWorkflow = (props) => { } var tmpAuth = JSON.parse(JSON.stringify(newAppAuth)) - console.log("FOUND AUTH: ", tmpAuth) + //console.log("FOUND AUTH: ", tmpAuth) //console.log("Checking authentication: ", tmpAuth) for (var key in tmpAuth) { @@ -1781,7 +1781,7 @@ const AngularWorkflow = (props) => { } } - console.log("OPTIONS: ", authenticationOptions) + //console.log("OPTIONS: ", authenticationOptions) curaction.authentication = authenticationOptions //console.log("Authentication: ", authenticationOptions) if (curaction.selectedAuthentication === null || curaction.selectedAuthentication === undefined || curaction.selectedAuthentication.length === "") { @@ -1794,11 +1794,8 @@ const AngularWorkflow = (props) => { } //setSelectedAction(JSON.parse(JSON.stringify(curaction))) - console.log("CURAPP: ", curapp, selectedApp) - if (curapp.id !== selectedApp.id) { - setSelectedApp(curapp) - } - + //console.log("CURAPP: ", curapp, selectedApp) + setSelectedApp(curapp) setSelectedAction(curaction) cy.removeListener('drag') diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 55bcc297..5df39bbc 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -243,6 +243,9 @@ const Apps = (props) => { privateapps.push(...invalid) setApps(privateapps) + + //handleSearchChange(event.target.value) + //setCursearch(event.target.value) setFilteredApps(privateapps) if (privateapps.length > 0) { if (selectedApp.id === undefined || selectedApp.id === null) { @@ -995,26 +998,29 @@ const Apps = (props) => { }
- { - handleSearchChange(event.target.value) - setCursearch(event.target.value) - }} - /> +
+ { + handleSearchChange(event.target.value) + setCursearch(event.target.value) + }} + /> +
{apps.length > 0 ? filteredApps.length > 0 ? diff --git a/frontend/src/views/LoginPage.jsx b/frontend/src/views/LoginPage.jsx index af60a11d..9f128551 100644 --- a/frontend/src/views/LoginPage.jsx +++ b/frontend/src/views/LoginPage.jsx @@ -2,9 +2,7 @@ import React, { useState } from 'react'; import { makeStyles } from '@material-ui/styles'; -import TextField from '@material-ui/core/TextField'; -import Button from '@material-ui/core/Button'; -import Paper from '@material-ui/core/Paper'; +import {CircularProgress, TextField, Button, Paper, Typography} from '@material-ui/core' import { useTheme } from '@material-ui/core/styles'; @@ -33,6 +31,7 @@ const LoginDialog = props => { const [username, setUsername] = useState(""); const [password, setPassword] = useState(""); const [firstRequest, setFirstRequest] = useState(true); + const [loginLoading, setLoginLoading] = useState(false); // Used to swap from login to register. True = login, false = register @@ -69,7 +68,7 @@ const LoginDialog = props => { }), ) .catch(error => { - setLoginInfo("Error logging in: ", error) + setLoginInfo("Error logging in - please refresh in a minute: ", error) }) } @@ -79,6 +78,7 @@ const LoginDialog = props => { } const onSubmit = (e) => { + setLoginLoading(true) e.preventDefault() setLoginInfo("") // FIXME - add some check here ROFL @@ -101,6 +101,7 @@ const LoginDialog = props => { }) .then(response => response.json().then(responseJson => { + setLoginLoading(false) if (responseJson["success"] === false) { setLoginInfo(responseJson["reason"]) } else { @@ -116,6 +117,7 @@ const LoginDialog = props => { }), ) .catch(error => { + setLoginLoading(false) setLoginInfo("Error logging in: " + error) }); } else { @@ -231,8 +233,9 @@ const LoginDialog = props => { />
- - +
{loginInfo} From d920158cd69314f09db282cf6c9ddf0511e07feb Mon Sep 17 00:00:00 2001 From: frikky Date: Thu, 27 May 2021 21:01:23 +0200 Subject: [PATCH 60/96] Fixed password change bug --- .env | 5 ++-- backend/go-app/go.mod | 4 +-- backend/go-app/go.sum | 2 ++ backend/go-app/main.go | 37 ++++++++++++++++++------ backend/go-app/walkoff.go | 4 +-- docker-compose.yml | 5 ++-- frontend/src/components/ParsedAction.jsx | 25 +++++++++++----- frontend/src/views/Admin.jsx | 2 +- frontend/src/views/AngularWorkflow.jsx | 2 +- frontend/src/views/Apps.jsx | 2 +- frontend/src/views/SettingsPage.jsx | 3 +- 11 files changed, 62 insertions(+), 29 deletions(-) diff --git a/.env b/.env index 3413f275..11e0f712 100644 --- a/.env +++ b/.env @@ -46,8 +46,9 @@ SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-0.8.60" # Used for auto-cleanup of containers. REALLY important at scale. SHUFFLE_CONTAINER_AUTO_CLEANUP=false +SHUFFLE_ELASTIC=true -SHUFFLE_OPENSEARCH_URL="http://shuffle-opensearch:9200" +SHUFFLE_OPENSEARCH_URL=http://shuffle-opensearch:9200 SHUFFLE_OPENSEARCH_USERNAME="" SHUFFLE_OPENSEARCH_PASSWORD="" -SHUFFLE_ELASTIC="true" +SHUFFLE_OPENSEARCH_CERTIFICATE_FILE="" diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 74d4f4f7..cfc22946 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -2,7 +2,7 @@ module shuffle go 1.13 -//replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared +replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi @@ -24,7 +24,7 @@ require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 // indirect github.com/elastic/go-elasticsearch/v8 v8.0.0-20210519083322-55daf7425ecb // indirect github.com/frikky/kin-openapi v0.39.0 - github.com/frikky/shuffle-shared v0.0.50 + github.com/frikky/shuffle-shared v0.0.51 github.com/fsouza/go-dockerclient v1.7.2 // indirect github.com/ghodss/yaml v1.0.0 github.com/go-git/go-billy/v5 v5.0.0 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 759c5d66..1a88952a 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -159,6 +159,8 @@ github.com/frikky/shuffle-shared v0.0.49 h1:fChF0Nh/bMuXZg67Pt9XXn9+mH4IlKgB3dAz github.com/frikky/shuffle-shared v0.0.49/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= github.com/frikky/shuffle-shared v0.0.50 h1:dQIXf4mwUHuEVsXiMtZaSznz6vWt+C0KjyTAsAgMs3s= github.com/frikky/shuffle-shared v0.0.50/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= +github.com/frikky/shuffle-shared v0.0.51 h1:JrCGoRNj/LkAvSlkyx7tLv7ToNtJDIAqKqiA/poO+G4= +github.com/frikky/shuffle-shared v0.0.51/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= github.com/fsouza/go-dockerclient v1.7.2 h1:bBEAcqLTkpq205jooP5RVroUKiVEWgGecHyeZc4OFjo= github.com/fsouza/go-dockerclient v1.7.2/go.mod h1:+ugtMCVRwnPfY7d8/baCzZ3uwB0BrG5DB8OzbtxaRz8= github.com/getkin/kin-openapi v0.8.0 h1:a6TQjTqwkyscC4/hShJX7WhCVE+4bi9lzw61XHQW5hE= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 72fa2ae8..e315afbd 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -15,6 +15,7 @@ import ( "errors" "path/filepath" + "crypto/tls" "fmt" "io" "io/ioutil" @@ -4240,7 +4241,7 @@ func runInitEs(ctx context.Context) { cloneOptions.ReferenceName = plumbing.ReferenceName(branch) } - log.Printf("Getting apps from %s", url) + log.Printf("[DEBUG] Getting apps from %s", url) r, err := git.Clone(storer, fs, cloneOptions) @@ -5649,7 +5650,7 @@ func initHandlers() { //requestCache = cache.New(5*time.Minute, 10*time.Minute) dbclient, err = datastore.NewClient(ctx, gceProject, option.WithGRPCDialOption(grpc.WithNoProxy())) if err != nil { - panic(fmt.Sprintf("[DEBUG] Database client error during init: %s", err)) + log.Fatalf("[DEBUG] Database client error during init: %s", err) } esUrl := os.Getenv("SHUFFLE_OPENSEARCH_URL") @@ -5657,15 +5658,33 @@ func initHandlers() { esUrl = "http://shuffle-opensearch:9200" } - es, err := elasticsearch.NewClient( - elasticsearch.Config{ - Addresses: []string{esUrl}, - Username: os.Getenv("SHUFFLE_OPENSEARCH_USERNAME"), - Password: os.Getenv("SHUFFLE_OPENSEARCH_PASSWORD"), + config := elasticsearch.Config{ + Addresses: []string{esUrl}, + Username: os.Getenv("SHUFFLE_OPENSEARCH_USERNAME"), + Password: os.Getenv("SHUFFLE_OPENSEARCH_PASSWORD"), + Transport: &http.Transport{ + MaxIdleConnsPerHost: 100, + ResponseHeaderTimeout: time.Second, + TLSClientConfig: &tls.Config{ + MinVersion: tls.VersionTLS11, + }, }, - ) + } + + certificateLocation := os.Getenv("SHUFFLE_OPENSEARCH_CERTIFICATE_FILE") + if len(certificateLocation) > 0 { + cert, err := ioutil.ReadFile(certificateLocation) + if err != nil { + log.Fatalf("[WARNING] Failed configuring certificates: %s not found", err) + } else { + config.CACert = cert + } + log.Printf("[INFO] Added certificate %#v elastic client.", certificateLocation) + } + + es, err := elasticsearch.NewClient(config) if err != nil { - panic(fmt.Sprintf("[DEBUG] Database client for ELASTICSEARCH error during init: %s", err)) + log.Fatalf("[DEBUG] Database client for ELASTICSEARCH error during init (fatal): %s", err) } elasticConfig := "elasticsearch" diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index c3adae0e..e03ed4df 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -3421,7 +3421,7 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, //log.Printf("%s", string(readFile)) swagger, err := openapi3.NewSwaggerLoader().LoadSwaggerFromData([]byte(parsedOpenApi.Body)) if err != nil { - log.Printf("Swagger validation error in loop (%s): %s", filename, err) + log.Printf("[WARNING] Swagger validation error in loop (%s): %s. Continuing.", filename, err) continue } @@ -3432,7 +3432,7 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, //log.Printf("Should generate yaml") swagger, api, _, err := shuffle.GenerateYaml(swagger, parsedOpenApi.ID) if err != nil { - log.Printf("Failed building and generating yaml in loop (%s): %s", filename, err) + log.Printf("Failed building and generating yaml in loop (2) (%s): %s. Continuing.", filename, err) continue } diff --git a/docker-compose.yml b/docker-compose.yml index 9fcdbcf5..bae6d5bf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,7 +17,7 @@ services: - backend backend: #build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.93 + image: ghcr.io/frikky/shuffle-backend:0.8.95 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: @@ -29,9 +29,10 @@ services: - /var/run/docker.sock:/var/run/docker.sock - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps - ${SHUFFLE_FILE_LOCATION}:/shuffle-files + #- ${SHUFFLE_OPENSEARCH_CERTIFICATE_FILE}:/shuffle-files/es_certificate environment: - DATASTORE_EMULATOR_HOST=shuffle-database:8000 - - SHUFFLE_OPENSEARCH_URL=http://shuffle-opensearch:9200 + #- SHUFFLE_OPENSEARCH_URL=${SHUFFLE_OPENSEARCH_URL} - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps - SHUFFLE_FILE_LOCATION=/shuffle-files - ORG_ID=${ORG_ID} diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index be5588cc..ba331a27 100644 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -6,6 +6,7 @@ import { GetIconInfo } from "../views/Workflows.jsx"; import { sortByKey } from "../views/AngularWorkflow.jsx"; import { useTheme } from '@material-ui/core/styles'; import NestedMenuItem from "material-ui-nested-menu-item"; +//import NestedMenuItem from "./NestedMenu.jsx"; import {Popper, TextField, Drawer, Button, Paper, Grid, Tabs, InputAdornment, Tab, ButtonBase, Tooltip, Select, MenuItem, Divider, Dialog, Modal, DialogActions, DialogTitle, InputLabel, DialogContent, FormControl, IconButton, Menu, Input, FormGroup, FormControlLabel, Typography, Checkbox, Breadcrumbs, CircularProgress, Switch, Fade} from '@material-ui/core'; import {GetApp as GetAppIcon, Search as SearchIcon, ArrowUpward as ArrowUpwardIcon, Visibility as VisibilityIcon, Done as DoneIcon, Close as CloseIcon, Error as ErrorIcon, FindReplace as FindreplaceIcon, ArrowLeft as ArrowLeftIcon, Cached as CachedIcon, DirectionsRun as DirectionsRunIcon, Add as AddIcon, Polymer as PolymerIcon, FormatListNumbered as FormatListNumberedIcon, Create as CreateIcon, PlayArrow as PlayArrowIcon, AspectRatio as AspectRatioIcon, MoreVert as MoreVertIcon, Apps as AppsIcon, Schedule as ScheduleIcon, FavoriteBorder as FavoriteBorderIcon, Pause as PauseIcon, Delete as DeleteIcon, AddCircleOutline as AddCircleOutlineIcon, Save as SaveIcon, KeyboardArrowLeft as KeyboardArrowLeftIcon, KeyboardArrowRight as KeyboardArrowRightIcon, ArrowBack as ArrowBackIcon, Settings as SettingsIcon, LockOpen as LockOpenIcon, ExpandMore as ExpandMoreIcon, VpnKey as VpnKeyIcon} from '@material-ui/icons'; @@ -921,6 +922,7 @@ const ParsedAction = (props) => { border: `2px solid #f85a3e`, color: "white", marginTop: 2, + maxHeight: 400, }} > {actionlist.map(innerdata => { @@ -970,11 +972,11 @@ const ParsedAction = (props) => { const handleMouseover = () => { if (innerdata.type === "Execution Argument") { - handleExecArgumentHover(true) - } else if (innerdata.type === "action") { - handleActionHover(true, innerdata.id) - } - } + handleExecArgumentHover(true) + } else if (innerdata.type === "action") { + handleActionHover(true, innerdata.id) + } + } const handleMouseOut = () => { if (innerdata.type === "Execution Argument") { @@ -999,8 +1001,15 @@ const ParsedAction = (props) => {
} parentMenuOpen={!!menuPosition} - style={{backgroundColor: theme.palette.inputColor, color: "white", minWidth: 250,}} + style={{backgroundColor: theme.palette.inputColor, color: "white", minWidth: 250, maxWidth: 250, maxHeight: 400, scrollX: "", }} + //PaperProps={{ + // style: { + // maxHeight: 400, + // width: 250, + // } + //}} onClick={() => { + console.log("CLICKED: ", innerdata) handleItemClick([innerdata]) }} > @@ -1008,7 +1017,7 @@ const ParsedAction = (props) => { // FIXME: Should be recursive in here const icon = pathdata.type === "value" ? : pathdata.type === "list" ? : return ( - {}} + {console.log("HOVER: ", pathdata)}} onClick={() => { handleItemClick([innerdata, pathdata]) }} @@ -1024,7 +1033,7 @@ const ParsedAction = (props) => { })} : - handleMouseover()} onMouseOut={() => {handleMouseOut()}} + handleMouseover()} onMouseOut={() => {handleMouseOut()}} onClick={() => { handleItemClick([innerdata]) }} diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index e3c0f921..02b2063a 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -2700,7 +2700,7 @@ const Admin = (props) => { App Authentication/> Files /> Schedules /> - {/*isCloud ? null : Environments/>*/} + {isCloud ? null : Environments/>} {window.location.protocol == "http:" && window.location.port === "3000" ? Hybrid/> : null} {window.location.protocol == "http:" && window.location.port === "3000" ? Organizations/> : null} {window.location.protocol === "http:" && window.location.port === "3000" ? Categories/> : null} diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index f79df0e0..bc0d4f50 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -2270,7 +2270,7 @@ const AngularWorkflow = (props) => { case 90: if (previouskey === 17) { console.log("CTRL+Z") - handleHistoryUndo() + //handleHistoryUndo() } break; diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 5df39bbc..3e08a13b 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -1144,7 +1144,7 @@ const Apps = (props) => { .then((response) => { setIsLoading(false) if (response.status === 200) { - alert.success("Hotloaded apps!") + //alert.success("Hotloaded apps!") getApps() } diff --git a/frontend/src/views/SettingsPage.jsx b/frontend/src/views/SettingsPage.jsx index b1f53c1c..6a1cdd4a 100644 --- a/frontend/src/views/SettingsPage.jsx +++ b/frontend/src/views/SettingsPage.jsx @@ -29,6 +29,7 @@ const Settings = (props) => { const [userInfo, ] = useState(userdata) const [userSettings, setUserSettings] = useState({}) + const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" const bodyDivStyle = { margin: "auto", @@ -421,7 +422,7 @@ const Settings = (props) => { />
+ Start + + Stop +
@@ -5953,6 +6158,14 @@ const AngularWorkflow = (props) => { startNode = branch.destination_id } + const param = trigger.parameters.find(param => param.name === "auth_headers") + console.log("PARAM: ", param) + var auth = "" + if (param !== undefined && param !== null) { + auth = param.value + } + + console.log("TRIG: ", trigger) const data = { "name": hookname, "type": "webhook", @@ -5960,6 +6173,7 @@ const AngularWorkflow = (props) => { "workflow": workflow.id, "start": startNode, "environment": trigger.environment, + "auth": auth, } fetch(globalUrl+"/api/v1/hooks/new", { @@ -6943,6 +7157,7 @@ const AngularWorkflow = (props) => { } console.log("NEW: ", copy) + console.log("NAVIGATOR: ", navigator) navigator.clipboard.writeText(JSON.stringify(copy)) copyText.select() @@ -6993,6 +7208,7 @@ const AngularWorkflow = (props) => { const elementName = "copy_element_shuffle" var copyText = document.getElementById(elementName) if (copyText !== null && copyText !== undefined) { + console.log("NAVIGATOR: ", navigator) navigator.clipboard.writeText(to_be_copied) copyText.select() copyText.setSelectionRange(0, 99999); /* For mobile devices */ @@ -7525,6 +7741,7 @@ const AngularWorkflow = (props) => { console.log("PRECOPY: ", to_be_copied) if (copyText !== null && copyText !== undefined) { console.log("COPY: ", copyText) + console.log("NAVIGATOR: ", navigator) navigator.clipboard.writeText(to_be_copied) copyText.select(); copyText.setSelectionRange(0, 99999); /* For mobile devices */ diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index abd711ef..adea35af 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -659,6 +659,11 @@ const AppCreator = (props) => { var newbody = {} for (var propkey in parameter.properties) { const parsedkey = propkey.replaceAll(" ", "_").toLowerCase() + if (parameter.properties[propkey].type === undefined) { + console.log("Skipping: ", parameter.properties[propkey]) + continue + } + if (parameter.properties[propkey].type === "string") { if (parameter.properties[propkey].description !== undefined) { newbody[parsedkey] = parameter.properties[propkey].description @@ -668,7 +673,7 @@ const AppCreator = (props) => { } else if (parameter.properties[propkey].type.includes("int")) { newbody[parsedkey] = 0 } else { - console.log("CANT HANDLE TYPE ", parameter.properties[propkey].type) + console.log("CANT HANDLE JSON TYPE ", parameter.properties[propkey].type, parameter.properties[propkey]) newbody[parsedkey] = [] } } @@ -690,6 +695,11 @@ const AppCreator = (props) => { var newbody = {} for (var propkey in parameter.properties) { const parsedkey = propkey.replaceAll(" ", "_").toLowerCase() + if (parameter.properties[propkey].type === undefined) { + console.log("Skipping: ", parameter.properties[propkey]) + continue + } + if (parameter.properties[propkey].type === "string") { if (parameter.properties[propkey].description !== undefined) { newbody[parsedkey] = parameter.properties[propkey].description @@ -699,7 +709,7 @@ const AppCreator = (props) => { } else if (parameter.properties[propkey].type.includes("int")) { newbody[parsedkey] = 0 } else { - console.log("CANT HANDLE TYPE ", parameter.properties[propkey].type) + console.log("CANT HANDLE JSON TYPE (2) ", parameter.properties[propkey].type) newbody[parsedkey] = [] } } @@ -720,16 +730,22 @@ const AppCreator = (props) => { var newbody = {} for (var propkey in parameter.properties) { const parsedkey = propkey.replaceAll(" ", "_").toLowerCase() + if (parameter.properties[propkey].type === undefined) { + console.log("Skipping: ", parameter.properties[propkey]) + continue + } + if (parameter.properties[propkey].type === "string") { if (parameter.properties[propkey].description !== undefined) { newbody[parsedkey] = parameter.properties[propkey].description } else { newbody[parsedkey] = "" } + console.log(parameter.properties[propkey]) } else if (parameter.properties[propkey].type.includes("int")) { newbody[parsedkey] = 0 } else { - console.log("CANT HANDLE TYPE ", parameter.properties[propkey].type) + console.log("CANT HANDLE JSON TYPE (3) ", parameter.properties[propkey].type) newbody[parsedkey] = [] } } @@ -882,6 +898,8 @@ const AppCreator = (props) => { if (value.scheme === "bearer") { setAuthenticationOption("Bearer auth") setAuthenticationRequired(true) + } else if (key === "oauth2") { + alert.info("Can't handle Oauth2 auth yet.") } else if (key === "ApiKeyAuth") { setAuthenticationOption("API key") @@ -1460,7 +1478,7 @@ const AppCreator = (props) => { const extraKeys =
- Extra authentication options + Add global headers or queries {extraAuth.length === 0 ?
} parentMenuOpen={!!menuPosition} - style={{backgroundColor: theme.palette.inputColor, color: "white", minWidth: 250, maxWidth: 250, maxHeight: 400, scrollX: "", }} + style={{backgroundColor: theme.palette.inputColor, color: "white", minWidth: 250, maxWidth: 250, maxHeight: 650, scrollX: "", }} //PaperProps={{ // style: { // maxHeight: 400, @@ -1040,7 +1039,7 @@ const ParsedAction = (props) => { // FIXME: Should be recursive in here const icon = pathdata.type === "value" ? : pathdata.type === "list" ? : return ( - {console.log("HOVER: ", pathdata)}} + {console.log("HOVER: ", pathdata)}} onClick={() => { handleItemClick([innerdata, pathdata]) }} @@ -1189,7 +1188,7 @@ const ParsedAction = (props) => { onClick={() => setShowAutocomplete(true)} fullWidth open={showAutocomplete} - style={{border: `2px solid #f85a3e`, color: "white", height: 50, marginTop: 2, borderRadius: theme.palette.borderRadius,}} + style={{color: "white", height: 50, marginTop: 2, borderRadius: theme.palette.borderRadius,}} onChange={(e) => { if (selectedActionParameters[count].value[selectedActionParameters[count].value.length-1] === ".") { e.target.value.autocomplete = e.target.value.autocomplete.slice(1, e.target.value.autocomplete.length) @@ -1240,6 +1239,7 @@ const ParsedAction = (props) => { // return //} + const baselabel = selectedAction.label return (
{hideExtraTypes === true ? null : @@ -1339,6 +1339,21 @@ const ParsedAction = (props) => { color="primary" placeholder={selectedAction.label} onChange={selectedNameChange} + onBlur={(e) => { + const name = e.target.value + console.log("CHANGED FROM2: ", baselabel) + console.log("CHANGED TO: ", name) + for (var key in workflow.actions) { + for (var subkey in workflow.actions[key].parameters) { + const param = workflow.actions[key].parameters[subkey] + if (param.value.includes(baselabel)) { + //if (param.value.toLowerCase().includes(baselabel)) { + console.log("FOUND: ", param) + workflow.actions[key].parameters[subkey].value.replaceAll(baselabel, e.target.value) + } + } + } + }} /> } diff --git a/frontend/src/defaultCytoscapeStyle.js b/frontend/src/defaultCytoscapeStyle.js index b0f135a7..48642d60 100644 --- a/frontend/src/defaultCytoscapeStyle.js +++ b/frontend/src/defaultCytoscapeStyle.js @@ -131,7 +131,7 @@ const data = [{ 'height': '15px', 'z-index': '5002', 'font-size': '0px', - 'border': '1px solid black', + 'border': '1px solid rgba(255,255,255,0.9)', 'background-image': 'data(icon)', 'background-color': 'data(iconBackground)', }, diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 5fbd3f1e..ca1771b8 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1749,7 +1749,7 @@ const AngularWorkflow = (props) => { if (data.isButton) { //console.log("BUTTON CLICKED: ", data) if (data.buttonType === "delete") { - console.log("DELETE!") + //console.log("DELETE!") const parentNode = cy.getElementById(data.attachedTo) if (parentNode !== null && parentNode !== undefined) { parentNode.remove() @@ -2007,6 +2007,247 @@ const AngularWorkflow = (props) => { }) } + const GetExampleResult = (item) => { + var exampledata = item.example === undefined ? "" : item.example + if (workflowExecutions.length > 0) { + // Look for the ID + const found = false + for (var key in workflowExecutions) { + if (workflowExecutions[key].results === undefined || workflowExecutions[key].results === null) { + continue + } + + var foundResult = {"result": ""} + if (item.id === "exec") { + //console.log("EXEC: ", workflowExecutions[key].execution_argument) + if (workflowExecutions[key].execution_argument !== undefined && workflowExecutions[key].execution_argument !== null && workflowExecutions[key].execution_argument.length > 0) { + foundResult.result = workflowExecutions[key].execution_argument + } else { + continue + } + } else { + foundResult = workflowExecutions[key].results.find(result => result.action.id === item.id) + if (foundResult === undefined) { + continue + } + } + + foundResult.result = foundResult.result.trim() + foundResult.result = foundResult.result.split(" None").join(" \"None\"") + foundResult.result = foundResult.result.split(" False").join(" false") + foundResult.result = foundResult.result.split(" True").join(" true") + + var jsonvalid = true + try { + const tmp = String(JSON.parse(foundResult.result)) + if (!foundResult.result.includes("{") && !foundResult.result.includes("[")) { + jsonvalid = false + } + } catch (e) { + try { + foundResult.result = foundResult.result.split("\'").join("\"") + const tmp = String(JSON.parse(foundResult.result)) + if (!foundResult.result.includes("{") && !foundResult.result.includes("[")) { + jsonvalid = false + } + } catch (e) { + jsonvalid = false + } + } + + // Finds the FIRST json only + if (jsonvalid) { + //console.log("VALID!") + exampledata = JSON.parse(foundResult.result) + break + } else { + //console.log("INVALID: ", foundResult.result) + } + } + } + + //console.log("EXAMPLE: ", exampledata) + return exampledata + } + + const GetParamMatch = (paramname, exampledata, basekey) => { + const splitkey = "." + //console.log(typeof(exampledata)) + //console.log("MATCHING WITH: ", exampledata) + if (typeof(exampledata) !== "object") { + return "" + } + + // Basically just a stupid if-else :) + const synonyms = { + "id": ["id", "ref", "sourceref", "reference", "sourcereference", "alert id", "case id", "incident id", "service id",], + "title": ["title", "name", "message"], + "description": ["description", "explanation", "story", "details",], + "email": ["mail", "email", "sender", "receiver", "recipient"], + "data": ["data", "ip", "domain", "url", "hash", "md5", "sha2", "sha256", "value", "item",], + } + + // 1. Find the right synonym + // 2. + var selectedsynonyms = [paramname] + for (const [key, value] of Object.entries(synonyms)) { + if (key === paramname || value.includes(paramname)) { + if (!value.includes(key)) { + value.push(key.toLowerCase()) + } + + selectedsynonyms = value + break + } + } + //console.log("SELECTED: ", selectedsynonyms) + + //console.log("SYNONYMS FOR ", paramname, selectedsynonyms) + var toreturn = "" + + for (const [key, value] of Object.entries(exampledata)) { + // Check if loop or JSON + const extra = basekey.length > 0 ? splitkey : "" + const basekeyname = `${basekey.slice(1, basekey.length).split(".").join(splitkey)}${extra}${key}` + + // Handle direct loop! + //if (!isNaN(key) && basekey === "") { + // //console.log("Handling direct loop: ", key, value) + // //parsedValues.push({"type": "object", "name": "Node", "autocomplete": `${basekey}`}) + // //parsedValues.push({"type": "list", "name": `${splitkey}list`, "autocomplete": `${basekey}.#`}) + // //for (var subkey in returnValues) { + // // parsedValues.push(returnValues[subkey]) + // //} + + // toreturn = GetParsedPaths(paramname, value, `${basekey}.#`) + // console.log("LIST, TORETURN: ", value, toreturn) + // if (toreturn.length > 0) { + // break + // } + //} + + //console.log("KEY: ", key, "VALUE: ", value, "BASEKEY: ", basekeyname) + if (typeof(value) === 'object') { + if (Array.isArray(value)) { + //console.log("LIST!!: ", value, key) + var selectedkey = "" + if (isNaN(key)) { + selectedkey = `.${key}` + } + + for (var subitem in value) { + toreturn = GetParamMatch(paramname, value[subitem], `${basekey}${selectedkey}.#`) + if (toreturn.length > 0) { + break + } + } + + if (toreturn.length > 0) { + break + } + } else { + var selectedkey = "" + if (isNaN(key)) { + selectedkey = `.${key}` + } + + toreturn = GetParamMatch(paramname, value, `${basekey}${selectedkey}`) + //console.log("OBJECT: ", value, toreturn, key) + if (toreturn.length > 0) { + break + } + } + //console.log("VALUE IS OBJECT: ", key, value) + } else { + //console.log("SINGLE ITEM: ", key) + if (selectedsynonyms.includes(key.toLowerCase())) { + //parsedValues.push({"type": "value", "name": basekeyname, "autocomplete": `${basekey}.${key}`, "value": value,}) + //console.log("STRING: ", key, value) + toreturn = `${basekey}.${key}` + //toreturn = basekeyname + break + } + } + } + + return toreturn + } + + // Takes an action as input, then runs through and updates the relevant fields + // based on previous actions' + const RunAutocompleter = (dstdata) => { + // **PS: The right action should already be set here** + // 1. Check execution argument + // 2. Check parents in order + var exampledata = GetExampleResult({"id": "exec", "name": "exec",}) + //console.log("EXAMPLE RETURN: ", exampledata) + var parentlabel = "exec" + for (var paramkey in dstdata.parameters) { + const param = dstdata.parameters[paramkey] + // Skip authentication params + if (param.configuration) { + continue + } + + const paramname = param.name.toLowerCase().trim().replaceAll("_", " ") + //console.log("PARAM: ", param) + //console.log("PARAMNAME: ", paramname) + + const foundresult = GetParamMatch(paramname, exampledata, "") + if (foundresult.length > 0) { + //console.log("FOUND: ", paramname, foundresult) + + if (dstdata.parameters[paramkey].value.length === 0) { + dstdata.parameters[paramkey].value = `$${parentlabel}${foundresult}` + } else { + //console.log("Skipping ", dstdata.parameters[paramkey], " because it already has a value") + } + } + } + + var parents = getParents(dstdata) + console.log("PARENTS: ", parents) + if (parents.length > 1) { + for (var key in parents) { + const item = parents[key] + if (item.label === "Execution Argument") { + continue + } + + parentlabel = item.label.toLowerCase().trim().replaceAll(" ", "_") + exampledata = GetExampleResult(item) + for (var paramkey in dstdata.parameters) { + const param = dstdata.parameters[paramkey] + // Skip authentication params + if (param.configuration) { + continue + } + + const paramname = param.name.toLowerCase().trim().replaceAll("_", " ") + //console.log("PARAM: ", param) + //console.log("PARAMNAME: ", paramname) + + const foundresult = GetParamMatch(paramname, exampledata, "") + if (foundresult.length > 0) { + //console.log("FOUND: ", paramname, foundresult) + + if (dstdata.parameters[paramkey].value.length === 0) { + dstdata.parameters[paramkey].value = `$${parentlabel}${foundresult}` + } else { + //console.log("Skipping ", dstdata.parameters[paramkey], " because it already has a value") + } + } + } + // Check agains every param + } + } + + return dstdata + } + + //const FixNameUpdater = (sourcenode) => { + //} + // Checks for errors in edges when they're added const onEdgeAdded = (event) => { setLastSaved(false) @@ -2021,6 +2262,7 @@ const AngularWorkflow = (props) => { } } + targetnode = -1 var sourcenode = workflow.triggers.findIndex(data => data.id === edge.source) console.log("SOURCENODE: ", sourcenode) @@ -2057,6 +2299,7 @@ const AngularWorkflow = (props) => { } } + //console.log(workflow.branches) // Check if: @@ -2120,6 +2363,19 @@ const AngularWorkflow = (props) => { } } + + // 1. Guess what the next node's action should be + // 2. Get result from previous nodes (if any) + // 3. TRY to automatically map them in based on synonyms + const newsource = cy.getElementById(edge.source) + const newdst = cy.getElementById(edge.target) + if (newsource !== undefined && newsource !== null && newdst !== undefined && newdst !== null) { + //const srcdata = newsource.data() + //console.log("EDGE: ", edge) + const dstdata = RunAutocompleter(newdst.data()) + console.log("DST: ", dstdata) + } + var newbranch = { "source_id": edge.source, "destination_id": edge.target, @@ -2426,6 +2682,9 @@ const AngularWorkflow = (props) => { switch( event.keyCode ) { case 27: console.log("ESCAPE") + if (configureWorkflowModalOpen === true) { + setConfigureWorkflowModalOpen(false) + } break; case 46: //removeNode() @@ -2730,6 +2989,8 @@ const AngularWorkflow = (props) => { }) } + const buttonColor = "rgba(255,255,255,0.9)" + const buttonBackgroundColor = "#1f2023" const addCopyButton = (event) => { var parentNode = cy.$('#' + event.target.data("id")); if (parentNode.data('isButton') || parentNode.data('buttonId')) @@ -2744,9 +3005,10 @@ const AngularWorkflow = (props) => { const iconInfo = { "icon": "M16 1H4c-1.1 0-2 .9-2 2v14h2V3h12V1zm-1 4l6 6v10c0 1.1-.9 2-2 2H7.99C6.89 23 6 22.1 6 21l.01-14c0-1.1.89-2 1.99-2h7zm-1 7h5.5L14 6.5V12z", - "iconColor": "black", - "iconBackgroundColor": "white", + "iconColor": buttonColor, + "iconBackgroundColor": buttonBackgroundColor, } + const svg_pin = `` const svgpin_Url = encodeURI("data:image/svg+xml;utf-8," + svg_pin) @@ -2783,8 +3045,8 @@ const AngularWorkflow = (props) => { const iconInfo = { "icon": "M6 19c0 1.1.9 2 2 2h8c1.1 0 2-.9 2-2V7H6v12zM19 4h-3.5l-1-1h-5l-1 1H5v2h14V4z", - "iconColor": "black", - "iconBackgroundColor": "white", + "iconColor": buttonColor, + "iconBackgroundColor": buttonBackgroundColor, } const svg_pin = `` const svgpin_Url = encodeURI("data:image/svg+xml;utf-8," + svg_pin) @@ -2811,6 +3073,10 @@ const AngularWorkflow = (props) => { const onNodeHover = (event) => { //console.log("TAR: ", event.target) const nodedata = event.target.data() + if (nodedata.finished === false) { + return + } + var parentNode = cy.$('#' + event.target.data("id")); if (parentNode.data('isButton') || parentNode.data('buttonId')) return @@ -3687,8 +3953,8 @@ const AngularWorkflow = (props) => { } const handleDragStop = (e, app) => { - console.log("STOP!: ", e) - console.log("APP!: ", parsedApp) + //console.log("STOP!: ", e) + //console.log("APP!: ", parsedApp) //const onNodeAdded = (event) => { //const node = event.target //const nodedata = event.target.data() @@ -3798,19 +4064,16 @@ const AngularWorkflow = (props) => { currentnode[0].renderedPosition("x", e.pageX-cycontainer.offsetLeft) currentnode[0].renderedPosition("y", e.pageY-cycontainer.offsetTop) } else { - console.log("IN NEW NODE!") if (workflow.public) { console.log("workflow is public - not adding") return } - console.log("IN NEW NODE2!") if (app.actions === undefined || app.actions === null || app.actions.length === 0) { alert.error("App "+app.name+" currently has no actions to perform. Please go to https://shuffler.io/apps to edit it.") return } - console.log("IN NEW NODE3!") newNodeId = uuid.v4() const actionType = "ACTION" const actionLabel = getNextActionName(app.name) @@ -3867,7 +4130,6 @@ const AngularWorkflow = (props) => { } } - console.log("IN NEW NODE4: !", nodeToBeAdded) parsedApp = nodeToBeAdded cy.add(nodeToBeAdded) return @@ -4046,40 +4308,45 @@ const AngularWorkflow = (props) => { } // Does this one find the wrong one? - selectedAction.name = newaction.name - selectedAction.parameters = JSON.parse(JSON.stringify(newaction.parameters)) - selectedAction.errors = [] - selectedAction.isValid = true - selectedAction.is_valid = true + var newSelectedAction = selectedAction + newSelectedAction.name = newaction.name + newSelectedAction.parameters = JSON.parse(JSON.stringify(newaction.parameters)) + newSelectedAction.errors = [] + newSelectedAction.isValid = true + newSelectedAction.is_valid = true - if (selectedAction.app_name === "Shuffle Tools") { - const iconInfo = GetIconInfo(selectedAction) + if (newSelectedAction.app_name === "Shuffle Tools") { + const iconInfo = GetIconInfo(newSelectedAction) console.log("ICONINFO: ", iconInfo) const svg_pin = `` const svgpin_Url = encodeURI("data:image/svg+xml;utf-8," + svg_pin) - selectedAction.large_image = svgpin_Url - selectedAction.fillGradient = iconInfo.fillGradient - selectedAction.fillstyle = "solid" - if (selectedAction.fillGradient !== undefined && selectedAction.fillGradient !== null && selectedAction.fillGradient.length > 0) { - selectedAction.fillstyle = 'linear-gradient' - console.log("GRADIENT!: ", selectedAction) + newSelectedAction.large_image = svgpin_Url + newSelectedAction.fillGradient = iconInfo.fillGradient + newSelectedAction.fillstyle = "solid" + if (newSelectedAction.fillGradient !== undefined && newSelectedAction.fillGradient !== null && newSelectedAction.fillGradient.length > 0) { + newSelectedAction.fillstyle = 'linear-gradient' + console.log("GRADIENT!: ", newSelectedAction) //action.fillstyle = //'background-fill': 'data(fillstyle)', } else { - selectedAction.iconBackground = iconInfo.iconBackgroundColor + newSelectedAction.iconBackground = iconInfo.iconBackgroundColor } - const foundnode = cy.getElementById(selectedAction.id) + const foundnode = cy.getElementById(newSelectedAction.id) if (foundnode !== null && foundnode !== undefined) { console.log("UPDATING NODE!") - foundnode.data(selectedAction) + foundnode.data(newSelectedAction) } } - console.log("ACTION: ", selectedAction) + // Takes an action as input, then runs through and updates the relevant fields + // based on previous actions' + newSelectedAction = RunAutocompleter(newSelectedAction) + + console.log("ACTION: ", newSelectedAction) if (newaction.returns.example !== undefined && newaction.returns.example !== null && newaction.returns.example.length > 0) { - selectedAction.example = newaction.returns.example + newSelectedAction.example = newaction.returns.example } // FIXME - this is broken sometimes lol @@ -4089,7 +4356,7 @@ const AngularWorkflow = (props) => { //} //setSelectedActionEnvironment(env) - setSelectedAction(selectedAction) + setSelectedAction(newSelectedAction) setUpdate(Math.random()) // FIXME - should change icon-node (descriptor) as well diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index ffb8e668..d9b5fc18 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -29,7 +29,7 @@ export const FixName = (name) => { // Parses JSON data into keys that can be used everywhere :) export const GetParsedPaths = (inputdata, basekey) => { - const splitkey = " > " + const splitkey = "." var parsedValues = [] if (inputdata === undefined || inputdata === null) { return parsedValues @@ -47,8 +47,8 @@ export const GetParsedPaths = (inputdata, basekey) => { // Handle direct loop! if (!isNaN(key) && basekey === "") { console.log("Handling direct loop.") - parsedValues.push({"type": "object", "name": "Node", "autocomplete": `${basekey}`}) - parsedValues.push({"type": "list", "name": `${splitkey}list`, "autocomplete": `${basekey}.#`}) + parsedValues.push({"type": "object", "name": "Node", "autocomplete": `${basekey}`.toLowerCase()}) + parsedValues.push({"type": "list", "name": `${splitkey}list`, "autocomplete": `${basekey}.#`.toLowerCase()}) const returnValues = GetParsedPaths(value, `${basekey}.#`) for (var subkey in returnValues) { parsedValues.push(returnValues[subkey]) @@ -61,8 +61,8 @@ export const GetParsedPaths = (inputdata, basekey) => { if (typeof(value) === 'object') { if (Array.isArray(value)) { // Check if each item is object - parsedValues.push({"type": "object", "name": basekeyname, "autocomplete": `${basekey}.${key}`}) - parsedValues.push({"type": "list", "name": `${basekeyname}${splitkey}list`, "autocomplete": `${basekey}.${key}.#`}) + parsedValues.push({"type": "object", "name": basekeyname, "autocomplete": `${basekey}.${key}`.toLowerCase()}) + parsedValues.push({"type": "list", "name": `${basekeyname}${splitkey}list`, "autocomplete": `${basekey}.${key}.#`.toLowerCase()}) // Only check the first. This would be probably be dumb otherwise. for (var subkey in value) { @@ -79,14 +79,14 @@ export const GetParsedPaths = (inputdata, basekey) => { } //console.log(key+" is array") } else { - parsedValues.push({"type": "object", "name": basekeyname, "autocomplete": `${basekey}.${key}`}) + parsedValues.push({"type": "object", "name": basekeyname, "autocomplete": `${basekey}.${key}`.toLowerCase()}) const returnValues = GetParsedPaths(value, `${basekey}.${key}`) for (var subkey in returnValues) { parsedValues.push(returnValues[subkey]) } } } else { - parsedValues.push({"type": "value", "name": basekeyname, "autocomplete": `${basekey}.${key}`, "value": value,}) + parsedValues.push({"type": "value", "name": basekeyname, "autocomplete": `${basekey}.${key}`.toLowerCase(), "value": value,}) } } diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index aa888b07..e215a9c7 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -3,7 +3,7 @@ import { useInterval } from 'react-powerhooks'; import { makeStyles } from '@material-ui/core/styles'; import {Avatar, Grid, Paper, Tooltip, Divider, Button, TextField, FormControl, IconButton, Menu, MenuItem, FormControlLabel, Chip, Switch, Typography, Zoom, CircularProgress, Dialog, DialogTitle, DialogActions, DialogContent} from '@material-ui/core'; -import {Compare as CompareIcon, Maximize as MaximizeIcon, Minimize as MinimizeIcon, AddCircle as AddCircleIcon, Toc as TocIcon, Send as SendIcon, Search as SearchIcon, FileCopy as FileCopyIcon, Delete as DeleteIcon, BubbleChart as BubbleChartIcon, Restore as RestoreIcon, Cached as CachedIcon, GetApp as GetAppIcon, Apps as AppsIcon, Edit as EditIcon, MoreVert as MoreVertIcon, PlayArrow as PlayArrowIcon, Add as AddIcon, Publish as PublishIcon, CloudUpload as CloudUploadIcon, CloudDownload as CloudDownloadIcon} from '@material-ui/icons'; +import {Close as CloseIcon, Compare as CompareIcon, Maximize as MaximizeIcon, Minimize as MinimizeIcon, AddCircle as AddCircleIcon, Toc as TocIcon, Send as SendIcon, Search as SearchIcon, FileCopy as FileCopyIcon, Delete as DeleteIcon, BubbleChart as BubbleChartIcon, Restore as RestoreIcon, Cached as CachedIcon, GetApp as GetAppIcon, Apps as AppsIcon, Edit as EditIcon, MoreVert as MoreVertIcon, PlayArrow as PlayArrowIcon, Add as AddIcon, Publish as PublishIcon, CloudUpload as CloudUploadIcon, CloudDownload as CloudDownloadIcon} from '@material-ui/icons'; //import {Search as SearchIcon, ArrowUpward as ArrowUpwardIcon, Visibility as VisibilityIcon, Done as DoneIcon, Close as CloseIcon, Error as ErrorIcon, FindReplace as FindreplaceIcon, ArrowLeft as ArrowLeftIcon, Cached as CachedIcon, DirectionsRun as DirectionsRunIcon, Add as AddIcon, Polymer as PolymerIcon, FormatListNumbered as FormatListNumberedIcon, Create as CreateIcon, PlayArrow as PlayArrowIcon, AspectRatio as AspectRatioIcon, MoreVert as MoreVertIcon, Apps as AppsIcon, Schedule as ScheduleIcon, FavoriteBorder as FavoriteBorderIcon, Pause as PauseIcon, Delete as DeleteIcon, AddCircleOutline as AddCircleOutlineIcon, Save as SaveIcon, KeyboardArrowLeft as KeyboardArrowLeftIcon, KeyboardArrowRight as KeyboardArrowRightIcon, ArrowBack as ArrowBackIcon, Settings as SettingsIcon, LockOpen as LockOpenIcon, ExpandMore as ExpandMoreIcon, VpnKey as VpnKeyIcon} from '@material-ui/icons'; //https://next.material-ui.com/components/material-icons/ @@ -78,6 +78,7 @@ const useStyles = makeStyles((theme) => ({ })); +// Takes an action in Shuffle and // Returns information about the icon, the color etc to be used // This can be used for actions of all types export const GetIconInfo = (action) => { @@ -95,10 +96,11 @@ export const GetIconInfo = (action) => { {"key": "send", "values": ["send", "dispatch", "mail", "forward", "post", "submit", "mark", "set"]}, {"key": "repeat", "values": ["repeat", "retry", "pause",]}, {"key": "execute", "values": ["execute", "run", "play", "raise",]}, - {"key": "extract", "values": ["extract", "unpack", "decompress"]}, + {"key": "extract", "values": ["extract", "unpack", "decompress", "open"]}, {"key": "inflate", "values": ["inflate", "pack", "compress",]}, {"key": "edit", "values": ["update", "create", "edit", "put", "patch", "change", "replace", "conver", "map", "format", "escape"]}, {"key": "compare", "values": ["compare", "convert", "to", "filter", "translate", "parse"]}, + {"key": "close", "values": ["close", "stop", "cancel",]}, ] var selectedKey = "" @@ -205,6 +207,13 @@ export const GetIconInfo = (action) => { "originalIcon": , "fillGradient": ["#03030e", "#205d66"] }, + "close": { + "icon": "M19 6.41L17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z", + "iconColor": "white", + "iconBackgroundColor": "#03030e", + "originalIcon": , + "fillGradient": ["#03030e", "#205d66"] + }, "send": { "icon": "M2.01 21L23 12 2.01 3 2 10l15 2-15 2z", "iconColor": "white", From e643d261eba62827dc1eb6dc895618a3c18139b8 Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 30 May 2021 22:38:47 +0200 Subject: [PATCH 69/96] Added uncommited files --- backend/tests/migrate_db.sh | 2 + backend/tests/test_wrappers.py | 210 +++++++++++++++++++++++++ backend/tests/validate_app_values.sh | 14 ++ frontend/src/components/NestedMenu.jsx | 207 ++++++++++++++++++++++++ 4 files changed, 433 insertions(+) create mode 100644 backend/tests/migrate_db.sh create mode 100644 backend/tests/test_wrappers.py create mode 100644 backend/tests/validate_app_values.sh create mode 100644 frontend/src/components/NestedMenu.jsx diff --git a/backend/tests/migrate_db.sh b/backend/tests/migrate_db.sh new file mode 100644 index 00000000..d1cbd039 --- /dev/null +++ b/backend/tests/migrate_db.sh @@ -0,0 +1,2 @@ +curl -XPOST -v localhost:5001/api/v1/migrate_database -H 'Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4' + diff --git a/backend/tests/test_wrappers.py b/backend/tests/test_wrappers.py new file mode 100644 index 00000000..d6ec96c6 --- /dev/null +++ b/backend/tests/test_wrappers.py @@ -0,0 +1,210 @@ +import re +import json + +def parse_nested_param(string, level): + """ + Generate strings contained in nested (), indexing i = level + """ + if len(re.findall("\(", string)) == len(re.findall("\)", string)): + LeftRightIndex = [x for x in zip( + [Left.start()+1 for Left in re.finditer('\(', string)], + reversed([Right.start() for Right in re.finditer('\)', string)]))] + + elif len(re.findall("\(", string)) > len(re.findall("\)", string)): + return parse_nested_param(string + ')', level) + elif len(re.findall("\(", string)) < len(re.findall("\)", string)): + return parse_nested_param('(' + string, level) + + else: + return 'Failed to parse params' + + try: + return [string[LeftRightIndex[level][0]:LeftRightIndex[level][1]]] + except IndexError: + return [string[LeftRightIndex[level+1][0]:LeftRightIndex[level+1][1]]] + +# Parses the deepest part +def maxDepth(S): + current_max = 0 + max = 0 + n = len(S) + + # Traverse the input string + for i in range(n): + if S[i] == '(': + current_max += 1 + + if current_max > max: + max = current_max + elif S[i] == ')': + if current_max > 0: + current_max -= 1 + else: + return -1 + + # finally check for unbalanced string + if current_max != 0: + return -1 + + return max-1 + +def parse_type(data, thistype): + if data == None: + return "Empty" + + if "int" in thistype: + try: + return int(data) + except ValueError: + print("ValueError while casting %s" % data) + return data + if "lower" in thistype: + return data.lower() + if "upper" in thistype: + return data.upper() + if "trim" in thistype: + return data.strip() + if "strip" in thistype: + return data.strip() + if "split" in thistype: + # Should be able to split anything + return data.split() + if "len" in thistype or "length" in thistype: + return len(data) + if "parse" in thistype: + splitvalues = [] + default_error = """Error. Expected syntax: parse(["hello","test1"],0:1)""" + if "," in data: + splitvalues = data.split(",") + + for item in range(len(splitvalues)): + splitvalues[item] = splitvalues[item].strip() + else: + return default_error + + lastsplit = [] + if ":" in splitvalues[-1]: + lastsplit = splitvalues[-1].split(":") + else: + try: + lastsplit = [int(splitvalues[-1])] + except ValueError: + return default_error + + try: + parsedlist = ",".join(splitvalues[0:-1]) + print(parsedlist) + print(lastsplit) + + if len(lastsplit) > 1: + tmp = json.loads(parsedlist)[int(lastsplit[0]):int(lastsplit[1])] + else: + tmp = json.loads(parsedlist)[lastsplit[0]] + + print(tmp) + return tmp + except IndexError as e: + return default_error + +# Parses the INNER value and recurses until everything is done +def parse_wrapper(data): + try: + if "(" not in data or ")" not in data: + return data + except TypeError: + return data + + print("Running %s" % data) + + # Look for the INNER wrapper first, then move out + wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length"] + found = False + for wrapper in wrappers: + if wrapper not in data.lower(): + continue + + found = True + break + + if not found: + return data + + # Do stuff here. + innervalue = parse_nested_param(data, maxDepth(data)-0) + outervalue = parse_nested_param(data, maxDepth(data)-1) + print("INNER: ", outervalue) + print("OUTER: ", outervalue) + + if outervalue != innervalue: + #print("Outer: ", outervalue, " inner: ", innervalue) + for key in range(len(innervalue)): + # Replace OUTERVALUE[key] with INNERVALUE[key] in data. + print("Replace %s with %s in %s" % (outervalue[key], innervalue[key], data)) + data = data.replace(outervalue[key], innervalue[key]) + else: + for thistype in wrappers: + if thistype not in data.lower(): + continue + + parsed_value = parse_type(innervalue[0], thistype) + return parsed_value + + print("DATA: %s\n" % data) + return parse_wrapper(data) + +def parse_wrapper_start(data): + newdata = [] + newstring = "" + record = True + paranCnt = 0 + for char in data: + if char == "(": + paranCnt += 1 + + if not record: + record = True + + if record: + newstring += char + + if paranCnt == 0 and char == " ": + newdata.append(newstring) + newstring = "" + record = True + + if char == ")": + paranCnt -= 1 + + if paranCnt == 0: + record = False + + if len(newstring) > 0: + newdata.append(newstring) + + parsedlist = [] + non_string = False + for item in newdata: + ret = parse_wrapper(item) + if not isinstance(ret, str): + non_string = True + + parsedlist.append(ret) + + if len(parsedlist) > 0 and not non_string: + return " ".join(parsedlist) + elif len(parsedlist) == 1 and non_string: + return parsedlist[0] + else: + print("Casting back to string because multi: ", parsedlist) + newlist = [] + for item in parsedlist: + try: + newlist.append(str(item)) + except ValueError: + newlist.append("parsing_error") + return " ".join(newlist) + +data = "split(hello there)" +data = """parse(["testing", "what", "is this"], 0:2)""" +#data = "int(int(2))" +print("RET: ", parse_wrapper_start(data)) diff --git a/backend/tests/validate_app_values.sh b/backend/tests/validate_app_values.sh new file mode 100644 index 00000000..69cf9d50 --- /dev/null +++ b/backend/tests/validate_app_values.sh @@ -0,0 +1,14 @@ +# ExecutionOrg MUST be executing. +curl -XPOST http://localhost:5001/api/v1/orgs/b199646b-16d2-456d-9fd6-b9972e929466/validate_app_values -d '{ + "append": true, + "workflow_check": true, + "authorization": "1aae630c-ccaf-4cb5-87f9-8a9e0a9afd11", + "execution_ref": "c59ff288-4f02-4d02-b839-133d55c7fdf0", + "org_id": "b199646b-16d2-456d-9fd6-b9972e929466", + "values": [{ + "app": "testing", + "action": "repeat_back_to_me", + "parameternames": ["call"], + "parametervalues": ["hey", "ho", "lets", "go"] + }] +}' diff --git a/frontend/src/components/NestedMenu.jsx b/frontend/src/components/NestedMenu.jsx new file mode 100644 index 00000000..cabf1b24 --- /dev/null +++ b/frontend/src/components/NestedMenu.jsx @@ -0,0 +1,207 @@ +import React, {useState, useRef, useImperativeHandle} from 'react' +import {makeStyles} from '@material-ui/core/styles' +import Menu, {MenuProps} from '@material-ui/core/Menu' +import MenuItem, {MenuItemProps} from '@material-ui/core/MenuItem' +import ArrowRight from '@material-ui/icons/ArrowRight' +import clsx from 'clsx' + +// + +//export interface NestedMenuItemProps { +// /** +// * Open state of parent ``, used to close decendent menus when the +// * root menu is closed. +// */ +// parentMenuOpen: boolean; +// /** +// * Component for the container element. +// * @default 'div' +// */ +// component: React.ElementType; +// /** +// * Effectively becomes the `children` prop passed to the `` +// * element. +// */ +// label: React.ReactNode; +// /** +// * @default +// */ +// rightIcon: React.ReactNode; +// /** +// * Props passed to container element. +// */ +// ContainerProps: React.HTMLAttributes; +// // &React.RefAttributes +// /** +// * Props passed to sub `` element +// */ +// MenuProps: Omit; +// /** +// * @see https://material-ui.com/api/list-item/ +// */ +// button: true; +//} + +const TRANSPARENT = 'rgba(0,0,0,0)' +const useMenuItemStyles = makeStyles((theme) => ({ + root: (props: any) => ({ + backgroundColor: props.open ? theme.palette.action.hover : TRANSPARENT + }) +})) + +/** + * Use as a drop-in replacement for `` when you need to add cascading + * menu elements as children to this component. + */ +//const NestedMenuItem = React.forwardRef( +const NestedMenuItem = (props, ref) => { + console.log(props, ref) + //function NestedMenuItem(props, ref) { + const { + parentMenuOpen, + component = 'div', + label, + rightIcon = , + children, + className, + tabIndex: tabIndexProp, + MenuProps = {}, + ContainerProps: ContainerPropsProp = {}, + ...MenuItemProps + } = props + + const [isSubMenuOpen, setIsSubMenuOpen] = useState(false) + + const {ref: containerRefProp, ...ContainerProps} = ContainerPropsProp + + + const menuItemRef = useRef(null) + useImperativeHandle(ref, () => menuItemRef.current) + const containerRef = useRef(null) + useImperativeHandle(containerRefProp, () => containerRef.current) + const menuContainerRef = useRef(null) + + console.log("PAST THIS: ", containerRefProp, menuItemRef, containerRef, menuContainerRef, ContainerProps) + + const handleMouseEnter = (event: React.MouseEvent) => { + setIsSubMenuOpen(true) + + if (ContainerProps?.onMouseEnter) { + ContainerProps.onMouseEnter(event) + } + } + const handleMouseLeave = (event: React.MouseEvent) => { + setIsSubMenuOpen(false) + + if (ContainerProps?.onMouseLeave) { + ContainerProps.onMouseLeave(event) + } + } + + // Check if any immediate children are active + const isSubmenuFocused = () => { + const active = containerRef.current?.ownerDocument?.activeElement + for (const child of menuContainerRef.current?.children ?? []) { + if (child === active) { + return true + } + } + return false + } + + const handleFocus = (event: React.FocusEvent) => { + if (event.target === containerRef.current) { + setIsSubMenuOpen(true) + } + + if (ContainerProps?.onFocus) { + ContainerProps.onFocus(event) + } + } + + const handleKeyDown = (event: React.KeyboardEvent) => { + if (event.key === 'Escape') { + return + } + + if (isSubmenuFocused()) { + event.stopPropagation() + } + + const active = containerRef.current?.ownerDocument?.activeElement + + if (event.key === 'ArrowLeft' && isSubmenuFocused()) { + containerRef.current?.focus() + } + + if ( + event.key === 'ArrowRight' && + event.target === containerRef.current && + event.target === active + ) { + console.log("MENU: ", menuContainerRef) + const firstChild = menuContainerRef.current.children[0] + console.log("FIRST: ", firstChild) + firstChild.focus() + } + } + + const open = isSubMenuOpen && parentMenuOpen + const menuItemClasses = useMenuItemStyles({open}) + + // Root element must have a `tabIndex` attribute for keyboard navigation + let tabIndex + if (!props.disabled) { + tabIndex = tabIndexProp !== undefined ? tabIndexProp : -1 + } + + console.log("PAST 2! ", tabIndex) + + return ( +
+ + {label} + {rightIcon} + + { + setIsSubMenuOpen(false) + }} + > +
+ {children} +
+ +
+ ) +} + +export default NestedMenuItem From bb4c088c04959afc803d949bf6e501591358500c Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 31 May 2021 19:11:15 +0200 Subject: [PATCH 70/96] #164: Added more drag&drop functionality to the UI autocompleter --- docker-compose.yml | 2 +- frontend/src/views/AngularWorkflow.jsx | 32 ++++++++++++++++++++++---- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 027485c4..b5a1cd8d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: frontend: #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.97 + image: ghcr.io/frikky/shuffle-frontend:0.8.98 container_name: shuffle-frontend hostname: shuffle-frontend ports: diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index ca1771b8..e27e04ef 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1552,10 +1552,11 @@ const AngularWorkflow = (props) => { const curElement = document.getElementById(styledElements[0]) if (curElement !== null && curElement !== undefined) { - console.log("ELE: ", curElement) + //console.log("ELE: ", curElement) curElement.style.border = curElement.style.original_border - const newValue = "$"+nodedata.label.toLowerCase().replaceAll(" ", "_") - curElement.value = newValue + var newValue = "$"+nodedata.label.toLowerCase().replaceAll(" ", "_") + var paramname = "" + var idnumber = -1 if (curElement.id.startsWith("rightside_field_")) { console.log("FOUND FIELD WITH NUMBER: ", curElement.id) const idsplit = curElement.id.split("_") @@ -1563,10 +1564,31 @@ const AngularWorkflow = (props) => { if (idsplit.length === 3 && !isNaN(idsplit[2])) { console.log("ADDING TO PARAM ", idsplit[2]) console.log("PARAM: ", selectedAction) - + selectedAction.parameters[idsplit[2]].value = newValue + paramname = selectedAction.parameters[idsplit[2]].name + idnumber = idsplit[2] } } + + console.log("ID ETC: ", idnumber, paramname) + if (idnumber >= 0 && paramname.length > 0) { + const exampledata = GetExampleResult(nodedata) + const parsedname = paramname.toLowerCase().trim().replaceAll("_", " ") + console.log("EX: ", exampledata) + + console.log("NAME: ", parsedname) + const foundresult = GetParamMatch(parsedname, exampledata, "") + console.log("RESULT: ", foundresult) + if (foundresult.length > 0) { + console.log("FOUND: ", paramname, foundresult) + newValue = `${newValue}${foundresult}` + } + + selectedAction.parameters[idnumber].value = newValue + } + + curElement.value = newValue } } @@ -1686,7 +1708,7 @@ const AngularWorkflow = (props) => { // Color for #f85a3e translated to rgb const newBorder = "3px solid rgb(248, 90, 62)" if (elementMouseIsOver.style.border != newBorder && elementMouseIsOver.id.includes("rightside")) { - console.log(elementMouseIsOver.style.border) + //console.log(elementMouseIsOver.style.border) if (elementMouseIsOver.style.border !== undefined) { elementMouseIsOver.style.original_border = elementMouseIsOver.style.border } else { From a1bb49c5e60f21263c5d3c1f7eda19c167fc1b3f Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 31 May 2021 20:53:30 +0200 Subject: [PATCH 71/96] Made backend run with .env file, and changed files to nightly --- .env | 14 +++++++++---- backend/go-app/main.go | 46 ++++++++++++++++++++++++++++++------------ docker-compose.yml | 19 +++-------------- 3 files changed, 46 insertions(+), 33 deletions(-) diff --git a/.env b/.env index 11e0f712..d4418b9a 100644 --- a/.env +++ b/.env @@ -21,6 +21,7 @@ SHUFFLE_DEFAULT_APIKEY= # Local location of your app directory. Can't use ~/ # Files will get better at some point. Right now: local saving. +SHUFFLE_APP_HOTLOAD_FOLDER=./shuffle-apps SHUFFLE_APP_HOTLOAD_LOCATION=./shuffle-apps SHUFFLE_FILE_LOCATION=./shuffle-files @@ -42,13 +43,18 @@ SHUFFLE_PASS_APP_PROXY=FALSE SHUFFLE_BASE_IMAGE_REGISTRY=ghcr.io SHUFFLE_BASE_IMAGE_NAME=frikky -SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-0.8.60" +SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-0.8.80" # Used for auto-cleanup of containers. REALLY important at scale. SHUFFLE_CONTAINER_AUTO_CLEANUP=false SHUFFLE_ELASTIC=true +# DATABASE CONFIGURATIONS +DATASTORE_EMULATOR_HOST=shuffl-database:8000 SHUFFLE_OPENSEARCH_URL=http://shuffle-opensearch:9200 -SHUFFLE_OPENSEARCH_USERNAME="" -SHUFFLE_OPENSEARCH_PASSWORD="" -SHUFFLE_OPENSEARCH_CERTIFICATE_FILE="" +SHUFFLE_OPENSEARCH_USERNAME= +SHUFFLE_OPENSEARCH_PASSWORD= +SHUFFLE_OPENSEARCH_CERTIFICATE_FILE= +SHUFFLE_OPENSEARCH_APIKEY= +SHUFFLE_OPENSEARCH_CLOUDID= +SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY=true diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 52a11b87..dd71a38f 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -4,18 +4,14 @@ import ( "github.com/frikky/shuffle-shared" "bufio" - "bytes" "context" "crypto/md5" + "crypto/tls" + //"crypto/x509" "encoding/hex" "encoding/json" - //"unicode/utf8" - "errors" - "path/filepath" - - "crypto/tls" "fmt" "io" "io/ioutil" @@ -24,6 +20,7 @@ import ( "net/url" "os" "os/exec" + "path/filepath" //"regexp" "strings" "time" @@ -5669,19 +5666,31 @@ func initHandlers() { esUrl = "http://shuffle-opensearch:9200" } + // https://github.com/elastic/go-elasticsearch/blob/f741c073f324c15d3d401d945ee05b0c410bd06d/elasticsearch.go#L98 config := elasticsearch.Config{ Addresses: []string{esUrl}, Username: os.Getenv("SHUFFLE_OPENSEARCH_USERNAME"), Password: os.Getenv("SHUFFLE_OPENSEARCH_PASSWORD"), - Transport: &http.Transport{ - MaxIdleConnsPerHost: 100, - ResponseHeaderTimeout: time.Second, - TLSClientConfig: &tls.Config{ - MinVersion: tls.VersionTLS11, - }, - }, + APIKey: os.Getenv("SHUFFLE_OPENSEARCH_APIKEY"), + CloudID: os.Getenv("SHUFFLE_OPENSEARCH_CLOUDID"), } + //config.Transport.TLSClientConfig + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.MaxIdleConnsPerHost = 1000 + transport.ResponseHeaderTimeout = time.Second * 10 + + skipSSLVerify := false + if strings.ToLower(os.Getenv("SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY")) == "true" { + skipSSLVerify = true + } + + transport.TLSClientConfig = &tls.Config{ + MinVersion: tls.VersionTLS11, + InsecureSkipVerify: skipSSLVerify, + } + + //https://github.com/elastic/go-elasticsearch/blob/master/_examples/security/elasticsearch-cluster.yml certificateLocation := os.Getenv("SHUFFLE_OPENSEARCH_CERTIFICATE_FILE") if len(certificateLocation) > 0 { cert, err := ioutil.ReadFile(certificateLocation) @@ -5689,9 +5698,20 @@ func initHandlers() { log.Fatalf("[WARNING] Failed configuring certificates: %s not found", err) } else { config.CACert = cert + + //if transport.TLSClientConfig.RootCAs, err = x509.SystemCertPool(); err != nil { + // log.Fatalf("[ERROR] Problem adding system CA: %s", err) + //} + + //// --> Add the custom certificate authority + //if ok := transport.TLSClientConfig.RootCAs.AppendCertsFromPEM(cert); !ok { + // log.Fatalf("[ERROR] Problem adding CA from file %q", *cert) + //} } + log.Printf("[INFO] Added certificate %#v elastic client.", certificateLocation) } + config.Transport = transport es, err := elasticsearch.NewClient(config) if err != nil { diff --git a/docker-compose.yml b/docker-compose.yml index b5a1cd8d..7cc35878 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: frontend: #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.98 + image: ghcr.io/frikky/shuffle-frontend:nightly container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -17,7 +17,7 @@ services: - backend backend: #build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.97 + image: ghcr.io/frikky/shuffle-backend:nightly container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: @@ -30,20 +30,7 @@ services: - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps - ${SHUFFLE_FILE_LOCATION}:/shuffle-files #- ${SHUFFLE_OPENSEARCH_CERTIFICATE_FILE}:/shuffle-files/es_certificate - environment: - - DATASTORE_EMULATOR_HOST=shuffle-database:8000 - #- SHUFFLE_OPENSEARCH_URL=${SHUFFLE_OPENSEARCH_URL} - - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps - - SHUFFLE_FILE_LOCATION=/shuffle-files - - ORG_ID=${ORG_ID} - - SHUFFLE_APP_DOWNLOAD_LOCATION=${SHUFFLE_APP_DOWNLOAD_LOCATION} - - SHUFFLE_DOWNLOAD_AUTH_BRANCH=${SHUFFLE_DOWNLOAD_AUTH_BRANCH} - - SHUFFLE_DEFAULT_USERNAME=${SHUFFLE_DEFAULT_USERNAME} - - SHUFFLE_DEFAULT_PASSWORD=${SHUFFLE_DEFAULT_PASSWORD} - - SHUFFLE_DEFAULT_APIKEY=${SHUFFLE_DEFAULT_APIKEY} - - SHUFFLE_APP_FORCE_UPDATE=${SHUFFLE_APP_FORCE_UPDATE} - - HTTP_PROXY=${SHUFFLE_HTTP_PROXY} - - HTTPS_PROXY=${SHUFFLE_HTTPS_PROXY} + env_file: .env restart: unless-stopped depends_on: - opensearch From 2c9bac9eb04b18e1bbd3b7085c4db0f4316fec98 Mon Sep 17 00:00:00 2001 From: frikky Date: Wed, 2 Jun 2021 15:09:28 +0200 Subject: [PATCH 72/96] Fixed clipboard issues - requires HTTPS or localhost --- backend/go-app/go.mod | 2 +- backend/go-app/main.go | 2 +- backend/go-app/walkoff.go | 83 +++++ docker-compose.yml | 5 +- frontend/package.json | 8 +- frontend/src/views/Admin.jsx | 24 +- frontend/src/views/AngularWorkflow.jsx | 36 ++- frontend/src/views/AppCreator.jsx | 407 ++++++++++++++++++++++++- 8 files changed, 552 insertions(+), 15 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 74585b9b..569852b4 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -2,7 +2,7 @@ module shuffle go 1.13 -//replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared +replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi diff --git a/backend/go-app/main.go b/backend/go-app/main.go index dd71a38f..bcf2abac 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5805,7 +5805,6 @@ func initHandlers() { r.HandleFunc("/api/v1/apps/authentication", shuffle.GetAppAuthentication).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/apps/authentication", shuffle.AddAppAuthentication).Methods("PUT", "OPTIONS") r.HandleFunc("/api/v1/apps/authentication/{appauthId}/config", shuffle.SetAuthenticationConfig).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/apps/authentication/{appauthId}", shuffle.DeleteAppAuthentication).Methods("DELETE", "OPTIONS") // Legacy app things @@ -5863,6 +5862,7 @@ func initHandlers() { r.HandleFunc("/api/v1/orgs/{orgId}/validate_app_values", shuffle.HandleKeyValueCheck).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/get_cache", shuffle.HandleGetCacheKey).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}/set_cache", shuffle.HandleSetCacheKey).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/apps/{key}/execute", executeSingleAction).Methods("POST", "OPTIONS") // Docker orborus specific - downloads an image r.HandleFunc("/api/v1/get_docker_image", getDockerImage).Methods("POST", "OPTIONS") diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 42623468..a0614772 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -4385,3 +4385,86 @@ func handleUserInput(trigger shuffle.Trigger, organizationId string, workflowId return nil } + +func executeSingleAction(resp http.ResponseWriter, request *http.Request) { + cors := shuffle.HandleCors(resp, request) + if cors { + return + } + + user, err := shuffle.HandleApiAuthentication(resp, request) + if err != nil { + log.Printf("[WARNING] Api authentication failed in execute SINGLE workflow - CONTINUING ANYWAY: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "You need to sign up to try it out}`)) + return + } + + location := strings.Split(request.URL.String(), "/") + var fileId string + if location[1] == "api" { + if len(location) <= 4 { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + fileId = location[4] + } + + body, err := ioutil.ReadAll(request.Body) + if err != nil { + log.Printf("[INFO] Failed workflowrequest POST read: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + ctx := context.Background() + workflowExecution, err := shuffle.PrepareSingleAction(ctx, user, fileId, body) + if err != nil { + log.Printf("[INFO] Failed workflowrequest POST read: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + //workflowExecution.ProjectId = gceProject + //workflowExecution.Locations = []string{defaultLocation} + + environments, err := shuffle.GetEnvironments(ctx, user.ActiveOrg.Id) + environment := "Shuffle" + if len(environments) >= 1 { + environment = environments[0].Name + } else { + log.Printf("[ERROR] No environments found for org %s. Exiting", user.ActiveOrg.Id) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + log.Printf("[INFO] Execution: %s should execute onprem with execution environment \"%s\". Workflow: %s", workflowExecution.ExecutionId, environment, workflowExecution.Workflow.ID) + + executionRequest := shuffle.ExecutionRequest{ + ExecutionId: workflowExecution.ExecutionId, + WorkflowId: workflowExecution.Workflow.ID, + Authorization: workflowExecution.Authorization, + Environments: []string{environment}, + } + + err = shuffle.SetWorkflowQueue(ctx, executionRequest, environment) + if err != nil { + log.Printf("[ERROR] Failed adding execution to db: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + time.Sleep(2 * time.Second) + log.Printf("[INFO] Starting validation of execution %s", workflowExecution.ExecutionId) + + returnBytes := shuffle.HandleRetValidation(ctx, workflowExecution) + + resp.WriteHeader(200) + resp.Write(returnBytes) +} diff --git a/docker-compose.yml b/docker-compose.yml index 7cc35878..6b863b6e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,6 +31,9 @@ services: - ${SHUFFLE_FILE_LOCATION}:/shuffle-files #- ${SHUFFLE_OPENSEARCH_CERTIFICATE_FILE}:/shuffle-files/es_certificate env_file: .env + environment: + - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps + - SHUFFLE_FILE_LOCATION=/shuffle-files restart: unless-stopped depends_on: - opensearch @@ -46,7 +49,7 @@ services: - /var/run/docker.sock:/var/run/docker.sock environment: - SHUFFLE_APP_SDK_VERSION=0.8.97 - - SHUFFLE_WORKER_VERSION=nightly + - SHUFFLE_WORKER_VERSION=0.8.97 - ORG_ID=${ORG_ID} - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT} diff --git a/frontend/package.json b/frontend/package.json index 7f872271..5b290522 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -4,7 +4,6 @@ "version": "0.8.92", "private": true, "dependencies": { - "@babel/helper-regex": "^7.10.5", "@material-ui/core": "^4.5.2", "@material-ui/data-grid": "^4.0.0-alpha.22", "@material-ui/icons": "^4.5.1", @@ -12,17 +11,15 @@ "@material-ui/styles": "^4.5.2", "@use-it/interval": "^1.0.0", "babel-eslint": "^10.1.0", - "cache-base": "^4.0.0", "class-transformer": "^0.3.1", "create-react-app": "^2.0.3", - "cytoscape": "^3.19.0", + "cytoscape": "^3.11.0", "cytoscape-clipboard": "^2.2.1", "cytoscape-cxtmenu": "^3.1.1", "cytoscape-edgehandles": "^3.6.0", "cytoscape-grid-guide": "~2.1.2", "cytoscape-node-html-label": "^1.1.5", "cytoscape-panzoom": "^2.5.2", - "cytoscape-popper": "^2.0.0", "cytoscape-undo-redo": "^1.3.2", "d3": "~4.10.0", "dotenv": "^6.1.0", @@ -47,7 +44,7 @@ "react-cytoscapejs": "^1.2.0", "react-device-detect": "^1.9.10", "react-dom": "^16.14.0", - "react-draggable": "4.4.3", + "react-draggable": "^3.3.2", "react-dropzone": "^10.1.10", "react-ga": "^2.7.0", "react-iframe": "^1.8.0", @@ -59,7 +56,6 @@ "react-router": "^4.3.1", "react-router-dom": "^4.3.1", "react-scripts": "^4.0.1", - "react-scroll": "^1.8.2", "reactstrap": "^7.1.0", "shellwords": "^0.1.1", "simplebar": "^4.2.3", diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index 7905e069..7d8a37b8 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -1522,14 +1522,20 @@ const Admin = (props) => { const org_id = selectedOrganization.id var copyText = document.getElementById(elementName); if (copyText !== null && copyText !== undefined) { + const clipboard = navigator.clipboard + if (clipboard === undefined) { + alert.error("Can only copy over HTTPS (port 3443)") + return + } + navigator.clipboard.writeText(org_id) copyText.select(); copyText.setSelectionRange(0, 99999); /* For mobile devices */ - /* Copy the text inside the text field */ - document.execCommand("copy"); + /* Copy the text inside the text field */ + document.execCommand("copy"); - alert.info(org_id + " copied to clipboard") + alert.info(org_id + " copied to clipboard") } }}> @@ -1910,6 +1916,12 @@ const Admin = (props) => { const elementName = "copy_element_shuffle" var copyText = document.getElementById(elementName); if (copyText !== null && copyText !== undefined) { + const clipboard = navigator.clipboard + if (clipboard === undefined) { + alert.error("Can only copy over HTTPS (port 3443)") + return + } + navigator.clipboard.writeText(data.apikey) copyText.select(); copyText.setSelectionRange(0, 99999); /* For mobile devices */ @@ -2161,6 +2173,12 @@ const Admin = (props) => { const elementName = "copy_element_shuffle" var copyText = document.getElementById(elementName); if (copyText !== null && copyText !== undefined) { + const clipboard = navigator.clipboard + if (clipboard === undefined) { + alert.error("Can only copy over HTTPS (port 3443)") + return + } + navigator.clipboard.writeText(file.id) copyText.select(); copyText.setSelectionRange(0, 99999); /* For mobile devices */ diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index e27e04ef..5a7a8c79 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -2228,7 +2228,7 @@ const AngularWorkflow = (props) => { } var parents = getParents(dstdata) - console.log("PARENTS: ", parents) + //console.log("PARENTS: ", parents) if (parents.length > 1) { for (var key in parents) { const item = parents[key] @@ -2272,8 +2272,9 @@ const AngularWorkflow = (props) => { // Checks for errors in edges when they're added const onEdgeAdded = (event) => { - setLastSaved(false) const edge = event.target.data() + console.log("EDGE ADDED: ", edge) + //setLastSaved(false) var targetnode = workflow.triggers.findIndex(data => data.id === edge.target) if (targetnode !== -1) { console.log("TARGETNODE: ", targetnode) @@ -2892,6 +2893,9 @@ const AngularWorkflow = (props) => { setEstablished(true) // Validate if the node is just a node lol + console.log("CY: ", cy) + //console.log("CY: ", cy.edgehandles()) + //try { cy.edgehandles({ handleNodes: (el) => el.isNode() && !el.data("isButton") && !el.data("isDescriptor"), preview: false, @@ -2900,6 +2904,9 @@ const AngularWorkflow = (props) => { return false; }, }) + //} catch (e) { + // console.log("Error in edgehandles: ", e) + //} cy.fit(null, 200) @@ -6276,6 +6283,12 @@ const AngularWorkflow = (props) => { var copyText = document.getElementById("webhook_uri_field") if (copyText !== undefined && copyText !== null) { console.log("NAVIGATOR: ", navigator) + const clipboard = navigator.clipboard + if (clipboard === undefined) { + alert.error("Can only copy over HTTPS (port 3443)") + return + } + navigator.clipboard.writeText(copyText.value) copyText.select() copyText.setSelectionRange(0, 99999) /* For mobile devices */ @@ -7510,6 +7523,12 @@ const AngularWorkflow = (props) => { console.log("NEW: ", copy) console.log("NAVIGATOR: ", navigator) + const clipboard = navigator.clipboard + if (clipboard === undefined) { + alert.error("Can only copy over HTTPS (port 3443)") + return + } + navigator.clipboard.writeText(JSON.stringify(copy)) copyText.select() copyText.setSelectionRange(0, 99999) /* For mobile devices */ @@ -7560,6 +7579,12 @@ const AngularWorkflow = (props) => { var copyText = document.getElementById(elementName) if (copyText !== null && copyText !== undefined) { console.log("NAVIGATOR: ", navigator) + const clipboard = navigator.clipboard + if (clipboard === undefined) { + alert.error("Can only copy over HTTPS (port 3443)") + return + } + navigator.clipboard.writeText(to_be_copied) copyText.select() copyText.setSelectionRange(0, 99999); /* For mobile devices */ @@ -8093,7 +8118,14 @@ const AngularWorkflow = (props) => { if (copyText !== null && copyText !== undefined) { console.log("COPY: ", copyText) console.log("NAVIGATOR: ", navigator) + const clipboard = navigator.clipboard + if (clipboard === undefined) { + alert.error("Can only copy over HTTPS (port 3443)") + return + } + navigator.clipboard.writeText(to_be_copied) + copyText.select(); copyText.setSelectionRange(0, 99999); /* For mobile devices */ diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index e12fb05b..f2e759f3 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -1,10 +1,12 @@ import React, {useState, useEffect} from 'react'; import { makeStyles } from '@material-ui/styles'; +import { useTheme } from '@material-ui/core/styles'; import {BrowserView, MobileView} from "react-device-detect"; import {Paper, Typography, FormControlLabel, Button, Divider, Select, MenuItem, FormControl, Switch, Dialog, DialogTitle, DialogContent, DialogActions, TextField, Tooltip, Breadcrumbs, CircularProgress, Chip} from '@material-ui/core'; -import {FileCopy as FileCopyIcon, Delete as DeleteIcon, Remove as RemoveIcon, Add as AddIcon, CheckCircle as CheckCircleIcon, AttachFile as AttachFileIcon, Apps as AppsIcon, ErrorOutline as ErrorOutlineIcon} from '@material-ui/icons'; +import {LockOpen as LockOpenIcon, FileCopy as FileCopyIcon, Delete as DeleteIcon, Remove as RemoveIcon, Add as AddIcon, CheckCircle as CheckCircleIcon, AttachFile as AttachFileIcon, Apps as AppsIcon, ErrorOutline as ErrorOutlineIcon} from '@material-ui/icons'; +import uuid from "uuid"; import {Link} from 'react-router-dom'; import YAML from 'yaml' import ChipInput from 'material-ui-chip-input' @@ -192,6 +194,7 @@ const AppCreator = (props) => { const { globalUrl, isLoaded } = props; const classes = useStyles(); const alert = useAlert() + const theme = useTheme(); var upload = "" const increaseAmount = 50 @@ -237,6 +240,12 @@ const AppCreator = (props) => { } const [extraAuth, setExtraAuth] = useState([]) + + const [app, setApp] = useState({}) + const [appAuthentication, setAppAuthentication] = React.useState([]); + const [selectedAction, setSelectedAction] = useState({}) + const [authLoaded, setAuthLoaded] = useState(false) + //const [actions, setActions] = useState([{ // "name": "Get workflows", // "description": "Get workflows", @@ -2416,6 +2425,401 @@ const AppCreator = (props) => { />
+ const ParsedActionHandler = () => { + const passedOrg = {"id": "", "name": ""} + const owner = "" + const passedTags = ["single test"] + + const [, setUpdate] = useState() + const [authenticationModalOpen, setAuthenticationModalOpen] = useState(false); + const [selectedApp, setSelectedApp] = useState({ + versions: [{ + "id": selectedAction.app_id, + "version": selectedAction.app_version, + }], + loop_versions: [selectedAction.app_version], + id: selectedAction.app_id, + name: selectedAction.app_name, + version: selectedAction.app_version, + }) + + const [requiresAuthentication, setRequiresAuthentication] = useState(app.authentication.required && app.authentication.parameters !== undefined && app.authentication.parameters !== null) + const [workflow, setWorkflow] = useState({ + name: "", + description: "", + actions: [selectedAction], + start: selectedAction.id, + tags: passedTags, + execution_org: passedOrg, + org_id: passedOrg.id, + id: uuid.v4(), + isValid: true, + owner: owner, + created: Date.now(), + }) + + const EndpointData = () => { + const [tmpVar, setTmpVar] = React.useState("") + + return ( +
+ The API endpoint to use (URL) - predefined in the app + { + setTmpVar(event.target.value) + }} + onBlur={() => { + selectedApp.link = tmpVar + console.log("LINK: ", selectedApp.link) + setSelectedApp(selectedApp) + }} + /> +
+ ) + } + + const setAppActionAuthentication = (newauth) => { + if (app.authentication.required) { + var findAuthId = "" + if (selectedAction.authentication_id !== null && selectedAction.authentication_id !== undefined && selectedAction.authentication_id.length > 0) { + findAuthId = selectedAction.authentication_id + } + + var baseAuthOptions = [] + for (var key in newauth) { + var item = newauth[key] + + const newfields = {} + for (var filterkey in item.fields) { + newfields[item.fields[filterkey].key] = item.fields[filterkey].value + } + + item.fields = newfields + if (item.app.name === app.name) { + baseAuthOptions.push(item) + + if (item.id === findAuthId) { + selectedAction.selectedAuthentication = item + } + } + } + + selectedAction.authentication = baseAuthOptions + //console.log("Authentication: ", authenticationOptions) + if (selectedAction.selectedAuthentication === null || selectedAction.selectedAuthentication === undefined || selectedAction.selectedAuthentication.length === "") { + selectedAction.selectedAuthentication = {} + } + } else { + selectedAction.authentication = [] + selectedAction.authentication_id = "" + selectedAction.selectedAuthentication = {} + } + + setSelectedAction(selectedAction) + console.log(selectedAction) + } + + //{selectedAction.authentication !== undefined && selectedAction.authentication.length > 0 ? + const getAppAuthentication = () => { + fetch(globalUrl+"/api/v1/apps/authentication", { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for apps :O!") + } + + return response.json() + }) + .then((responseJson) => { + if (responseJson.success && responseJson.data !== undefined && responseJson.data !== null && responseJson.data.length !== 0) { + var newauth = [] + for (var key in responseJson.data) { + if (responseJson.data[key].defined === false) { + continue + } + + newauth.push(responseJson.data[key]) + } + + setAppAuthentication(newauth) + setAppActionAuthentication(newauth) + } else { + if (app.authentication.required) { + const tmpParams = selectedAction.parameters + selectedAction.parameters = [] + + for (var paramkey in app.authentication.parameters) { + var item = app.authentication.parameters[paramkey] + item.configuration = true + + const found = selectedAction.parameters.find(param => param.name === item.name) + if (found === null || found === undefined) { + selectedAction.parameters.push(item) + } + } + + for (var paramkey in tmpParams) { + var item = tmpParams[paramkey] + //item.configuration = true + + const found = selectedAction.parameters.find(param => param.name === item.name) + if (found === null || found === undefined) { + selectedAction.parameters.push(item) + } + } + + setSelectedAction(selectedAction) + } + + //alert.error("Failed getting authentications") + } + }) + .catch(error => { + alert.error("Auth loading error: "+error.toString()) + }) + } + + if (!authLoaded && appAuthentication.length === 0 && selectedAction.id !== undefined) { + setAuthLoaded(true) + getAppAuthentication() + } else if (selectedAction.id === undefined && currentAction.name !== undefined && currentAction.name !== null && currentAction.name.length > 0) { + var methodName = `${currentAction.method}_${currentAction.name}` + if (currentAction.method.toLowerCase() === "custom" || currentAction.name.toLowerCase().startsWith(currentAction.method.toLowerCase())) { + methodName = currentAction.name + } + + methodName = methodName.toLowerCase().replaceAll(" ", "_") + if (app.actions !== null && app.actions !== undefined) { + var newselectedaction = app.actions.find(item => item.name.toLowerCase().replaceAll(" ", "_") === methodName) + if (newselectedaction !== undefined && newselectedaction !== null) { + newselectedaction.app_id = app.id + newselectedaction.app_name = app.name + newselectedaction.app_version = app.app_version + newselectedaction.authentication = [] + newselectedaction.authentication_id = "" + newselectedaction.selectedAuthentication = {} + setSelectedAction(newselectedaction) + } + } + } + + const setNewAppAuth = (appAuthData) => { + //console.log("DAta: ", appAuthData) + fetch(globalUrl+"/api/v1/apps/authentication", { + method: 'PUT', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + body: JSON.stringify(appAuthData), + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for setting app auth :O!") + } + + return response.json() + }) + .then((responseJson) => { + if (!responseJson.success) { + alert.error("Failed to set app auth: "+responseJson.reason) + } else { + getAppAuthentication(true) + setAuthenticationModalOpen(false) + + // Needs a refresh with the new authentication.. + //alert.success("Successfully saved new app auth") + } + }) + .catch(error => { + alert.error(error.toString()) + }) + } + + const AuthenticationData = (props) => { + const selectedApp = props.app + console.log("APP: ", selectedApp) + + const [authenticationOption, setAuthenticationOptions] = React.useState({ + app: JSON.parse(JSON.stringify(selectedApp)), + fields: {}, + label: "", + usage: [{ + workflow_id: workflow.id, + }], + id: uuid.v4(), + active: true, + }) + + if (selectedApp.authentication === undefined) { + return null + } + + if (selectedApp.authentication.parameters === null || selectedApp.authentication.parameters === undefined || selectedApp.authentication.parameters.length === 0) { + return null + } + + authenticationOption.app.actions = [] + + for (var key in selectedApp.authentication.parameters) { + if (authenticationOption.fields[selectedApp.authentication.parameters[key].name] === undefined) { + authenticationOption.fields[selectedApp.authentication.parameters[key].name] = "" + } + } + + const handleSubmitCheck = () => { + console.log("NEW AUTH: ", authenticationOption) + if (authenticationOption.label.length === 0) { + authenticationOption.label = `Auth for ${selectedApp.name}` + //alert.info("Label can't be empty") + //return + } + + for (var key in selectedApp.authentication.parameters) { + if (authenticationOption.fields[selectedApp.authentication.parameters[key].name].length === 0) { + alert.info("Field "+selectedApp.authentication.parameters[key].name+" can't be empty") + return + } + } + + console.log("Action: ", selectedAction) + selectedAction.authentication_id = authenticationOption.id + selectedAction.selectedAuthentication = authenticationOption + if (selectedAction.authentication === undefined || selectedAction.authentication === null) { + selectedAction.authentication = [authenticationOption] + } else { + selectedAction.authentication.push(authenticationOption) + } + + setSelectedAction(selectedAction) + + var newAuthOption = JSON.parse(JSON.stringify(authenticationOption)) + var newFields = [] + for (const key in newAuthOption.fields) { + const value = newAuthOption.fields[key] + newFields.push({ + key: key, + value: value, + }) + } + + console.log("FIELDS: ", newFields) + newAuthOption.fields = newFields + setNewAppAuth(newAuthOption) + //appAuthentication.push(newAuthOption) + //setAppAuthentication(appAuthentication) + // + + setUpdate(authenticationOption.id) + + /* + {selectedAction.authentication.map(data => ( + + */ + + } + + if (authenticationOption.label === null || authenticationOption.label === undefined) { + authenticationOption.label = selectedApp.name+" authentication" + } + + return ( +
+ + What is this?
+ These are required fields for authenticating with {selectedApp.name} +
+ Name - what is this used for? + { + authenticationOption.label = event.target.value + }} + /> + {selectedApp.link.length > 0 ?
: null} + +
+ {selectedApp.authentication.parameters.map((data, index) => { + return ( +
+ + {data.name} + { + authenticationOption.fields[data.name] = event.target.value + }} + /> +
+ ) + })} + + + + + +
+ ) + } + } + const actionView =
@@ -2628,6 +3032,7 @@ const AppCreator = (props) => { Continue + : null; From 24fd96fcf00cf4e775bb4082d527e1f04d91fe49 Mon Sep 17 00:00:00 2001 From: frikky Date: Wed, 2 Jun 2021 23:39:20 +0200 Subject: [PATCH 73/96] Added single app testing from GUI --- backend/app_sdk/app_base.py | 12 ++++----- backend/app_sdk/build.sh | 2 +- backend/go-app/main.go | 32 +++++++++++------------- backend/go-app/walkoff.go | 12 ++++----- frontend/src/App.jsx | 2 +- frontend/src/components/ParsedAction.jsx | 25 ++++++++++++++++++ frontend/src/views/Admin.jsx | 21 ++++++++-------- frontend/src/views/AngularWorkflow.jsx | 22 ++++++++-------- frontend/src/views/AppCreator.jsx | 7 +++--- frontend/src/views/Workflows.jsx | 6 +++-- 10 files changed, 83 insertions(+), 58 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 99911dfa..937e2309 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -1967,12 +1967,12 @@ class AppBase: params[parameter["name"]] = resultarray multi_parameters[parameter["name"]] = resultarray - if len(resultarray) == 0: - print("[WARNING] Returning empty array because the array length to be looped is 0 (1)") - action_result["status"] = "SUCCESS" - action_result["result"] = "[]" - self.send_result(action_result, headers, stream_path) - return + #if len(resultarray) == 0: + # print("[WARNING] Returning empty array because the array length to be looped is 0 (1)") + # action_result["status"] = "SUCCESS" + # action_result["result"] = "[]" + # self.send_result(action_result, headers, stream_path) + # return multi_execution_lists.append(new_replacement) #print("MULTI finished: %s" % json_replacement) diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index c6f4e7a9..14b549de 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,6 +1,6 @@ #!/bin/bash NAME=shuffle-app_sdk -VERSION=0.8.97 +VERSION=0.8.98 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/go-app/main.go b/backend/go-app/main.go index bcf2abac..f7a762c7 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3177,7 +3177,7 @@ func handleSwaggerValidation(body []byte) (shuffle.ParsedOpenApi, error) { //log.Printf("Json err: %s", err) err = yaml.Unmarshal(body, &version) if err != nil { - log.Printf("Yaml error (1): %s", err) + log.Printf("[WARNING] Yaml error (1): %s", err) } else { //log.Printf("Successfully parsed YAML!") } @@ -3220,7 +3220,7 @@ func handleSwaggerValidation(body []byte) (shuffle.ParsedOpenApi, error) { //log.Printf("Json error? %s", err) err = yaml.Unmarshal(body, &swagger) if err != nil { - log.Printf("Yaml error (2): %s", err) + log.Printf("[WARNING] Yaml error (2): %s", err) return shuffle.ParsedOpenApi{}, err } else { //log.Printf("Valid yaml!") @@ -4181,14 +4181,14 @@ func runInitEs(ctx context.Context) { for _, org := range activeOrgs { if !org.CloudSync { - log.Printf("Skipping org %s because sync isn't set (1).", org.Id) + log.Printf("[WARNING] Skipping org %s because sync isn't set (1).", org.Id) continue } //interval := int(org.SyncConfig.Interval) interval := 15 if interval == 0 { - log.Printf("Skipping org %s because sync isn't set (0).", org.Id) + log.Printf("[WARNING] Skipping org %s because sync isn't set (0).", org.Id) continue } @@ -4204,7 +4204,7 @@ func runInitEs(ctx context.Context) { if err != nil { log.Printf("[CRITICAL] Failed to schedule org: %s", err) } else { - log.Printf("Started sync on interval %d for org %s", interval, org.Name) + log.Printf("[INFO] Started sync on interval %d for org %s (%s)", interval, org.Name, org.Id) scheduledOrgs[org.Id] = jobret } } @@ -5035,7 +5035,7 @@ func handleStopCloudSync(syncUrl string, org shuffle.Org) (*shuffle.Org, error) return &org, errors.New(fmt.Sprintf("Couldn't find any sync key to disable org %s", org.Id)) } - log.Printf("Should run cloud sync disable for org %s with URL %s and sync key %s", org.Id, syncUrl, org.SyncConfig.Apikey) + log.Printf("[INFO] Should run cloud sync disable for org %s with URL %s and sync key %s", org.Id, syncUrl, org.SyncConfig.Apikey) client := &http.Client{} req, err := http.NewRequest( @@ -5071,7 +5071,7 @@ func handleStopCloudSync(syncUrl string, org shuffle.Org) (*shuffle.Org, error) return &org, errors.New(responseData.Reason) } - log.Printf("Everything is success. Should disable org sync for %s", org.Id) + log.Printf("[INFO] Everything is success. Should disable org sync for %s", org.Id) ctx := context.Background() org.CloudSync = false @@ -5080,15 +5080,14 @@ func handleStopCloudSync(syncUrl string, org shuffle.Org) (*shuffle.Org, error) err = shuffle.SetOrg(ctx, org, org.Id) if err != nil { - newerror := fmt.Sprintf("ERROR: Failed updating even though there was success: %s", err) + newerror := fmt.Sprintf("[WARNING] ERROR: Failed updating even though there was success: %s", err) log.Printf(newerror) return &org, errors.New(newerror) } - var environments []shuffle.Environment - q := datastore.NewQuery("Environments").Filter("org_id =", org.Id) - _, err = dbclient.GetAll(ctx, q, &environments) + environments, err := shuffle.GetEnvironments(ctx, org.Id) if err != nil { + log.Printf("[WARNING] Failed getting envs in stop sync: %s", err) return &org, err } @@ -5237,9 +5236,9 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { // Everything below here is to SET UP CLOUD SYNC. // If you want to disable cloud sync, see previous section. if org.CloudSync { - log.Printf("Org %s is already syncing. Skip", org.Id) + log.Printf("[WARNING] Org %s is already syncing. Skip", org.Id) resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Org is already syncing. Nothing to set up."}`))) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Your org is already syncing. Nothing to set up."}`))) return } @@ -5332,18 +5331,15 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { scheduledOrgs[org.Id] = jobret } - // FIXME: Add this for every feature + // ONLY checked added if workflows are allow huh if org.SyncFeatures.Workflows.Active { log.Printf("[INFO] Should activate cloud workflows for org %s!", org.Id) // 1. Find environment // 2. If cloud env found, enable it (un-archive) // 3. If it doesn't create it - - //var environments []shuffle.Environment - //q := datastore.NewQuery("Environments").Filter("org_id =", org.Id) - //_, err = dbclient.GetAll(ctx, q, &environments) environments, err := shuffle.GetEnvironments(ctx, org.Id) + log.Printf("GETTING ENVS: %#s", environments) if err == nil { // Don't disable, this will be deleted entirely diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index a0614772..6c91d2ec 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -1179,10 +1179,10 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { } } - err = increaseStatisticsField(ctx, "total_workflow_triggers", workflow.ID, -1, workflow.OrgId) - if err != nil { - log.Printf("Failed to increase total workflows: %s", err) - } + //err = increaseStatisticsField(ctx, "total_workflow_triggers", workflow.ID, -1, workflow.OrgId) + //if err != nil { + // log.Printf("Failed to increase total workflows: %s", err) + //} } // FIXME - maybe delete workflow executions @@ -3408,14 +3408,14 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, readFile, err := ioutil.ReadAll(fileReader) if err != nil { - log.Printf("Filereader error yaml for %s: %s", filename, err) + log.Printf("[WARNING] Filereader error yaml for %s: %s", filename, err) continue } // 1. This parses OpenAPI v2 to v3 etc, for use. parsedOpenApi, err := handleSwaggerValidation(readFile) if err != nil { - log.Printf("Validation error for %s: %s", filename, err) + log.Printf("[WARNING] Validation error for %s: %s", filename, err) continue } diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 35422c5f..70df8462 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -115,8 +115,8 @@ const App = (message, props) => { } /> } /> } /> - } /> } /> + } /> } /> } /> } /> diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index b739a2f1..9794d0f6 100644 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -50,6 +50,7 @@ const ParsedAction = (props) => { const theme = useTheme(); const classes = useStyles() + const keywords = ["len(", "lower(", "upper(", "trim(", "split(", "length(", "number(", "parse(", "join("] const getParents = (action) => { if (cy === undefined) { return [] @@ -757,6 +758,23 @@ const ParsedAction = (props) => { }} /> + console.log("FIELD VALUE: ", data.value) + //const regexp = new RegExp("\W+\.", "g") + //let match + //while ((match = regexp.exec(data.value)) !== null) { + // console.log(`Found ${match[0]} start=${match.index} end=${regexp.lastIndex}.`); + //} + + //const str = = data.value.search(submatch) + //console.log("FOUND? ", n) + for (var key in keywords) { + const keyword = keywords[key] + if (data.value.includes(keyword)) { + console.log("INCLUDED: ", keyword) + } + } + + //const keywords = ["len", "lower", "upper", "trim", "split", "length", "number", "parse", "join"] if (selectedActionParameters[count].schema !== undefined && selectedActionParameters[count].schema !== null && selectedActionParameters[count].schema.type === "file") { datafield = { return } + console.log("AUTOCOMPLETE1: ", values) + var toComplete = selectedActionParameters[count].value.trim().endsWith("$") ? values[0].autocomplete : "$"+values[0].autocomplete + toComplete = toComplete.toLowerCase().replaceAll(" ", "_") + console.log("AUTOCOMPLETE: ", toComplete) for (var key in values) { if (key == 0 || values[key].autocomplete.length === 0) { continue @@ -1353,6 +1375,9 @@ const ParsedAction = (props) => { } } } + + console.log("DID REPLACE ACTUALLY WORK?? - Something is buggy.") + setWorkflow(workflow) }} /> diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index 7d8a37b8..0e629790 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -154,7 +154,7 @@ const Admin = (props) => { // Need to wait because query in ES is too fast setTimeout(() => { getAppAuthentication() - }, 1000) + }, 500) alert.success("Successfully deleted authentication!") } }), @@ -228,7 +228,7 @@ const Admin = (props) => { .then((responseJson) => { setTimeout(() => { handleGetOrg(org_id) - }, 1000) + }, 500) }) .catch(error => { alert.error("Err: " + error.toString()) @@ -264,12 +264,13 @@ const Admin = (props) => { console.log("Cloud sync fail?") } - setTimeout(() => { - return response.json() - }, 1000) + return response.json() + //setTimeout(() => { + //}, 1000) }) .then((responseJson) => { - if (!responseJson.success && responseJson.reason !== undefined) { + console.log("RESP: ", responseJson) + if (responseJson.success === false && responseJson.reason !== undefined) { setOrgSyncResponse(responseJson.reason) alert.error("Failed to handle sync: "+responseJson.reason) } else if (!responseJson.success) { @@ -355,7 +356,7 @@ const Admin = (props) => { setSelectedUserModalOpen(false) setTimeout(() => { getAppAuthentication() - }, 1000) + }, 500) } }), ) @@ -513,7 +514,7 @@ const Admin = (props) => { setModalOpen(false) setTimeout(() => { getUsers() - }, 1000) + }, 500) } }), ) @@ -548,7 +549,7 @@ const Admin = (props) => { setModalOpen(false) setTimeout(() => { getUsers() - }, 1000) + }, 500) } }), ) @@ -1649,7 +1650,7 @@ const Admin = (props) => {
{orgSyncResponse.length > 0 ? - Message from Shuffle: {orgSyncResponse} + Message from Shuffle Cloud: {orgSyncResponse} : null } diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 5a7a8c79..73cc80ab 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1571,17 +1571,13 @@ const AngularWorkflow = (props) => { } } - console.log("ID ETC: ", idnumber, paramname) if (idnumber >= 0 && paramname.length > 0) { const exampledata = GetExampleResult(nodedata) const parsedname = paramname.toLowerCase().trim().replaceAll("_", " ") - console.log("EX: ", exampledata) - console.log("NAME: ", parsedname) const foundresult = GetParamMatch(parsedname, exampledata, "") - console.log("RESULT: ", foundresult) if (foundresult.length > 0) { - console.log("FOUND: ", paramname, foundresult) + console.log("FOUND RESULT: ", paramname, foundresult) newValue = `${newValue}${foundresult}` } @@ -2100,9 +2096,13 @@ const AngularWorkflow = (props) => { return "" } + if (exampledata === null) { + return "" + } + // Basically just a stupid if-else :) const synonyms = { - "id": ["id", "ref", "sourceref", "reference", "sourcereference", "alert id", "case id", "incident id", "service id",], + "id": ["id", "ref", "sourceref", "reference", "sourcereference", "alert id", "case id", "incident id", "service id", "sid", "uid", "uuid"], "title": ["title", "name", "message"], "description": ["description", "explanation", "story", "details",], "email": ["mail", "email", "sender", "receiver", "recipient"], @@ -7535,7 +7535,7 @@ const AngularWorkflow = (props) => { /* Copy the text inside the text field */ document.execCommand("copy") - alert.success("Copied data") + //alert.success("Copied data") } else { console.log("Failed to copy from "+elementName+": ", copyText) } @@ -7544,7 +7544,7 @@ const AngularWorkflow = (props) => { const HandleJsonCopy = (base, copy, base_node_name) => { console.log("COPY: ", copy) var newitem = JSON.parse(base) - to_be_copied = "$"+base_node_name + to_be_copied = "$"+base_node_name.toLowerCase().replaceAll(" ", "_") for (var key in copy.namespace) { if (copy.namespace[key].includes("Results for")) { continue @@ -7591,7 +7591,7 @@ const AngularWorkflow = (props) => { /* Copy the text inside the text field */ document.execCommand("copy"); - alert.success("Copied "+to_be_copied) + //alert.success("Copied "+to_be_copied) console.log("COPYING!") } else { console.log("Couldn't find element ", elementName) @@ -8097,7 +8097,7 @@ const AngularWorkflow = (props) => { {validate.valid ? { handleReactJsonClipboard(copy) @@ -8131,7 +8131,7 @@ const AngularWorkflow = (props) => { /* Copy the text inside the text field */ document.execCommand("copy"); - alert.success("Copied "+to_be_copied) + //alert.success("Copied "+to_be_copied) } else { console.log("Failed to copy. copy_element_shuffle is undefined") } diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index f2e759f3..21e9859d 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -2443,7 +2443,7 @@ const AppCreator = (props) => { version: selectedAction.app_version, }) - const [requiresAuthentication, setRequiresAuthentication] = useState(app.authentication.required && app.authentication.parameters !== undefined && app.authentication.parameters !== null) + const [requiresAuthentication, setRequiresAuthentication] = useState(app.authentication !== null && app.authentication !== undefined && app.authentication.required && app.authentication.parameters !== undefined && app.authentication.parameters !== null ? true : false) const [workflow, setWorkflow] = useState({ name: "", description: "", @@ -2699,7 +2699,7 @@ const AppCreator = (props) => { for (var key in selectedApp.authentication.parameters) { if (authenticationOption.fields[selectedApp.authentication.parameters[key].name].length === 0) { alert.info("Field "+selectedApp.authentication.parameters[key].name+" can't be empty") - return + return null } } @@ -2744,6 +2744,7 @@ const AppCreator = (props) => { authenticationOption.label = selectedApp.name+" authentication" } + console.log("PRE RETURN") return (
@@ -3032,7 +3033,7 @@ const AppCreator = (props) => { Continue - + {/**/} : null; diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index e215a9c7..4cbba6fa 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -531,7 +531,7 @@ const Workflows = (props) => { deleteWorkflow(selectedWorkflowId) setTimeout(() => { getAvailableWorkflows() - }, 1000) + }, 500) } setDeleteModalOpen(false) }} color="primary"> @@ -1489,7 +1489,9 @@ const Workflows = (props) => { window.location.pathname = "/workflows/"+responseJson["id"] } else if (!redirect) { // Update :) - getAvailableWorkflows() + setTimeout(() => { + getAvailableWorkflows() + }, 500) setImportLoading(false) } else { alert.info("Successfully changed basic info for workflow") From f594f8f3152c0e05083806479fb9417591515f9a Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 4 Jun 2021 03:22:04 +0200 Subject: [PATCH 74/96] Updated install guide for opensearch --- .env | 1 + .github/install-guide.md | 66 +++++++++++--------------- backend/go-app/main.go | 15 ++++++ docker-compose.yml | 8 ++-- frontend/src/views/AngularWorkflow.jsx | 10 ++++ 5 files changed, 56 insertions(+), 44 deletions(-) diff --git a/.env b/.env index d4418b9a..4874bd40 100644 --- a/.env +++ b/.env @@ -57,4 +57,5 @@ SHUFFLE_OPENSEARCH_PASSWORD= SHUFFLE_OPENSEARCH_CERTIFICATE_FILE= SHUFFLE_OPENSEARCH_APIKEY= SHUFFLE_OPENSEARCH_CLOUDID= +SHUFFLE_OPENSEARCH_PROXY= SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY=true diff --git a/.github/install-guide.md b/.github/install-guide.md index d1c21169..4e076078 100644 --- a/.github/install-guide.md +++ b/.github/install-guide.md @@ -6,11 +6,21 @@ The Docker setup is done with docker-compose and is a single command to get set **PS: if you're setting up Shuffle on Windows, go to the next step (Windows Docker setup)** -1. Make sure you have Docker and [docker-compose](https://docs.docker.com/compose/install/) installed. -2. Run docker-compose. +1. Make sure you have [Docker](https://docs.docker.com/get-docker/) and [docker-compose](https://docs.docker.com/compose/install/) installed. +2. Download Shuffle ``` git clone https://github.com/frikky/Shuffle cd Shuffle +``` + +3. Fix prerequisites for the Opensearch database (Elasticsearch): +``` +sudo sysctl -w vm.max_map_count=262144 # https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html +sudo chown 1000:1000 -R shuffle-database # Requires for Opensearch +``` + +4. Run docker-compose. +``` docker-compose up -d ``` @@ -26,7 +36,18 @@ This step is for setting up with Docker on windows from scratch. ``` OUTER_HOSTNAME=YOUR.IP.HERE ``` -5. Run docker-compose + +5. Configure max memory (WSL) by opening a new CMD/Powershell window. Required for Elasticsearch +``` +wsl -d docker-desktop +sysctl -w vm.max_map_count=262144 +echo "vm.max_map_count = 262144" > /etc/sysctl.d/99-docker-desktop.conf +echo -e "\nvm.max_map_count = 262144\n" >> /etc/sysctl.d/00-alpine.conf + +# https://stackoverflow.com/questions/42111566/elasticsearch-in-windows-docker-image-vm-max-map-count +``` + +6. Run docker-compose ``` docker-compose up -d ``` @@ -35,7 +56,7 @@ docker-compose up -d https://shuffler.io/docs/configuration ### After installation -1. After installation, go to http://localhost:3001/adminsetup (or your servername) +1. After installation, go to http://localhost:3001/adminsetup (or your servername - https is on port 3443) 2. Now set up your admin account (username & password). Shuffle doesn't have a default username and password. 3. Check out https://shuffler.io/docs/configuration as it has a lot of useful information to get started @@ -44,43 +65,10 @@ https://shuffler.io/docs/configuration ### Useful info * Check out [getting started](https://shuffler.io/docs/getting_started) +* The default state of Shuffle is NOT scalable. See [production setup](https://shuffler.io/docs/configuration#production_readiness) for more info * The server is available on http://localhost:3001 (or your servername) * Further configurations can be done in docker-compose.yml and .env. -* Default database location is /etc/shuffle - -### Execution problems -If you have problems with your first execution (hello world), you might need to set the correct Docker API version. Here's how: - -1. Find your API version by running "docker version" -``` -$ docker version - -Client: - Version: 17.09.1-ce - API version: 1.32 # <-- this one - Go version: go1.8.3 - Git commit: 19e2cf6 - Built: Thu Dec 7 22:24:16 2017 - OS/Arch: linux/amd64 - -Server: - Version: 17.09.1-ce - API version: 1.32 (minimum version 1.12) - Go version: go1.8.3 - Git commit: 19e2cf6 - Built: Thu Dec 7 22:22:56 2017 - OS/Arch: linux/amd64 - Experimental: false -``` - -2. Open docker-compose.yml and change the line with "DOCKER_API_VERSION" to your version. -3. Restart docker-compose -``` -docker-compose down -docker-compose up -``` - -Related issue: #47 +* Default database location is in the same folder: ./shuffle-database # Local development installation Local development is pretty straight forward with **ReactJS** and **Golang**. This part is intended to help you run the code for development purposes. diff --git a/backend/go-app/main.go b/backend/go-app/main.go index f7a762c7..c859412a 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5675,9 +5675,24 @@ func initHandlers() { transport := http.DefaultTransport.(*http.Transport).Clone() transport.MaxIdleConnsPerHost = 1000 transport.ResponseHeaderTimeout = time.Second * 10 + transport.Proxy = nil + + if len(os.Getenv("SHUFFLE_OPENSEARCH_PROXY")) > 0 { + httpProxy := os.Getenv("SHUFFLE_OPENSEARCH_PROXY") + + url_i := url.URL{} + url_proxy, err := url_i.Parse(httpProxy) + if err == nil { + log.Printf("[DEBUG] Setting Opensearch proxy to %s", httpProxy) + transport.Proxy = http.ProxyURL(url_proxy) + } else { + log.PrintF("[ERROR] Failed setting proxy for %s", httpProxy) + } + } skipSSLVerify := false if strings.ToLower(os.Getenv("SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY")) == "true" { + log.Printf("[DEBUG] SKIPPING SSL verification with Opensearch") skipSSLVerify = true } diff --git a/docker-compose.yml b/docker-compose.yml index 6b863b6e..d7fa3eb0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,6 +34,8 @@ services: environment: - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps - SHUFFLE_FILE_LOCATION=/shuffle-files + - HTTP_PROXY=${SHUFFLE_HTTP_PROXY} + - HTTPS_PROXY=${SHUFFLE_HTTPS_PROXY} restart: unless-stopped depends_on: - opensearch @@ -70,14 +72,10 @@ services: hostname: shuffle-opensearch container_name: shuffle-opensearch environment: - - cluster.name=shuffle-cluster - - node.name=shuffle-opensearch - - discovery.seed_hosts=shuffle-opensearch - - cluster.initial_master_nodes=shuffle-opensearch - bootstrap.memory_lock=true - "OPENSEARCH_JAVA_OPTS=-Xms1024m -Xmx1024m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM - - cluster.routing.allocation.disk.threshold_enabled=false - opendistro_security.disabled=true + - discovery.seed_hosts=shuffle-opensearch ulimits: memlock: soft: -1 diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 73cc80ab..5cc65caa 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -4416,7 +4416,17 @@ const AngularWorkflow = (props) => { //console.log("OLDNAME: ", selectedAction.name) event.target.value = event.target.value.replaceAll("(", "") event.target.value = event.target.value.replaceAll(")", "") + event.target.value = event.target.value.replaceAll("]", "") + event.target.value = event.target.value.replaceAll("[", "") + event.target.value = event.target.value.replaceAll("{", "") + event.target.value = event.target.value.replaceAll("}", "") + event.target.value = event.target.value.replaceAll("*", "") + event.target.value = event.target.value.replaceAll("!", "") + event.target.value = event.target.value.replaceAll("@", "") + event.target.value = event.target.value.replaceAll("#", "") event.target.value = event.target.value.replaceAll("$", "") + event.target.value = event.target.value.replaceAll("%", "") + event.target.value = event.target.value.replaceAll("&", "") event.target.value = event.target.value.replaceAll("#", "") event.target.value = event.target.value.replaceAll(".", "") event.target.value = event.target.value.replaceAll(",", "") From 6b3e280caf042e5255d06e2d3f6e363f9a28b796 Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 4 Jun 2021 03:30:04 +0200 Subject: [PATCH 75/96] Updated install guide --- .github/install-guide.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/install-guide.md b/.github/install-guide.md index 4e076078..236103b3 100644 --- a/.github/install-guide.md +++ b/.github/install-guide.md @@ -1,7 +1,7 @@ # Installation guide Installation of Shuffle is currently only available in docker. Looking for how to update Shuffle? Check the [updating guide](https://shuffler.io/docs/configuration#updating_shuffle) -## Docker +## Docker - *nix OS The Docker setup is done with docker-compose and is a single command to get set up. **PS: if you're setting up Shuffle on Windows, go to the next step (Windows Docker setup)** @@ -57,9 +57,9 @@ https://shuffler.io/docs/configuration ### After installation 1. After installation, go to http://localhost:3001/adminsetup (or your servername - https is on port 3443) - -2. Now set up your admin account (username & password). Shuffle doesn't have a default username and password. -3. Check out https://shuffler.io/docs/configuration as it has a lot of useful information to get started +2. Now set up your admin account (username & password). Shuffle doesn't have a default username and password. +3. Sign in with the same Username & Password! Go to /apps and see if you have any apps yet. If not - you may need to [configure proxies](https://shuffler.io/docs/configuration#production_readiness) +4. Check out https://shuffler.io/docs/configuration as it has a lot of useful information to get started ![Admin account setup](https://github.com/frikky/Shuffle/blob/master/frontend/src/assets/img/shuffle_adminaccount.png) @@ -87,7 +87,7 @@ npm start ## Backend - Golang http://localhost:5001 - REST API - requires [>=go1.13](https://golang.org/dl/) ```bash -export DATASTORE_EMULATOR_HOST=0.0.0.0:8000 +export SHUFFLE_OPENSEARCH_URL="http://localhost:9200" cd backend/go-app go run *.go ``` @@ -108,7 +108,6 @@ cd functions/onprem/orborus go run orborus.go ``` - Environments (modify for Windows): ``` export ORG_ID=Shuffle From 08203730d295a067222cf0833064084dbf12a2a0 Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 4 Jun 2021 07:16:23 +0200 Subject: [PATCH 76/96] Fixed HTTP proxy things for env --- .env | 4 +-- .github/install-guide.md | 4 +-- backend/app_sdk/app_base.py | 54 +++++++++++++++++++++++++++++++ backend/go-app/go.mod | 4 +-- backend/go-app/go.sum | 2 ++ backend/go-app/main.go | 2 +- docker-compose.yml | 10 +++--- frontend/src/views/AppCreator.jsx | 9 +++--- 8 files changed, 71 insertions(+), 18 deletions(-) diff --git a/.env b/.env index 4874bd40..27cf372b 100644 --- a/.env +++ b/.env @@ -36,8 +36,8 @@ DB_LOCATION=./shuffle-database # Proxy configurations. SHUFFLE_PASS_WORKER_PROXY must be FALSE to not pass the proxy information to sub-apps. # PS: It will skip proxy for -SHUFFLE_HTTP_PROXY= -SHUFFLE_HTTPS_PROXY= +HTTP_PROXY= +HTTPS_PROXY= SHUFFLE_PASS_WORKER_PROXY=TRUE SHUFFLE_PASS_APP_PROXY=FALSE diff --git a/.github/install-guide.md b/.github/install-guide.md index 236103b3..165490b7 100644 --- a/.github/install-guide.md +++ b/.github/install-guide.md @@ -1,8 +1,8 @@ # Installation guide Installation of Shuffle is currently only available in docker. Looking for how to update Shuffle? Check the [updating guide](https://shuffler.io/docs/configuration#updating_shuffle) -## Docker - *nix OS -The Docker setup is done with docker-compose and is a single command to get set up. +# Docker - *nix +The Docker setup is done with docker-compose **PS: if you're setting up Shuffle on Windows, go to the next step (Windows Docker setup)** diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 937e2309..7c3b7ea2 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -682,6 +682,60 @@ class AppBase: else: return returns + def set_cache(self, key, value): + org_id = self.full_execution["workflow"]["execution_org"]["id"] + url = "%s/api/v1/orgs/%s/set_cache" % (self.url, org_id) + data = { + "workflow_id": self.full_execution["workflow"]["id"], + "execution_id": self.current_execution_id, + "authorization": self.authorization, + "org_id": org_id, + "key": key, + "value": str(value), + } + + response = requests.post(url, json=data) + try: + allvalues = response.json() + allvalues["key"] = key + allvalues["value"] = str(value) + return allvalues + except: + print("Value couldn't be parsed") + #return response.json() + return {"success": False} + + def get_cache(self, key): + org_id = self.full_execution["workflow"]["execution_org"]["id"] + url = "%s/api/v1/orgs/%s/get_cache" % (self.url, org_id) + data = { + "workflow_id": self.full_execution["workflow"]["id"], + "execution_id": self.current_execution_id, + "authorization": self.authorization, + "org_id": org_id, + "key": key, + } + + value = requests.post(url, json=data) + try: + allvalues = value.json() + print("VAL1: ", allvalues) + allvalues["key"] = key + print("VAL2: ", allvalues) + + try: + parsedvalue = json.loads(allvalues["value"]) + allvalues["value"] = parsedvalue + except: + print("Parsing of value as JSON failed") + return {"success": False} + + return allvalues + except: + print("Value couldn't be parsed, or json dump of value failed") + #return value.json() + return {"success": False} + # Sets files in the backend def set_files(self, infiles): full_execution = self.full_execution diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 569852b4..5f4de234 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -2,7 +2,7 @@ module shuffle go 1.13 -replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared +//replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi @@ -24,7 +24,7 @@ require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 // indirect github.com/elastic/go-elasticsearch/v8 v8.0.0-20210519083322-55daf7425ecb // indirect github.com/frikky/kin-openapi v0.39.0 - github.com/frikky/shuffle-shared v0.0.53 + github.com/frikky/shuffle-shared v0.0.54 github.com/fsouza/go-dockerclient v1.7.2 // indirect github.com/ghodss/yaml v1.0.0 github.com/go-git/go-billy/v5 v5.0.0 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index f695f50a..04cc6d1b 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -165,6 +165,8 @@ github.com/frikky/shuffle-shared v0.0.52 h1:sCSJl6WakYit32UjaRn0wsy4YhTBE6QZbCof github.com/frikky/shuffle-shared v0.0.52/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= github.com/frikky/shuffle-shared v0.0.53 h1:TszF/PoJ3JrfEf7qCGdN0So3bDbofNxh4Fqqz1KlH7Q= github.com/frikky/shuffle-shared v0.0.53/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= +github.com/frikky/shuffle-shared v0.0.54 h1:rc8JcavY6uDxaIkXFwLVotfv3/epXUzuCWzRCozZNGg= +github.com/frikky/shuffle-shared v0.0.54/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= github.com/fsouza/go-dockerclient v1.7.2 h1:bBEAcqLTkpq205jooP5RVroUKiVEWgGecHyeZc4OFjo= github.com/fsouza/go-dockerclient v1.7.2/go.mod h1:+ugtMCVRwnPfY7d8/baCzZ3uwB0BrG5DB8OzbtxaRz8= github.com/getkin/kin-openapi v0.8.0 h1:a6TQjTqwkyscC4/hShJX7WhCVE+4bi9lzw61XHQW5hE= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index c859412a..438ebc88 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5686,7 +5686,7 @@ func initHandlers() { log.Printf("[DEBUG] Setting Opensearch proxy to %s", httpProxy) transport.Proxy = http.ProxyURL(url_proxy) } else { - log.PrintF("[ERROR] Failed setting proxy for %s", httpProxy) + log.Printf("[ERROR] Failed setting proxy for %s", httpProxy) } } diff --git a/docker-compose.yml b/docker-compose.yml index d7fa3eb0..318a3627 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: frontend: - #build: ./frontend + build: ./frontend image: ghcr.io/frikky/shuffle-frontend:nightly container_name: shuffle-frontend hostname: shuffle-frontend @@ -16,7 +16,7 @@ services: depends_on: - backend backend: - #build: ./backend + build: ./backend image: ghcr.io/frikky/shuffle-backend:nightly container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} @@ -34,8 +34,6 @@ services: environment: - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps - SHUFFLE_FILE_LOCATION=/shuffle-files - - HTTP_PROXY=${SHUFFLE_HTTP_PROXY} - - HTTPS_PROXY=${SHUFFLE_HTTPS_PROXY} restart: unless-stopped depends_on: - opensearch @@ -59,8 +57,8 @@ services: - SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME} - SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY} - SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX} - - HTTP_PROXY=${SHUFFLE_HTTP_PROXY} - - HTTPS_PROXY=${SHUFFLE_HTTPS_PROXY} + - HTTP_PROXY=${HTTP_PROXY} + - HTTPS_PROXY=${HTTPS_PROXY} - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY} - SHUFFLE_PASS_APP_PROXY=${SHUFFLE_PASS_APP_PROXY} - SHUFFLE_ORBORUS_EXECUTION_TIMEOUT=600 diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 21e9859d..01dade27 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -200,8 +200,8 @@ const AppCreator = (props) => { const increaseAmount = 50 const actionNonBodyRequest = ["GET", "HEAD", "DELETE", "CONNECT"] const actionBodyRequest = ["POST", "PUT", "PATCH",] - //const authenticationOptions = ["No authentication", "API key", "Bearer auth", "Basic auth", "Oauth2"] - const authenticationOptions = ["No authentication", "API key", "Bearer auth", "Basic auth", "JWT"] + //const authenticationOptions = ["No authentication", "API key", "Bearer auth", "Basic auth", "JWT", "Oauth2"] + const authenticationOptions = ["No authentication", "API key", "Bearer auth", "Basic auth"] const apikeySelection = ["Header", "Query",] const [name, setName] = useState(""); @@ -2058,7 +2058,7 @@ const AppCreator = (props) => {
New action
 - Learn more about actions + Learn more about actions
Name { : null}

Actions {actionAmount > 0 ? ({actionAmount} / {actions.length}) : null}

- Actions are the tasks performed by an app. Read more about actions and apps - here. + Actions are the tasks performed by an app - usually single URL paths for REST API's.
{loopActions}
From 90469f9d0f18bd7e9f9a179c6d036e02d3f23152 Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 4 Jun 2021 07:22:15 +0200 Subject: [PATCH 77/96] Fixed docker-compose builds --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 318a3627..20bd3ec9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: frontend: - build: ./frontend + #build: ./frontend image: ghcr.io/frikky/shuffle-frontend:nightly container_name: shuffle-frontend hostname: shuffle-frontend @@ -16,7 +16,7 @@ services: depends_on: - backend backend: - build: ./backend + #build: ./backend image: ghcr.io/frikky/shuffle-backend:nightly container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} From d8d64b858f7f3e8f08a47de73985340ae94c3519 Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 4 Jun 2021 08:45:59 +0200 Subject: [PATCH 78/96] Moved documentation to shared --- .env | 2 +- backend/go-app/go.mod | 2 +- backend/go-app/main.go | 198 +----------------------------- frontend/src/views/AppCreator.jsx | 8 +- 4 files changed, 9 insertions(+), 201 deletions(-) diff --git a/.env b/.env index 27cf372b..9fadb1fd 100644 --- a/.env +++ b/.env @@ -50,7 +50,7 @@ SHUFFLE_CONTAINER_AUTO_CLEANUP=false SHUFFLE_ELASTIC=true # DATABASE CONFIGURATIONS -DATASTORE_EMULATOR_HOST=shuffl-database:8000 +DATASTORE_EMULATOR_HOST=shuffle-database:8000 SHUFFLE_OPENSEARCH_URL=http://shuffle-opensearch:9200 SHUFFLE_OPENSEARCH_USERNAME= SHUFFLE_OPENSEARCH_PASSWORD= diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 5f4de234..1dd998ab 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -2,7 +2,7 @@ module shuffle go 1.13 -//replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared +replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 438ebc88..fee98601 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -44,8 +44,6 @@ import ( "github.com/frikky/kin-openapi/openapi3" */ - "github.com/google/go-github/v28/github" - "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-git/v5" @@ -2824,197 +2822,7 @@ type Result struct { List []string `json:"list"` } -var docs_list = Result{List: []string{}} - -func getDocList(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - ctx := context.Background() - //if item, err := memcache.Get(ctx, "docs_list"); err == memcache.ErrCacheMiss { - // // Not in cache - //} else if err != nil { - // // Error with cache - // log.Printf("Error getting item: %v", err) - //} else { - // resp.WriteHeader(200) - // resp.Write([]byte(item.Value)) - // return - //} - - if len(docs_list.List) > 0 { - b, err := json.Marshal(docs_list) - if err != nil { - log.Printf("Failed marshaling result: %s", err) - //http.Error(resp, err.Error(), 500) - } else { - resp.WriteHeader(200) - resp.Write(b) - return - } - } - - client := github.NewClient(nil) - _, item1, _, err := client.Repositories.GetContents(ctx, "frikky", "shuffle-docs", "docs", nil) - if err != nil { - log.Printf("Github error: %s", err) - resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error listing directory: %s"`, err))) - return - } - - if len(item1) == 0 { - resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No docs available."}`))) - return - } - - names := []string{} - for _, item := range item1 { - if !strings.HasSuffix(*item.Name, "md") { - continue - } - - names = append(names, (*item.Name)[0:len(*item.Name)-3]) - } - - log.Println(names) - - var result Result - result.Success = true - result.Reason = "Success" - result.List = names - docs_list = result - - b, err := json.Marshal(result) - if err != nil { - http.Error(resp, err.Error(), 500) - return - } - - //item := &memcache.Item{ - // Key: "docs_list", - // Value: b, - // Expiration: time.Minute * 60, - //} - - //if err := memcache.Add(ctx, item); err == memcache.ErrNotStored { - // if err := memcache.Set(ctx, item); err != nil { - // log.Printf("Error setting item: %v", err) - // } - //} else if err != nil { - // log.Printf("error adding item: %v", err) - //} else { - // log.Printf("Set cache for %s", item.Key) - //} - - resp.WriteHeader(200) - resp.Write(b) -} - // r.HandleFunc("/api/v1/docs/{key}", getDocs).Methods("GET", "OPTIONS") -var alldocs = map[string][]byte{} - -func getDocs(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - location := strings.Split(request.URL.String(), "/") - if len(location) != 5 { - resp.WriteHeader(404) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad path. Use e.g. /api/v1/docs/workflows.md"}`))) - return - } - - //ctx := context.Background() - docPath := fmt.Sprintf("https://raw.githubusercontent.com/shaffuru/shuffle-docs/master/docs/%s.md", location[4]) - //location[4] - //var, ok := alldocs["asd"] - key, ok := alldocs[fmt.Sprintf("%s", location[4])] - // Custom cache for github issues lol - if ok { - resp.WriteHeader(200) - resp.Write(key) - return - } - - client := &http.Client{} - req, err := http.NewRequest( - "GET", - docPath, - nil, - ) - - if err != nil { - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad path. Use e.g. /api/v1/docs/workflows.md"}`))) - resp.WriteHeader(404) - //setBadMemcache(ctx, docPath) - return - } - - newresp, err := client.Do(req) - if err != nil { - resp.WriteHeader(404) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad path. Use e.g. /api/v1/docs/workflows.md"}`))) - //setBadMemcache(ctx, docPath) - return - } - - body, err := ioutil.ReadAll(newresp.Body) - if err != nil { - resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't parse data"}`))) - //setBadMemcache(ctx, docPath) - return - } - - type Result struct { - Success bool `json:"success"` - Reason string `json:"reason"` - } - - var result Result - result.Success = true - - //applog.Infof(ctx, string(body)) - //applog.Infof(ctx, "Url: %s", docPath) - //applog.Infof(ctx, "Status: %d", newresp.StatusCode) - //applog.Infof(ctx, "GOT BODY OF LENGTH %d", len(string(body))) - - result.Reason = string(body) - b, err := json.Marshal(result) - if err != nil { - http.Error(resp, err.Error(), 500) - //setBadMemcache(ctx, docPath) - return - } - - alldocs[location[4]] = b - - // Add to cache if it doesn't exist - //item := &memcache.Item{ - // Key: docPath, - // Value: b, - // Expiration: time.Minute * 60, - //} - - //if err := memcache.Add(ctx, item); err == memcache.ErrNotStored { - // if err := memcache.Set(ctx, item); err != nil { - // log.Printf("Error setting item: %v", err) - // } - //} else if err != nil { - // log.Printf("error adding item: %v", err) - //} else { - // log.Printf("Set cache for %s", item.Key) - //} - - resp.WriteHeader(200) - resp.Write(b) -} func getOpenapi(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) @@ -5735,7 +5543,7 @@ func initHandlers() { } for { - _, err = shuffle.RunInit(*dbclient, *es, storage.Client{}, gceProject, "onprem", true, "elasticsearch") + _, err = shuffle.RunInit(*dbclient, *es, storage.Client{}, gceProject, "onprem", true, elasticConfig) if err != nil { log.Printf("[DEBUG] Error in initial database connection. Retrying in 5 seconds. %s", err) time.Sleep(5 * time.Second) @@ -5786,8 +5594,8 @@ func initHandlers() { r.HandleFunc("/api/v1/getenvironments", shuffle.HandleGetEnvironments).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/setenvironments", shuffle.HandleSetEnvironments).Methods("PUT", "OPTIONS") - r.HandleFunc("/api/v1/docs", getDocList).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/docs/{key}", getDocs).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/docs", shuffle.GetDocList).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/docs/{key}", shuffle.GetDocs).Methods("GET", "OPTIONS") // Queuebuilder and Workflow streams. First is to update a stream, second to get a stream // Changed from workflows/streams to streams, as appengine was messing up diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 01dade27..6088f02f 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -2058,7 +2058,7 @@ const AppCreator = (props) => {
New action
 - Learn more about actions + Learn more about actions
Name { return (
- What is this?
+ What is this?
These are required fields for authenticating with {selectedApp.name}
Name - what is this used for? @@ -2871,7 +2871,7 @@ const AppCreator = (props) => {

Test

Test an action to see whether it performs in an expected way. -  TBD: Click here to learn more about testing. +  TBD: Click here to learn more about testing.
Test :)
@@ -3055,7 +3055,7 @@ const AppCreator = (props) => { upload = ref} onChange={editHeaderImage} />

General information

- Click here to learn more about app creation + Click here to learn more about app creation
{ From c81140babbd706f72769968a3fb11f4b847faadd Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 6 Jun 2021 08:20:39 +0200 Subject: [PATCH 79/96] Fixed execution colors for shuffle tools --- backend/go-app/walkoff.go | 11 +- docker-compose.yml | 4 +- frontend/src/components/ConfigureWorkflow.jsx | 2 +- frontend/src/views/AngularWorkflow.jsx | 109 ++++++++++++------ functions/onprem/orborus/build.sh | 2 +- functions/onprem/orborus/orborus.go | 17 ++- functions/onprem/worker/build.sh | 2 +- functions/onprem/worker/go.mod | 2 +- 8 files changed, 96 insertions(+), 53 deletions(-) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 6c91d2ec..899e40d5 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -691,7 +691,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { executionRequests.Data = []shuffle.ExecutionRequest{} } else { log.Printf("[INFO] Executionrequests (%s): %d", id, len(executionRequests.Data)) - log.Printf("IDS: %#v", executionRequests.Data[0].ExecutionId) + //log.Printf("IDS: %#v", executionRequests.Data[0].ExecutionId) } newjson, err := json.Marshal(executionRequests) @@ -768,6 +768,7 @@ func validateNewWorkerExecution(body []byte) error { log.Printf("[WARNING] Failed execution unmarshaling: %s", err) return err } + //log.Printf("\n\nGOT EXEC WITH RESULT %#v (%d)\n\n", execution.Status, len(execution.Results)) baseExecution, err := shuffle.GetWorkflowExecution(ctx, execution.ExecutionId) if err != nil { @@ -788,9 +789,9 @@ func validateNewWorkerExecution(body []byte) error { return errors.New(fmt.Sprintf("Bad length of trigger: %d (probably normal app)", len(execution.Workflow.Triggers))) } - if baseExecution.Status != "WAITING" && baseExecution.Status != "EXECUTING" { - return errors.New(fmt.Sprintf("Workflow is already finished or failed. Can't update")) - } + //if baseExecution.Status != "WAITING" && baseExecution.Status != "EXECUTING" { + // return errors.New(fmt.Sprintf("Workflow is already finished or failed. Can't update")) + //} if execution.Status == "EXECUTING" { //log.Printf("[INFO] Inside executing.") @@ -818,7 +819,7 @@ func validateNewWorkerExecution(body []byte) error { //log.Printf("\n\nSHOULD SET BACKEND DATA FOR EXEC \n\n") err = shuffle.SetWorkflowExecution(ctx, execution, true) if err == nil { - log.Printf("[INFO] Set workflowexecution based on new worker (>0.8.53) for execution %s. Actions: %d, Triggers: %d, Results: %d, Status: %s, Result: %s", execution.ExecutionId, len(execution.Workflow.Actions), len(execution.Workflow.Triggers), len(execution.Results), execution.Status, execution.Result) + log.Printf("[INFO] Set workflowexecution based on new worker (>0.8.53) for execution %s. Actions: %d, Triggers: %d, Results: %d, Status: %s", execution.ExecutionId, len(execution.Workflow.Actions), len(execution.Workflow.Triggers), len(execution.Results), execution.Status) //, execution.Result) //log.Printf("[INFO] Successfully set the execution to wait.") } else { log.Printf("[WARNING] Failed to set the execution to wait.") diff --git a/docker-compose.yml b/docker-compose.yml index 20bd3ec9..4f648a84 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -49,11 +49,11 @@ services: - /var/run/docker.sock:/var/run/docker.sock environment: - SHUFFLE_APP_SDK_VERSION=0.8.97 - - SHUFFLE_WORKER_VERSION=0.8.97 + - SHUFFLE_WORKER_VERSION=nightly - ORG_ID=${ORG_ID} - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT} - - DOCKER_API_VERSION=1.40 + - DOCKER_API_VERSION=1.35 - SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME} - SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY} - SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX} diff --git a/frontend/src/components/ConfigureWorkflow.jsx b/frontend/src/components/ConfigureWorkflow.jsx index c77bbf54..0cc6182a 100644 --- a/frontend/src/components/ConfigureWorkflow.jsx +++ b/frontend/src/components/ConfigureWorkflow.jsx @@ -99,7 +99,7 @@ const Workflow = (props) => { var filled = true for (var key in action.parameters) { if (action.parameters[key].configuration) { - console.log("Found config: ", action.parameters[key]) + //console.log("Found config: ", action.parameters[key]) if (action.parameters[key].value === null || action.parameters[key].value.length === 0) { filled = false break diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 5cc65caa..8951a295 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -11,7 +11,7 @@ import NestedMenuItem from "material-ui-nested-menu-item"; import {TextField, Drawer, Button, Paper, Grid, Tabs, InputAdornment, Tab, ButtonBase, Tooltip, Select, MenuItem, Divider, Dialog, Modal, DialogActions, DialogTitle, InputLabel, DialogContent, FormControl, IconButton, Menu, Input, FormGroup, FormControlLabel, Typography, Checkbox, Breadcrumbs, CircularProgress, Switch, Fade} from '@material-ui/core'; -import {Undo as UndoIcon, FileCopy as FileCopyIcon, GetApp as GetAppIcon, Search as SearchIcon, ArrowUpward as ArrowUpwardIcon, Visibility as VisibilityIcon, Done as DoneIcon, Close as CloseIcon, Error as ErrorIcon, FindReplace as FindreplaceIcon, ArrowLeft as ArrowLeftIcon, Cached as CachedIcon, DirectionsRun as DirectionsRunIcon, Add as AddIcon, Polymer as PolymerIcon, FormatListNumbered as FormatListNumberedIcon, Create as CreateIcon, PlayArrow as PlayArrowIcon, AspectRatio as AspectRatioIcon, MoreVert as MoreVertIcon, Apps as AppsIcon, Schedule as ScheduleIcon, FavoriteBorder as FavoriteBorderIcon, Pause as PauseIcon, Delete as DeleteIcon, AddCircleOutline as AddCircleOutlineIcon, Save as SaveIcon, KeyboardArrowLeft as KeyboardArrowLeftIcon, KeyboardArrowRight as KeyboardArrowRightIcon, ArrowBack as ArrowBackIcon, Settings as SettingsIcon, LockOpen as LockOpenIcon, ExpandMore as ExpandMoreIcon, VpnKey as VpnKeyIcon} from '@material-ui/icons'; +import {OpenInNew as OpenInNewIcon,Undo as UndoIcon, FileCopy as FileCopyIcon, GetApp as GetAppIcon, Search as SearchIcon, ArrowUpward as ArrowUpwardIcon, Visibility as VisibilityIcon, Done as DoneIcon, Close as CloseIcon, Error as ErrorIcon, FindReplace as FindreplaceIcon, ArrowLeft as ArrowLeftIcon, Cached as CachedIcon, DirectionsRun as DirectionsRunIcon, Add as AddIcon, Polymer as PolymerIcon, FormatListNumbered as FormatListNumberedIcon, Create as CreateIcon, PlayArrow as PlayArrowIcon, AspectRatio as AspectRatioIcon, MoreVert as MoreVertIcon, Apps as AppsIcon, Schedule as ScheduleIcon, FavoriteBorder as FavoriteBorderIcon, Pause as PauseIcon, Delete as DeleteIcon, AddCircleOutline as AddCircleOutlineIcon, Save as SaveIcon, KeyboardArrowLeft as KeyboardArrowLeftIcon, KeyboardArrowRight as KeyboardArrowRightIcon, ArrowBack as ArrowBackIcon, Settings as SettingsIcon, LockOpen as LockOpenIcon, ExpandMore as ExpandMoreIcon, VpnKey as VpnKeyIcon} from '@material-ui/icons'; import * as cytoscape from 'cytoscape'; import * as edgehandles from 'cytoscape-edgehandles'; @@ -7864,7 +7864,8 @@ const AngularWorkflow = (props) => { if (triggers.length > 2) { if (data.action.app_name === "shuffle-subflow") { - actionimg = {"Shuffle + const parsedImage = triggers[1].large_image + actionimg = {"Shuffle } if (data.action.app_name === "User Input") { @@ -7872,8 +7873,40 @@ const AngularWorkflow = (props) => { } } + if (data.action.app_name === "Shuffle Tools") { + console.log("APP (TOOLS): ", data.action) + + const nodedata = cy.getElementById(data.action.id).data() + if (nodedata.fillstyle === "linear-gradient") { + console.log("LINEAR :D") + var imgStyle = { + marginRight: 20, + width: imgsize, + height: imgsize, + border: `2px solid ${statusColor}`, + borderRadius: executionData.start === data.action.id ? 25 : 5, + background: `linear-gradient(to right, ${nodedata.fillGradient})` + } + + console.log("STYLE: ", imgStyle) + + actionimg = {nodedata.label} + } + } + + if (validate.valid && typeof(validate.result) === "string") { validate.result = JSON.parse(validate.result) + } + + if (validate.valid && typeof(validate.result) === "object") { + if (validate.result.result !== undefined && validate.result.result !== null) { + try { + validate.result.result = JSON.parse(validate.result.result) + } catch (e) { + console.log("ERROR PARSING: ", e) + } + } } return ( @@ -7888,24 +7921,46 @@ const AngularWorkflow = (props) => { currentnode.removeClass('shuffle-hover-highlight') } }}> -
- { - setSelectedResult(data) - setCodeModalOpen(true) - }}> - - - - - {actionimg} -
-
{data.action.label}
-
- - {data.action.name} - +
+
+ { + setSelectedResult(data) + setCodeModalOpen(true) + }}> + + + + + {actionimg} +
+
{data.action.label}
+
+ + {data.action.name} + +
+ {data.action.app_name === "shuffle-subflow" && validate.result.success !== undefined && validate.result.success === true ? + + {validate.valid && data.action.parameters !== undefined && data.action.parameters !== null ? + data.action.parameters[0].value === props.match.params.key ? + { + getWorkflowExecution(props.match.params.key, validate.result.execution_id) + }}> + See sub-execution + + : + { + }}> + + + : + "TBD: Load subexecution result for" + } + + : null + }
@@ -7930,24 +7985,6 @@ const AngularWorkflow = (props) => { }} name={"Results for "+data.action.label} /> - {data.action.app_name === "shuffle-subflow" && validate.result.success !== undefined && validate.result.success === true ? - - {validate.valid && data.action.parameters !== undefined && data.action.parameters !== null ? - data.action.parameters[0].value === props.match.params.key ? - { - getWorkflowExecution(props.match.params.key, validate.result.execution_id) - }}> - See sub-execution - - : - { - }}>See subflow execution - : - "TBD: Load subexecution result for" - } - - : null - } :
diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index 61413801..8a35018c 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-orborus -VERSION=0.8.97 +VERSION=0.8.98 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 3df6657d..a3b28c4e 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -254,7 +254,7 @@ func initializeImages() { log.Printf("[WARNING] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %s", appSdkVersion) } if workerVersion == "" { - workerVersion = "nighty" + workerVersion = "nightly" log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %s", workerVersion) } @@ -438,7 +438,6 @@ func main() { //log.Printf("Prerequest") //go getStats() newresp, err := client.Do(req) - executionCount := getRunningWorkers(ctx, workerTimeout) //log.Printf("Postrequest") if err != nil { log.Printf("[WARNING] Failed making request: %s", err) @@ -501,7 +500,8 @@ func main() { continue } - // Anything below here verifies concurrency virification + // Anything below here verifies concurrency + executionCount := getRunningWorkers(ctx, workerTimeout) if executionCount >= maxConcurrency { if zombiecounter*sleepTime > workerTimeout { go zombiecheck(ctx, workerTimeout) @@ -641,12 +641,17 @@ func getRunningWorkers(ctx context.Context, workerTimeout int) int { containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{ All: true, }) - //Filters: filters.Args{ - // map[string][]string{"ancestor": {":"}}, - //}, + // Automatically updates the version if err != nil { log.Printf("[ERROR] Error getting containers: %s", err) + + newVersionSplit := strings.Split(fmt.Sprintf("%s", err), "version is") + if len(newVersionSplit) > 1 { + dockerApiVersion = strings.TrimSpace(newVersionSplit[1]) + log.Printf("[INFO] Changed the API version to default to %s", dockerApiVersion) + } + return maxConcurrency } diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh index 214ab948..c5fc2dbf 100644 --- a/functions/onprem/worker/build.sh +++ b/functions/onprem/worker/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-worker -VERSION=0.8.97 +VERSION=0.8.99 echo "Running docker build with $NAME:$VERSION" #CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin . diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 6d099c67..82422869 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -12,7 +12,7 @@ require ( github.com/docker/docker v20.10.5+incompatible github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect - github.com/frikky/shuffle-shared v0.0.45 + github.com/frikky/shuffle-shared v0.0.55 github.com/fsouza/go-dockerclient v1.7.2 github.com/go-git/go-billy/v5 v5.3.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect From d6f1b0e3ae6280e2188bb2058269e8a685c8b19d Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 6 Jun 2021 13:43:51 +0200 Subject: [PATCH 80/96] Added frontend bugfixes for workflow view --- backend/app_sdk/app_base.py | 2 +- frontend/src/components/ParsedAction.jsx | 2 +- frontend/src/views/AngularWorkflow.jsx | 33 ++++++++++++++----- frontend/src/views/Workflows.jsx | 2 +- .../extensions/scripts/disable_disk_check.sh | 11 +++++++ 5 files changed, 38 insertions(+), 12 deletions(-) create mode 100644 functions/extensions/scripts/disable_disk_check.sh diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 7c3b7ea2..88e5a4a3 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -2265,7 +2265,7 @@ class AppBase: print("[INFO] Running normal execution\n") #newres = await func(**params) - print("PARAMS: %s" % params) + #print("PARAMS: %s" % params) while True: try: newres = await func(**params) diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index 9794d0f6..6faf26d3 100644 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -758,7 +758,7 @@ const ParsedAction = (props) => { }} /> - console.log("FIELD VALUE: ", data.value) + //console.log("FIELD VALUE: ", data.value) //const regexp = new RegExp("\W+\.", "g") //let match //while ((match = regexp.exec(data.value)) !== null) { diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 8951a295..610f269b 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -536,7 +536,7 @@ const AngularWorkflow = (props) => { if (response.status !== 200) { console.log("Status not 200 for ABORT EXECUTION :O!") } else { - alert.success("Execution aborted") + //alert.success("Execution aborted") } return response.json() @@ -1108,10 +1108,10 @@ const AngularWorkflow = (props) => { // This can be used to only show prioritzed ones later // Right now, it can prioritize authenticated ones + //"Testing", const internalIds = [ "Shuffle Tools", - "Testing", - "Http", + "http", ] const getAppAuthentication = (reset) => { @@ -4231,7 +4231,22 @@ const AngularWorkflow = (props) => { const runSearch = (value) => { if (value.length > 0) { - const newApps = allApps.filter(app => (app.name.toLowerCase().includes(value.trim().toLowerCase() || app.description.toLowerCase().includes(value.trim().toLowerCase()))) && !(!app.activated && app.generated)) + var newApps = allApps.filter(app => (app.name.toLowerCase().includes(value.trim().toLowerCase() || app.description.toLowerCase().includes(value.trim().toLowerCase()))) && !(!app.activated && app.generated)) + + // Extend search + if (newApps.length === 0) { + const searchvalue = value.trim().toLowerCase() + newApps = allApps.filter(app => { + for (var key in app.actions) { + const inneraction = app.actions[key] + if (inneraction.name.toLowerCase().includes(searchvalue)) { + return true + } + } + + return false + }) + } //setFilteredApps(responseJson.filter(app => !internalIds.includes(app.name) && !(!app.activated && app.generated))) //console.log("FOUND: ", newApps) @@ -7874,11 +7889,11 @@ const AngularWorkflow = (props) => { } if (data.action.app_name === "Shuffle Tools") { - console.log("APP (TOOLS): ", data.action) + //console.log("APP (TOOLS): ", data.action) const nodedata = cy.getElementById(data.action.id).data() - if (nodedata.fillstyle === "linear-gradient") { - console.log("LINEAR :D") + if (nodedata !== undefined && nodedata !== null && nodedata.fillstyle === "linear-gradient") { + //console.log("LINEAR :D") var imgStyle = { marginRight: 20, width: imgsize, @@ -7888,7 +7903,7 @@ const AngularWorkflow = (props) => { background: `linear-gradient(to right, ${nodedata.fillGradient})` } - console.log("STYLE: ", imgStyle) + //console.log("STYLE: ", imgStyle) actionimg = {nodedata.label} } @@ -8595,7 +8610,7 @@ const AngularWorkflow = (props) => { {/*selectedApp.link.length > 0 ?
: null*/}
{selectedApp.authentication.parameters.map((data, index) => { - console.log("AUTH: ", data) + //console.log("AUTH: ", data) return (
diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 4cbba6fa..d75ff690 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -88,7 +88,7 @@ export const GetIconInfo = (action) => { {"key": "cache_get", "values": ["get_cache"]}, {"key": "filter", "values": ["filter"]}, {"key": "merge", "values": ["join", "merge"]}, - {"key": "search", "values": ["search", "find", "locate"]}, + {"key": "search", "values": ["search", "find", "locate", "index",]}, {"key": "list", "values": ["list", "head", "options"]}, {"key": "download", "values": ["get", "download", "return", "hello_world", "curl",]}, {"key": "add", "values": ["add"]}, diff --git a/functions/extensions/scripts/disable_disk_check.sh b/functions/extensions/scripts/disable_disk_check.sh new file mode 100644 index 00000000..3b3ddcc6 --- /dev/null +++ b/functions/extensions/scripts/disable_disk_check.sh @@ -0,0 +1,11 @@ +curl -XPUT http://localhost:9200/_cluster/settings -H "Content-Type:application/json" -d \ +'{ + "transient": { + "cluster.routing.allocation.disk.threshold_enabled": false + } +}' + +curl -XPUT http://localhost:9200/_all/_settings -H "Content-Type: application/json" -d \ +'{ + "index.blocks.read_only_allow_delete": null +}' From 21279e7ea62ef22f2eec29da7fb9102cb7d95175 Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 7 Jun 2021 06:47:24 +0200 Subject: [PATCH 81/96] #396: Fixed a bug where workflows are randomly aborted --- functions/onprem/worker/worker.go | 61 +++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 11 deletions(-) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index fb75a086..621035d9 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -93,7 +93,7 @@ func init() { // removes every container except itself (worker) func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason string, handleResultSend bool) { - log.Printf("[INFO] Shutdown (%s) started with reason %s", workflowExecution.Status, reason) + log.Printf("[INFO] Shutdown (%s) started with reason %s. Result amount: %d. ResultsSent: %d, Send result: %#v", workflowExecution.Status, reason, len(workflowExecution.Results), requestsSent, handleResultSend) //reason := "Error in execution" sleepDuration := 1 @@ -101,9 +101,9 @@ func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason shutdownData, err := json.Marshal(workflowExecution) if err == nil { sendResult(workflowExecution, shutdownData) - log.Printf("[WARNING] Sent shutdown update") + log.Printf("[WARNING] Sent shutdown update with %d results and result value %s", len(workflowExecution.Results), reason) } else { - log.Printf("[WARNING] DIDNT send update") + log.Printf("[WARNING] Failed to send update: %s", err) } time.Sleep(time.Duration(sleepDuration) * time.Second) @@ -554,7 +554,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { for _, subresult := range workflowExecution.Results { if subresult.Action.ID == branch.SourceID { if subresult.Status != "SKIPPED" && subresult.Status != "FAILURE" { - log.Printf("\n\n\nSUBRESULT PARENT STATUS: %s\n\n\n", subresult.Status) + //log.Printf("\n\n\nSUBRESULT PARENT STATUS: %s\n\n\n", subresult.Status) isSkipped = false break @@ -617,7 +617,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { } if exit && len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions) { - log.Printf("Shutting down.") + log.Printf("[DEBUG] Shutting down (1)") shutdown(workflowExecution, "", "", true) } @@ -976,6 +976,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { err = deployApp(dockercli, images[0], identifier, env, workflowExecution) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { if strings.Contains(err.Error(), "exited prematurely") { + log.Printf("[DEBUG] Shutting down (2)") shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } @@ -983,12 +984,14 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { err = deployApp(dockercli, image, identifier, env, workflowExecution) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { if strings.Contains(err.Error(), "exited prematurely") { + log.Printf("[DEBUG] Shutting down (3)") shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } log.Printf("[WARNING] Failed CLEANUP execution. Downloading image remotely.") reader, err := dockercli.ImagePull(context.Background(), image, pullOptions) if err != nil { log.Printf("[ERROR] Failed getting %s. Couldn't be find locally, AND is missing.", image) + log.Printf("[DEBUG] Shutting down (4)") shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } @@ -996,10 +999,12 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { _, err = io.Copy(buildBuf, reader) if err != nil && !strings.Contains(fmt.Sprintf("%s", err.Error()), "Conflict. The container name") { log.Printf("[ERROR] Error in IO copy: %s", err) + log.Printf("[DEBUG] Shutting down (5)") shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } else { if strings.Contains(buildBuf.String(), "errorDetail") { log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), image) + log.Printf("[DEBUG] Shutting down (6)") shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } @@ -1011,12 +1016,14 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { log.Printf("[ERROR] Failed deploying image for the FOURTH time. Aborting if the image doesn't exist") if strings.Contains(err.Error(), "exited prematurely") { + log.Printf("[DEBUG] Shutting down (7)") shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } if strings.Contains(err.Error(), "No such image") { //log.Printf("[WARNING] Failed deploying %s from image %s: %s", identifier, image, err) log.Printf("[ERROR] Image doesn't exist. Shutting down") + log.Printf("[DEBUG] Shutting down (8)") shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } } @@ -1027,6 +1034,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { err = deployApp(dockercli, images[0], identifier, env, workflowExecution) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { if strings.Contains(err.Error(), "exited prematurely") { + log.Printf("[DEBUG] Shutting down (9)") shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } @@ -1036,6 +1044,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { err = deployApp(dockercli, image, identifier, env, workflowExecution) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { if strings.Contains(err.Error(), "exited prematurely") { + log.Printf("[DEBUG] Shutting down (10)") shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } @@ -1043,6 +1052,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { err = deployApp(dockercli, image, identifier, env, workflowExecution) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { if strings.Contains(err.Error(), "exited prematurely") { + log.Printf("[DEBUG] Shutting down (11)") shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } @@ -1050,6 +1060,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { reader, err := dockercli.ImagePull(context.Background(), image, pullOptions) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { log.Printf("[ERROR] Failed getting %s. The couldn't be find locally, AND is missing.", image) + log.Printf("[DEBUG] Shutting down (12)") shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } @@ -1057,10 +1068,12 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { _, err = io.Copy(buildBuf, reader) if err != nil { log.Printf("[ERROR] Error in IO copy: %s", err) + log.Printf("[DEBUG] Shutting down (13)") shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } else { if strings.Contains(buildBuf.String(), "errorDetail") { log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), image) + log.Printf("[DEBUG] Shutting down (14)") shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } @@ -1071,12 +1084,14 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { log.Printf("[ERROR] Failed deploying image for the FOURTH time. Aborting if the image doesn't exist") if strings.Contains(err.Error(), "exited prematurely") { + log.Printf("[DEBUG] Shutting down (15)") shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } if strings.Contains(err.Error(), "No such image") { //log.Printf("[WARNING] Failed deploying %s from image %s: %s", identifier, image, err) log.Printf("[ERROR] Image doesn't exist. Shutting down") + log.Printf("[DEBUG] Shutting down (16)") shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } } @@ -1117,6 +1132,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { if shutdownCheck { log.Println("[INFO] BREAKING BECAUSE RESULTS IS SAME LENGTH AS ACTIONS. SHOULD CHECK ALL RESULTS FOR WHETHER THEY'RE DONE") validateFinished(workflowExecution) + log.Printf("[DEBUG] Shutting down (17)") shutdown(workflowExecution, "", "", true) } } @@ -1255,6 +1271,7 @@ func handleDefaultExecution(client *http.Client, req *http.Request, workflowExec err := executionInit(workflowExecution) if err != nil { log.Printf("[INFO] Workflow setup failed: %s", workflowExecution.ExecutionId, err) + log.Printf("[DEBUG] Shutting down (18)") shutdown(workflowExecution, "", "", true) } @@ -1291,7 +1308,8 @@ func handleDefaultExecution(client *http.Client, req *http.Request, workflowExec log.Printf("[ERROR] Bad statuscode: %d, %s", newresp.StatusCode, string(body)) if strings.Contains(string(body), "Workflowexecution is already finished") { - shutdown(workflowExecution, "", "", false) + log.Printf("[DEBUG] Shutting down (19)") + shutdown(workflowExecution, "", "", true) } time.Sleep(time.Duration(sleepTime) * time.Second) @@ -1307,12 +1325,14 @@ func handleDefaultExecution(client *http.Client, req *http.Request, workflowExec if workflowExecution.Status == "FINISHED" || workflowExecution.Status == "SUCCESS" { log.Printf("[INFO] Workflow %s is finished. Exiting worker.", workflowExecution.ExecutionId) + log.Printf("[DEBUG] Shutting down (20)") shutdown(workflowExecution, "", "", true) } log.Printf("[INFO] Status: %s, Results: %d, actions: %d", workflowExecution.Status, len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)+extra) if workflowExecution.Status != "EXECUTING" { log.Printf("[WARNING] Exiting as worker execution has status %s!", workflowExecution.Status) + log.Printf("[DEBUG] Shutting down (21)") shutdown(workflowExecution, "", "", true) } @@ -1544,7 +1564,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl //dbSave := false if len(results) != len(workflowExecution.Results) { - log.Printf("\n\n[WARNING] There may have been an issue in transaction queue. Result lengths: %d vs %d. Should check which exists the base results, but not in entire execution, then append.\n\n", len(results), len(workflowExecution.Results)) + log.Printf("[DEBUG] There may have been an issue in transaction queue. Result lengths: %d vs %d. Should check which exists the base results, but not in entire execution, then append.", len(results), len(workflowExecution.Results)) } // Validating that action results hasn't changed @@ -1619,12 +1639,14 @@ func sendResult(workflowExecution shuffle.WorkflowExecution, data []byte) { if err != nil { log.Printf("[ERROR] Failed creating finishing request: %s", err) + log.Printf("[DEBUG] Shutting down (22)") shutdown(workflowExecution, "", "", false) } newresp, err := topClient.Do(req) if err != nil { log.Printf("[ERROR] Error running finishing request: %s", err) + log.Printf("[DEBUG] Shutting down (23)") shutdown(workflowExecution, "", "", false) } @@ -1649,6 +1671,7 @@ func validateFinished(workflowExecution shuffle.WorkflowExecution) { shutdownData, err := json.Marshal(workflowExecution) if err != nil { log.Printf("[ERROR] Failed to unmarshal data for backend") + log.Printf("[DEBUG] Shutting down (24)") shutdown(workflowExecution, "", "", true) } @@ -1704,10 +1727,8 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { } func setWorkflowExecution(ctx context.Context, workflowExecution shuffle.WorkflowExecution, dbSave bool) error { - //log.Printf("IN SET WORKFLOW EXEC!") - //log.Printf("\n\n\nRESULT: %s\n\n\n", workflowExecution.Status) if len(workflowExecution.ExecutionId) == 0 { - log.Printf("Workflowexeciton executionId can't be empty.") + log.Printf("[INFO] Workflowexecution executionId can't be empty.") return errors.New("ExecutionId can't be empty.") } @@ -1717,8 +1738,18 @@ func setWorkflowExecution(ctx context.Context, workflowExecution shuffle.Workflo handleExecutionResult(workflowExecution) validateFinished(workflowExecution) + // FIXME: Should this shutdown OR send the result? + // The worker may not be running the backend hmm if dbSave { - shutdown(workflowExecution, "", "", false) + if workflowExecution.ExecutionSource == "default" { + log.Printf("[DEBUG] Shutting down (25)") + shutdown(workflowExecution, "", "", true) + //log.Printf("[INFO] Not sending backend info since source is default") + //return + } else { + log.Printf("[DEBUG] NOT shutting down with dbSave (%s)", workflowExecution.ExecutionSource) + } + } return nil @@ -1761,6 +1792,7 @@ func webserverSetup(workflowExecution shuffle.WorkflowExecution) net.Listener { listener, err := getAvailablePort() if err != nil { log.Printf("Failed to created listener: %s", err) + log.Printf("[DEBUG] Shutting down (26)") shutdown(workflowExecution, "", "", true) } port := listener.Addr().(*net.TCPAddr).Port @@ -1918,11 +1950,13 @@ func main() { } if len(authorization) == 0 { log.Println("[INFO] No AUTHORIZATION key set in env") + log.Printf("[DEBUG] Shutting down (27)") shutdown(workflowExecution, "", "", false) } if len(executionId) == 0 { log.Println("[INFO] No EXECUTIONID key set in env") + log.Printf("[DEBUG] Shutting down (28)") shutdown(workflowExecution, "", "", false) } @@ -1936,6 +1970,7 @@ func main() { if err != nil { log.Println("[ERROR] Failed making request builder for backend") + log.Printf("[DEBUG] Shutting down (29)") shutdown(workflowExecution, "", "", true) } topClient = client @@ -1999,6 +2034,7 @@ func main() { err := executionInit(workflowExecution) if err != nil { log.Printf("[INFO] Workflow setup failed: %s", workflowExecution.ExecutionId, err) + log.Printf("[DEBUG] Shutting down (30)") shutdown(workflowExecution, "", "", true) } @@ -2024,6 +2060,7 @@ func main() { if workflowExecution.Status == "FINISHED" || workflowExecution.Status == "SUCCESS" { log.Printf("[INFO] Workflow %s is finished. Exiting worker.", workflowExecution.ExecutionId) + log.Printf("[DEBUG] Shutting down (31)") shutdown(workflowExecution, "", "", true) } @@ -2032,10 +2069,12 @@ func main() { err = handleDefaultExecution(client, req, workflowExecution) if err != nil { log.Printf("[INFO] Workflow %s is finished: %s", workflowExecution.ExecutionId, err) + log.Printf("[DEBUG] Shutting down (32)") shutdown(workflowExecution, "", "", true) } } else { log.Printf("[INFO] Workflow %s has status %s. Exiting worker.", workflowExecution.ExecutionId, workflowExecution.Status) + log.Printf("[DEBUG] Shutting down (33)") shutdown(workflowExecution, workflowExecution.Workflow.ID, "", true) } From e19f29bc238ced545aaf5e81181a74e13aed8ea8 Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 7 Jun 2021 07:04:45 +0200 Subject: [PATCH 82/96] #400: Fixed parsing of underscore/space issues in frontend and SDK --- backend/app_sdk/app_base.py | 53 +++++++++++++++++++------- backend/app_sdk/build.sh | 2 +- backend/go-app/go.mod | 2 +- backend/go-app/go.sum | 2 + backend/go-app/main.go | 1 + docker-compose.yml | 4 +- frontend/confd/templates/nginx.conf | 22 +++++++++++ frontend/src/views/AngularWorkflow.jsx | 6 ++- frontend/src/views/Apps.jsx | 12 +++--- functions/onprem/worker/go.mod | 4 +- 10 files changed, 82 insertions(+), 26 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 88e5a4a3..c1ddb498 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -1220,6 +1220,9 @@ class AppBase: # Loops over split values for value in parsersplit: + #if " " in value: + # value = value.replace(" ", "_", -1) + #print("VALUE: %s\n" % value) actualitem = re.findall(match, value, re.MULTILINE) if value == "#": @@ -1287,21 +1290,43 @@ class AppBase: if len(value) == 0: return basejson, False - if isinstance(basejson, list): - print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value) - return basejson, False - elif isinstance(basejson[value], str): - print(f"[INFO] LOADING STRING '%s' AS JSON" % basejson[value]) - try: - basejson = json.loads(basejson[value]) - print("BASEJSON: %s" % basejson) - except json.decoder.JSONDecodeError as e: - print("RETURNING BECAUSE '%s' IS A NORMAL STRING" % basejson[value]) - return basejson[value], False - else: - basejson = basejson[value] + try: + if isinstance(basejson, list): + print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value) + return basejson, False + elif isinstance(basejson[value], str): + print(f"[INFO] LOADING STRING '%s' AS JSON" % basejson[value]) + try: + basejson = json.loads(basejson[value]) + print("BASEJSON: %s" % basejson) + except json.decoder.JSONDecodeError as e: + print("RETURNING BECAUSE '%s' IS A NORMAL STRING" % basejson[value]) + return basejson[value], False + else: + basejson = basejson[value] + except KeyError as e: + print("[WARNING] Running secondary value check with replacement of underscore in %s: %s" % (value, e)) + if "_" in value: + value = value.replace("_", " ", -1) + elif " " in value: + value = value.replace(" ", "_", -1) - print("Parsed BASEJSON: %s" % basejson) + if isinstance(basejson, list): + print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value) + return basejson, False + elif isinstance(basejson[value], str): + print(f"[INFO] LOADING STRING '%s' AS JSON" % basejson[value]) + try: + basejson = json.loads(basejson[value]) + print("BASEJSON: %s" % basejson) + except json.decoder.JSONDecodeError as e: + print("RETURNING BECAUSE '%s' IS A NORMAL STRING" % basejson[value]) + return basejson[value], False + else: + basejson = basejson[value] + + + #print("Parsed BASEJSON: %s" % basejson) outercnt += 1 except KeyError as e: diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 14b549de..79b62908 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,6 +1,6 @@ #!/bin/bash NAME=shuffle-app_sdk -VERSION=0.8.98 +VERSION=0.8.99 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 1dd998ab..facd92e4 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -24,7 +24,7 @@ require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 // indirect github.com/elastic/go-elasticsearch/v8 v8.0.0-20210519083322-55daf7425ecb // indirect github.com/frikky/kin-openapi v0.39.0 - github.com/frikky/shuffle-shared v0.0.54 + github.com/frikky/shuffle-shared v0.0.56 github.com/fsouza/go-dockerclient v1.7.2 // indirect github.com/ghodss/yaml v1.0.0 github.com/go-git/go-billy/v5 v5.0.0 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 04cc6d1b..a2728936 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -167,6 +167,8 @@ github.com/frikky/shuffle-shared v0.0.53 h1:TszF/PoJ3JrfEf7qCGdN0So3bDbofNxh4Fqq github.com/frikky/shuffle-shared v0.0.53/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= github.com/frikky/shuffle-shared v0.0.54 h1:rc8JcavY6uDxaIkXFwLVotfv3/epXUzuCWzRCozZNGg= github.com/frikky/shuffle-shared v0.0.54/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= +github.com/frikky/shuffle-shared v0.0.56 h1:stC793SdQeBh98yJqCL74aXFo9YYhVIg8CQJ7hm4d6o= +github.com/frikky/shuffle-shared v0.0.56/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= github.com/fsouza/go-dockerclient v1.7.2 h1:bBEAcqLTkpq205jooP5RVroUKiVEWgGecHyeZc4OFjo= github.com/fsouza/go-dockerclient v1.7.2/go.mod h1:+ugtMCVRwnPfY7d8/baCzZ3uwB0BrG5DB8OzbtxaRz8= github.com/getkin/kin-openapi v0.8.0 h1:a6TQjTqwkyscC4/hShJX7WhCVE+4bi9lzw61XHQW5hE= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index fee98601..37f79b0c 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5675,6 +5675,7 @@ func initHandlers() { r.HandleFunc("/api/v1/orgs/", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}", shuffle.HandleGetOrg).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}", shuffle.HandleEditOrg).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/orgs/{orgId}/create_sub_org", shuffle.HandleCreateSubOrg).Methods("POST", "OPTIONS") // This is a new API that validates if a key has been seen before. // Not sure what the best course of action is for it. diff --git a/docker-compose.yml b/docker-compose.yml index 4f648a84..3d15f326 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: frontend: - #build: ./frontend + build: ./frontend image: ghcr.io/frikky/shuffle-frontend:nightly container_name: shuffle-frontend hostname: shuffle-frontend @@ -16,7 +16,7 @@ services: depends_on: - backend backend: - #build: ./backend + build: ./backend image: ghcr.io/frikky/shuffle-backend:nightly container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} diff --git a/frontend/confd/templates/nginx.conf b/frontend/confd/templates/nginx.conf index 6b61364a..bd2219a4 100644 --- a/frontend/confd/templates/nginx.conf +++ b/frontend/confd/templates/nginx.conf @@ -28,6 +28,28 @@ http { server { listen 80; server_name "localhost"; + + #location /static/js/* { + # # avoid clickjacking + # add_header X-Frame-Options DENY; + # add_header X-Content-Type-Options nosniff; + # add_header ; + # # block MIME sniffing + + # # security headers + # add_header X-XSS-Protection "1; mode=block"; + # # add_header Content-Security-Policy "default-src 'self'"; + # add_header Referrer-Policy "no-referrer"; + # server_tokens off; + + # root /usr/share/nginx/html; + # gzip_static on; + # expires 1y; + # add_header Cache-Control public; + # add_header ETag ""; + # try_files $uri /index.html; + #} + location / { # avoid clickjacking add_header X-Frame-Options DENY; diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 610f269b..a51195a7 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -7567,6 +7567,10 @@ const AngularWorkflow = (props) => { } const HandleJsonCopy = (base, copy, base_node_name) => { + if (typeof(copy.name) === "string") { + copy.name = copy.name.replaceAll(" ", "_") + } + console.log("COPY: ", copy) var newitem = JSON.parse(base) to_be_copied = "$"+base_node_name.toLowerCase().replaceAll(" ", "_") @@ -7919,7 +7923,7 @@ const AngularWorkflow = (props) => { try { validate.result.result = JSON.parse(validate.result.result) } catch (e) { - console.log("ERROR PARSING: ", e) + //console.log("ERROR PARSING: ", e) } } } diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index d9b5fc18..3b802b12 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -47,8 +47,8 @@ export const GetParsedPaths = (inputdata, basekey) => { // Handle direct loop! if (!isNaN(key) && basekey === "") { console.log("Handling direct loop.") - parsedValues.push({"type": "object", "name": "Node", "autocomplete": `${basekey}`.toLowerCase()}) - parsedValues.push({"type": "list", "name": `${splitkey}list`, "autocomplete": `${basekey}.#`.toLowerCase()}) + parsedValues.push({"type": "object", "name": "Node", "autocomplete": `${basekey.replaceAll(" ", "_")}`}) + parsedValues.push({"type": "list", "name": `${splitkey}list`, "autocomplete": `${basekey.replaceAll(" ", "_")}.#`}) const returnValues = GetParsedPaths(value, `${basekey}.#`) for (var subkey in returnValues) { parsedValues.push(returnValues[subkey]) @@ -61,8 +61,8 @@ export const GetParsedPaths = (inputdata, basekey) => { if (typeof(value) === 'object') { if (Array.isArray(value)) { // Check if each item is object - parsedValues.push({"type": "object", "name": basekeyname, "autocomplete": `${basekey}.${key}`.toLowerCase()}) - parsedValues.push({"type": "list", "name": `${basekeyname}${splitkey}list`, "autocomplete": `${basekey}.${key}.#`.toLowerCase()}) + parsedValues.push({"type": "object", "name": basekeyname, "autocomplete": `${basekey}.${key.replaceAll(" ", "_")}`}) + parsedValues.push({"type": "list", "name": `${basekeyname}${splitkey}list`, "autocomplete": `${basekey}.${key.replaceAll(" ", "_")}.#`}) // Only check the first. This would be probably be dumb otherwise. for (var subkey in value) { @@ -79,14 +79,14 @@ export const GetParsedPaths = (inputdata, basekey) => { } //console.log(key+" is array") } else { - parsedValues.push({"type": "object", "name": basekeyname, "autocomplete": `${basekey}.${key}`.toLowerCase()}) + parsedValues.push({"type": "object", "name": basekeyname, "autocomplete": `${basekey}.${key.replaceAll(" ", "_")}`}) const returnValues = GetParsedPaths(value, `${basekey}.${key}`) for (var subkey in returnValues) { parsedValues.push(returnValues[subkey]) } } } else { - parsedValues.push({"type": "value", "name": basekeyname, "autocomplete": `${basekey}.${key}`.toLowerCase(), "value": value,}) + parsedValues.push({"type": "value", "name": basekeyname, "autocomplete": `${basekey}.${key.replaceAll(" ", "_")}`, "value": value,}) } } diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 82422869..ba5bbeea 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -12,10 +12,12 @@ require ( github.com/docker/docker v20.10.5+incompatible github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect - github.com/frikky/shuffle-shared v0.0.55 + github.com/elastic/go-elasticsearch/v8 v8.0.0-20210531084204-f01628963386 // indirect + github.com/frikky/shuffle-shared v0.0.56 github.com/fsouza/go-dockerclient v1.7.2 github.com/go-git/go-billy/v5 v5.3.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/go-github/v28 v28.1.1 // indirect github.com/gorilla/mux v1.8.0 github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect From 8844544d537847f2671d9ac7932c9a98745c79b0 Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 7 Jun 2021 09:18:44 +0200 Subject: [PATCH 83/96] Added quickaccess to cache using ..value --- backend/app_sdk/app_base.py | 14 ++++++++++- backend/go-app/go.mod | 2 +- backend/go-app/go.sum | 2 ++ backend/go-app/walkoff.go | 2 +- functions/onprem/worker/go.mod | 3 +-- functions/onprem/worker/worker.go | 42 ++++++++++++++++++++++++------- 6 files changed, 51 insertions(+), 14 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index c1ddb498..473d2e1f 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -728,7 +728,6 @@ class AppBase: allvalues["value"] = parsedvalue except: print("Parsing of value as JSON failed") - return {"success": False} return allvalues except: @@ -1362,12 +1361,25 @@ class AppBase: appendresult += char actionname_lower = "exec" + elif actionname_lower.startswith("shuffle_cache "): + actionname_lower = "shuffle_cache" actionname_lower = actionname_lower.replace(" ", "_", -1) try: if actionname_lower == "exec" or actionname_lower == "webhook" or actionname_lower == "schedule" or actionname_lower == "userinput" or actionname_lower == "email_trigger" or actionname_lower == "trigger": baseresult = execution_data["execution_argument"] + elif actionname_lower == "shuffle_cache": + print("SHOULD GET CACHE KEY: %s" % parsersplit) + if len(parsersplit) > 1: + actual_key = parsersplit[1] + print("KEY: %s" % actual_key) + cachedata = self.get_cache(actual_key) + print("CACHE: %s" % cachedata) + parsersplit.pop(1) + baseresult = cachedata + + #returndata = str(baseresult)+str(appendresult) else: #print("Within execution data check. Execution data: %s", execution_data["results"]) if execution_data["results"] != None: diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index facd92e4..50c3f2da 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -24,7 +24,7 @@ require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 // indirect github.com/elastic/go-elasticsearch/v8 v8.0.0-20210519083322-55daf7425ecb // indirect github.com/frikky/kin-openapi v0.39.0 - github.com/frikky/shuffle-shared v0.0.56 + github.com/frikky/shuffle-shared v0.0.57 github.com/fsouza/go-dockerclient v1.7.2 // indirect github.com/ghodss/yaml v1.0.0 github.com/go-git/go-billy/v5 v5.0.0 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index a2728936..712aabf3 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -169,6 +169,8 @@ github.com/frikky/shuffle-shared v0.0.54 h1:rc8JcavY6uDxaIkXFwLVotfv3/epXUzuCWzR github.com/frikky/shuffle-shared v0.0.54/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= github.com/frikky/shuffle-shared v0.0.56 h1:stC793SdQeBh98yJqCL74aXFo9YYhVIg8CQJ7hm4d6o= github.com/frikky/shuffle-shared v0.0.56/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= +github.com/frikky/shuffle-shared v0.0.57 h1:YDlOVjg8bUBcinehcmorxzOoV5O55oUfG8q7TOiBOeU= +github.com/frikky/shuffle-shared v0.0.57/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= github.com/fsouza/go-dockerclient v1.7.2 h1:bBEAcqLTkpq205jooP5RVroUKiVEWgGecHyeZc4OFjo= github.com/fsouza/go-dockerclient v1.7.2/go.mod h1:+ugtMCVRwnPfY7d8/baCzZ3uwB0BrG5DB8OzbtxaRz8= github.com/getkin/kin-openapi v0.8.0 h1:a6TQjTqwkyscC4/hShJX7WhCVE+4bi9lzw61XHQW5hE= diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 899e40d5..480ea228 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -971,7 +971,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } //log.Printf("BASE LENGTH: %d", len(workflowExecution.Results)) - workflowExecution, dbSave, err := shuffle.ParsedExecutionResult(ctx, *workflowExecution, actionResult) + workflowExecution, dbSave, err := shuffle.ParsedExecutionResult(ctx, *workflowExecution, actionResult, false) if err != nil { log.Printf("[ERROR] Failed execution of parsedexecution: %s", err) resp.WriteHeader(401) diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index ba5bbeea..31879e60 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -13,7 +13,7 @@ require ( github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/elastic/go-elasticsearch/v8 v8.0.0-20210531084204-f01628963386 // indirect - github.com/frikky/shuffle-shared v0.0.56 + github.com/frikky/shuffle-shared v0.0.59 github.com/fsouza/go-dockerclient v1.7.2 github.com/go-git/go-billy/v5 v5.3.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -23,6 +23,5 @@ require ( github.com/opencontainers/image-spec v1.0.1 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 // indirect - github.com/sirupsen/logrus v1.8.1 // indirect google.golang.org/grpc v1.37.1 // indirect ) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 621035d9..862fcd54 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -518,7 +518,7 @@ func removeIndex(s []string, i int) []string { } func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { - log.Printf("Inside execution results with %d / %d results", len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)) + log.Printf("[INFO] Inside execution results with %d / %d results", len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)) if len(startAction) == 0 { startAction = workflowExecution.Start if len(startAction) == 0 { @@ -567,7 +567,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { if isSkipped { //log.Printf("Skipping %s as all parents are done", item.Action.Label) if !arrayContains(visited, item.Action.ID) { - log.Printf("[INFO] Adding visited (1): %s", item.Action.Label) + log.Printf("[INFO] Adding visited (1): %s\n", item.Action.Label) visited = append(visited, item.Action.ID) } } else { @@ -576,7 +576,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { } } else { if item.Status == "FINISHED" { - log.Printf("[INFO] Adding visited (2): %s", item.Action.Label) + log.Printf("[INFO] Adding visited (2): %s\n", item.Action.Label) visited = append(visited, item.Action.ID) } } @@ -1100,7 +1100,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { } } - log.Printf("[INFO] Adding visited (3): %s", action.Label) + log.Printf("[INFO] Adding visited (3): %s\n", action.Label) visited = append(visited, action.ID) executed = append(executed, action.ID) @@ -1553,12 +1553,36 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl resultLength := len(workflowExecution.Results) setExecution := true - workflowExecution, dbSave, err := shuffle.ParsedExecutionResult(ctx, *workflowExecution, actionResult) + workflowExecution, dbSave, err := shuffle.ParsedExecutionResult(ctx, *workflowExecution, actionResult, true) if err != nil { - log.Printf("[ERROR] Failed execution of parsedexecution: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting execution"}`))) - return + log.Printf("[DEBUG] Rerunning transaction? %s", err) + if strings.Contains(fmt.Sprintf("%s", err), "Rerun this transaction") { + workflowExecution, err := getWorkflowExecution(ctx, workflowExecutionId) + if err != nil { + log.Printf("[ERROR] Failed getting execution cache (2): %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting execution (2)"}`))) + return + } + + resultLength = len(workflowExecution.Results) + setExecution = true + + workflowExecution, dbSave, err = shuffle.ParsedExecutionResult(ctx, *workflowExecution, actionResult, false) + if err != nil { + log.Printf("[ERROR] Failed execution of parsedexecution (2): %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting execution (2)"}`))) + return + } else { + log.Printf("[DEBUG] Successfully got ParsedExecution with %d results!", len(workflowExecution.Results)) + } + } else { + log.Printf("[ERROR] Failed execution of parsedexecution: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting execution"}`))) + return + } } //log.Printf(`[INFO] Got result %s from %s`, actionResult.Status, actionResult.Action.ID) //dbSave := false From 60eeebb21054c539d68030de8b651e83e430c64d Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 7 Jun 2021 10:56:08 +0200 Subject: [PATCH 84/96] Added initial hybrid proxy setup --- backend/app_sdk/app_base.py | 7 +++++-- backend/go-app/main.go | 29 +++++++++++++++++++++++--- frontend/src/views/Admin.jsx | 10 ++++----- frontend/src/views/AngularWorkflow.jsx | 2 +- frontend/src/views/Workflows.jsx | 4 ++-- 5 files changed, 39 insertions(+), 13 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 473d2e1f..b270e8f3 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -727,7 +727,7 @@ class AppBase: parsedvalue = json.loads(allvalues["value"]) allvalues["value"] = parsedvalue except: - print("Parsing of value as JSON failed") + print("Parsing of value as JSON failed. Continue anyway!") return allvalues except: @@ -1377,7 +1377,10 @@ class AppBase: cachedata = self.get_cache(actual_key) print("CACHE: %s" % cachedata) parsersplit.pop(1) - baseresult = cachedata + try: + baseresult = json.dumps(cachedata) + except json.decoder.JSONDecodeError as e: + print("Failed json dumping: %s" % e) #returndata = str(baseresult)+str(appendresult) else: diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 37f79b0c..71816cf6 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -2106,7 +2106,11 @@ func executeCloudAction(action shuffle.CloudSyncJob, apikey string) error { return err } - client := &http.Client{} + transport := http.DefaultTransport.(*http.Transport).Clone() + client := &http.Client{ + Transport: transport, + } + syncUrl := fmt.Sprintf("%s/api/v1/cloud/sync/handle_action", syncUrl) req, err := http.NewRequest( "POST", @@ -3828,6 +3832,22 @@ func remoteOrgJobHandler(org shuffle.Org, interval int) error { return nil } +func runInitCloudSetup() { + action := shuffle.CloudSyncJob{ + Type: "setup", + Action: "init", + OrgId: "", + PrimaryItemId: "", + } + + err := executeCloudAction(action, "") + if err != nil { + log.Printf("[INFO] Failed initial setup: %s", err) + } else { + log.Printf("[INFO] Ran initial setup!") + } +} + func runInitEs(ctx context.Context) { log.Printf("[DEBUG] Starting INIT setup (ES)") @@ -3842,7 +3862,8 @@ func runInitEs(ctx context.Context) { return } - log.Printf("Error getting organizations: %s", err) + log.Printf("[DEBUG] Error getting organizations: %s", err) + runInitCloudSetup() } else { // Add all users to it if len(activeOrgs) == 1 { @@ -3850,7 +3871,9 @@ func runInitEs(ctx context.Context) { } if len(activeOrgs) == 0 { - log.Printf(`No orgs. Setting NEW org "default"`) + log.Printf(`[DEBUG] No orgs. Setting NEW org "default"`) + runInitCloudSetup() + //orgSetupName := "default" //orgId := uuid.NewV4().String() //newOrg := shuffle.Org{ diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index 0e629790..f32040fb 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -154,7 +154,7 @@ const Admin = (props) => { // Need to wait because query in ES is too fast setTimeout(() => { getAppAuthentication() - }, 500) + }, 1000) alert.success("Successfully deleted authentication!") } }), @@ -228,7 +228,7 @@ const Admin = (props) => { .then((responseJson) => { setTimeout(() => { handleGetOrg(org_id) - }, 500) + }, 1000) }) .catch(error => { alert.error("Err: " + error.toString()) @@ -356,7 +356,7 @@ const Admin = (props) => { setSelectedUserModalOpen(false) setTimeout(() => { getAppAuthentication() - }, 500) + }, 1000) } }), ) @@ -514,7 +514,7 @@ const Admin = (props) => { setModalOpen(false) setTimeout(() => { getUsers() - }, 500) + }, 1000) } }), ) @@ -549,7 +549,7 @@ const Admin = (props) => { setModalOpen(false) setTimeout(() => { getUsers() - }, 500) + }, 1000) } }), ) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index a51195a7..1c10af83 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -308,7 +308,7 @@ const AngularWorkflow = (props) => { } if (param.name === "startnode" && outersub.id !== undefined) { - console.log("SHOULD SET STARTNODE: ", outersub) + console.log("SHOULD SET STARTNODE IN SUBFLOW SELECTION: ", outersub) const innernode = outersub.actions.find(action => action.id === param.value) console.log("FOUND NODE: ", innernode) if (innernode !== undefined && subworkflowStartnode.id !== innernode.id) { diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index d75ff690..d3497e4b 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -531,7 +531,7 @@ const Workflows = (props) => { deleteWorkflow(selectedWorkflowId) setTimeout(() => { getAvailableWorkflows() - }, 500) + }, 1000) } setDeleteModalOpen(false) }} color="primary"> @@ -1491,7 +1491,7 @@ const Workflows = (props) => { // Update :) setTimeout(() => { getAvailableWorkflows() - }, 500) + }, 1000) setImportLoading(false) } else { alert.info("Successfully changed basic info for workflow") From 2e2769dabc2921e1b7e077f692b0fefb7d4981aa Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 8 Jun 2021 12:19:18 +0200 Subject: [PATCH 85/96] Fixed bug in new version of opensearch --- .env | 6 +++--- backend/go-app/go.mod | 4 ++-- backend/go-app/main.go | 18 +++++++++--------- docker-compose.yml | 8 ++++---- .../extensions/scripts/disable_disk_check.sh | 4 ++-- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.env b/.env index 9fadb1fd..131dcc43 100644 --- a/.env +++ b/.env @@ -51,9 +51,9 @@ SHUFFLE_ELASTIC=true # DATABASE CONFIGURATIONS DATASTORE_EMULATOR_HOST=shuffle-database:8000 -SHUFFLE_OPENSEARCH_URL=http://shuffle-opensearch:9200 -SHUFFLE_OPENSEARCH_USERNAME= -SHUFFLE_OPENSEARCH_PASSWORD= +SHUFFLE_OPENSEARCH_URL=https://shuffle-opensearch:9200 +SHUFFLE_OPENSEARCH_USERNAME=admin +SHUFFLE_OPENSEARCH_PASSWORD=admin SHUFFLE_OPENSEARCH_CERTIFICATE_FILE= SHUFFLE_OPENSEARCH_APIKEY= SHUFFLE_OPENSEARCH_CLOUDID= diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 50c3f2da..358e63c6 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -2,7 +2,7 @@ module shuffle go 1.13 -replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared +//replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi @@ -24,7 +24,7 @@ require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 // indirect github.com/elastic/go-elasticsearch/v8 v8.0.0-20210519083322-55daf7425ecb // indirect github.com/frikky/kin-openapi v0.39.0 - github.com/frikky/shuffle-shared v0.0.57 + github.com/frikky/shuffle-shared v0.0.60 github.com/fsouza/go-dockerclient v1.7.2 // indirect github.com/ghodss/yaml v1.0.0 github.com/go-git/go-billy/v5 v5.0.0 diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 71816cf6..3ca3ff67 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3082,7 +3082,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("[INFO] TRY TO SET APP TO LIVE!!!") + //log.Printf("[INFO] TRY TO SET APP TO LIVE!!!") user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in verify swagger: %s", err) @@ -3136,7 +3136,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("[INFO] EDITING APP WITH ID %s and md5 %s", app.ID, newmd5) + log.Printf("[INFO] %s is EDITING APP WITH ID %s and md5 %s", user.Id, app.ID, newmd5) newmd5 = app.ID } @@ -3944,9 +3944,9 @@ func runInitEs(ctx context.Context) { schedules, err := shuffle.GetAllSchedules(ctx, "ALL") if err != nil { - log.Printf("Failed getting schedules during service init: %s", err) + log.Printf("[WARNING] Failed getting schedules during service init: %s", err) } else { - log.Printf("Setting up %d schedule(s)", len(schedules)) + log.Printf("[INFO] Setting up %d schedule(s)", len(schedules)) url := &url.URL{} for _, schedule := range schedules { if schedule.Environment == "cloud" { @@ -3983,7 +3983,7 @@ func runInitEs(ctx context.Context) { users, err := shuffle.GetAllUsers(ctx) if len(users) == 0 { - log.Printf("Trying to set up user based on environments SHUFFLE_DEFAULT_USERNAME & SHUFFLE_DEFAULT_PASSWORD") + log.Printf("[INFO] Trying to set up user based on environments SHUFFLE_DEFAULT_USERNAME & SHUFFLE_DEFAULT_PASSWORD") username := os.Getenv("SHUFFLE_DEFAULT_USERNAME") password := os.Getenv("SHUFFLE_DEFAULT_PASSWORD") if len(username) == 0 || len(password) == 0 { @@ -4005,7 +4005,7 @@ func runInitEs(ctx context.Context) { } _ = setUsers - log.Printf("Starting cloud schedules for orgs!") + log.Printf("[INFO] Starting cloud schedules for orgs if enabled!") type requestStruct struct { ApiKey string `json:"api_key"` } @@ -4023,7 +4023,7 @@ func runInitEs(ctx context.Context) { continue } - log.Printf("Should start schedule for org %s", org.Name) + log.Printf("[DEBUG] Should start schedule for org %s", org.Name) job := func() { err := remoteOrgJobHandler(org, interval) if err != nil { @@ -4614,7 +4614,7 @@ func runInit(ctx context.Context) { continue } - log.Printf("Should start schedule for org %s", org.Name) + log.Printf("[DEBUG] Should start schedule for org %s", org.Name) job := func() { err := remoteOrgJobHandler(org, interval) if err != nil { @@ -5568,7 +5568,7 @@ func initHandlers() { for { _, err = shuffle.RunInit(*dbclient, *es, storage.Client{}, gceProject, "onprem", true, elasticConfig) if err != nil { - log.Printf("[DEBUG] Error in initial database connection. Retrying in 5 seconds. %s", err) + log.Printf("[ERROR] Error in initial database connection. Retrying in 5 seconds. %s", err) time.Sleep(5 * time.Second) continue } diff --git a/docker-compose.yml b/docker-compose.yml index 3d15f326..71bb1286 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: frontend: - build: ./frontend + #build: ./frontend image: ghcr.io/frikky/shuffle-frontend:nightly container_name: shuffle-frontend hostname: shuffle-frontend @@ -16,7 +16,7 @@ services: depends_on: - backend backend: - build: ./backend + #build: ./backend image: ghcr.io/frikky/shuffle-backend:nightly container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} @@ -66,14 +66,14 @@ services: - CLEANUP=${SHUFFLE_CONTAINER_AUTO_CLEANUP} restart: unless-stopped opensearch: - image: opensearchproject/opensearch:latest + image: opensearchproject/opensearch:1.0.0-rc1 hostname: shuffle-opensearch container_name: shuffle-opensearch environment: - bootstrap.memory_lock=true - "OPENSEARCH_JAVA_OPTS=-Xms1024m -Xmx1024m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM - - opendistro_security.disabled=true - discovery.seed_hosts=shuffle-opensearch + - cluster.routing.allocation.disk.threshold_enabled=false ulimits: memlock: soft: -1 diff --git a/functions/extensions/scripts/disable_disk_check.sh b/functions/extensions/scripts/disable_disk_check.sh index 3b3ddcc6..cadda899 100644 --- a/functions/extensions/scripts/disable_disk_check.sh +++ b/functions/extensions/scripts/disable_disk_check.sh @@ -1,11 +1,11 @@ -curl -XPUT http://localhost:9200/_cluster/settings -H "Content-Type:application/json" -d \ +curl -XPUT -u admin:admin https://localhost:9200/_cluster/settings -H "Content-Type:application/json" -k -d \ '{ "transient": { "cluster.routing.allocation.disk.threshold_enabled": false } }' -curl -XPUT http://localhost:9200/_all/_settings -H "Content-Type: application/json" -d \ +curl -XPUT -u admin:admin https://localhost:9200/_all/_settings -H "Content-Type: application/json" -k -d \ '{ "index.blocks.read_only_allow_delete": null }' From 5b3c4b7db17dadfd5813fe1b140015004d7ee5f4 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 8 Jun 2021 17:15:26 +0200 Subject: [PATCH 86/96] Rolled back database config to beta-1 --- .env | 7 ++++--- docker-compose.yml | 8 ++++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.env b/.env index 131dcc43..7e9ffd78 100644 --- a/.env +++ b/.env @@ -51,9 +51,10 @@ SHUFFLE_ELASTIC=true # DATABASE CONFIGURATIONS DATASTORE_EMULATOR_HOST=shuffle-database:8000 -SHUFFLE_OPENSEARCH_URL=https://shuffle-opensearch:9200 -SHUFFLE_OPENSEARCH_USERNAME=admin -SHUFFLE_OPENSEARCH_PASSWORD=admin +#SHUFFLE_OPENSEARCH_URL=https://shuffle-opensearch:9200 +SHUFFLE_OPENSEARCH_URL=http://shuffle-opensearch:9200 +SHUFFLE_OPENSEARCH_USERNAME= #admin +SHUFFLE_OPENSEARCH_PASSWORD= #admin SHUFFLE_OPENSEARCH_CERTIFICATE_FILE= SHUFFLE_OPENSEARCH_APIKEY= SHUFFLE_OPENSEARCH_CLOUDID= diff --git a/docker-compose.yml b/docker-compose.yml index 71bb1286..c0100d39 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -66,14 +66,18 @@ services: - CLEANUP=${SHUFFLE_CONTAINER_AUTO_CLEANUP} restart: unless-stopped opensearch: - image: opensearchproject/opensearch:1.0.0-rc1 + image: opensearchproject/opensearch:1.0.0-beta1 hostname: shuffle-opensearch container_name: shuffle-opensearch environment: - bootstrap.memory_lock=true - "OPENSEARCH_JAVA_OPTS=-Xms1024m -Xmx1024m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM - - discovery.seed_hosts=shuffle-opensearch + - opendistro_security.disabled=true - cluster.routing.allocation.disk.threshold_enabled=false + - cluster.name=shuffle-cluster + - node.name=shuffle-opensearch + - discovery.seed_hosts=shuffle-opensearch + - cluster.initial_master_nodes=shuffle-opensearch ulimits: memlock: soft: -1 From 1dddcb5dfc55023a3d3556b9caee039f5336a8c3 Mon Sep 17 00:00:00 2001 From: frikky Date: Wed, 9 Jun 2021 06:06:18 +0200 Subject: [PATCH 87/96] Fixed issue with badly parsed characters for OpenAPI --- backend/go-app/go.mod | 2 +- backend/go-app/go.sum | 2 ++ backend/go-app/main.go | 1 + frontend/src/views/AngularWorkflow.jsx | 2 +- frontend/src/views/AppCreator.jsx | 2 +- frontend/src/views/Apps.jsx | 2 +- frontend/src/views/LoginPage.jsx | 4 ++-- 7 files changed, 9 insertions(+), 6 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 358e63c6..89e77cd3 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -2,7 +2,7 @@ module shuffle go 1.13 -//replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared +replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 712aabf3..67cc7a23 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -171,6 +171,8 @@ github.com/frikky/shuffle-shared v0.0.56 h1:stC793SdQeBh98yJqCL74aXFo9YYhVIg8CQJ github.com/frikky/shuffle-shared v0.0.56/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= github.com/frikky/shuffle-shared v0.0.57 h1:YDlOVjg8bUBcinehcmorxzOoV5O55oUfG8q7TOiBOeU= github.com/frikky/shuffle-shared v0.0.57/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= +github.com/frikky/shuffle-shared v0.0.60 h1:o6/QLsu3Rbjr4+BQWs9DF4B5qZCg0gPpZWPYXmjVWqM= +github.com/frikky/shuffle-shared v0.0.60/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= github.com/fsouza/go-dockerclient v1.7.2 h1:bBEAcqLTkpq205jooP5RVroUKiVEWgGecHyeZc4OFjo= github.com/fsouza/go-dockerclient v1.7.2/go.mod h1:+ugtMCVRwnPfY7d8/baCzZ3uwB0BrG5DB8OzbtxaRz8= github.com/getkin/kin-openapi v0.8.0 h1:a6TQjTqwkyscC4/hShJX7WhCVE+4bi9lzw61XHQW5hE= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 3ca3ff67..6442b234 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3168,6 +3168,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { swagger, err := swaggerLoader.LoadSwaggerFromData(body) if err != nil { log.Printf("[ERROR] Swagger validation error: %s", err) + //log.Printf("%s", string(body)) resp.WriteHeader(500) resp.Write([]byte(`{"success": false, "reason": "Failed verifying openapi"}`)) return diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 1c10af83..91da6cdc 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -2102,7 +2102,7 @@ const AngularWorkflow = (props) => { // Basically just a stupid if-else :) const synonyms = { - "id": ["id", "ref", "sourceref", "reference", "sourcereference", "alert id", "case id", "incident id", "service id", "sid", "uid", "uuid"], + "id": ["id", "ref", "sourceref", "reference", "sourcereference", "alert id", "case id", "incident id", "service id", "sid", "uid", "uuid", "team id"], "title": ["title", "name", "message"], "description": ["description", "explanation", "story", "details",], "email": ["mail", "email", "sender", "receiver", "recipient"], diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 6088f02f..24006b3a 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -970,7 +970,7 @@ const AppCreator = (props) => { const basePath = "/"+(splitBase.slice(3, )).join("/") const data = { - "swagger": "3.0", + "openapi": "3.0", "info": { "title": name, "description": description, diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 3b802b12..d5d2b828 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -1639,7 +1639,7 @@ const Apps = (props) => { hidden type="file" ref={upload} - accept="application/JSON, text/yaml, text/x-yaml, application/x-yaml, application/vnd.yaml" + accept="application/JSON, application/YAML, text/yaml, text/x-yaml, application/x-yaml, application/vnd.yaml" multiple={false} onChange={uploadFile} /> diff --git a/frontend/src/views/LoginPage.jsx b/frontend/src/views/LoginPage.jsx index 9f128551..dac41677 100644 --- a/frontend/src/views/LoginPage.jsx +++ b/frontend/src/views/LoginPage.jsx @@ -68,7 +68,7 @@ const LoginDialog = props => { }), ) .catch(error => { - setLoginInfo("Error logging in - please refresh in a minute: ", error) + setLoginInfo("Error logging in - please refresh in a minute ", error) }) } @@ -139,7 +139,7 @@ const LoginDialog = props => { }), ) .catch(error => { - setLoginInfo("Error in userdata: ", error) + setLoginInfo("Error in from backend: ", error) }); } } From d19eb5fcaab273383ec161dd8ee6db661a7ed24e Mon Sep 17 00:00:00 2001 From: frikky Date: Thu, 10 Jun 2021 09:38:06 +0200 Subject: [PATCH 88/96] Added basic fix for orborus execution --- .env | 4 +- backend/app_sdk/app_base.py | 11 +- backend/app_sdk/build.sh | 3 +- backend/go-app/go.mod | 2 +- backend/go-app/main.go | 85 +++++- backend/go-app/walkoff.go | 397 +--------------------------- docker-compose.yml | 2 +- frontend/src/views/Workflows.jsx | 2 +- functions/onprem/orborus/orborus.go | 3 +- 9 files changed, 88 insertions(+), 421 deletions(-) diff --git a/.env b/.env index 7e9ffd78..be6f9de0 100644 --- a/.env +++ b/.env @@ -53,8 +53,8 @@ SHUFFLE_ELASTIC=true DATASTORE_EMULATOR_HOST=shuffle-database:8000 #SHUFFLE_OPENSEARCH_URL=https://shuffle-opensearch:9200 SHUFFLE_OPENSEARCH_URL=http://shuffle-opensearch:9200 -SHUFFLE_OPENSEARCH_USERNAME= #admin -SHUFFLE_OPENSEARCH_PASSWORD= #admin +SHUFFLE_OPENSEARCH_USERNAME= +SHUFFLE_OPENSEARCH_PASSWORD= SHUFFLE_OPENSEARCH_CERTIFICATE_FILE= SHUFFLE_OPENSEARCH_APIKEY= SHUFFLE_OPENSEARCH_CLOUDID= diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index b270e8f3..8c587b6c 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -1440,7 +1440,7 @@ class AppBase: print("[INFO] After second return") if len(parsersplit) == 1: returndata = str(baseresult)+str(appendresult) - print("RETURNING: %s" % returndata) + print("RETURNING!")#: %s" % returndata) return returndata, False baseresult = baseresult.replace(" True,", " true,") @@ -1523,7 +1523,8 @@ class AppBase: # Trying without string dumping. value, is_loop = get_json_value(fullexecution, to_be_replaced) - print("\n\nType of value: %s. Value: %s" % (type(value), value)) + #print("\n\nType of value: %s. Value: %s" % (type(value), value)) + print("\n\nType of value: %s" % type(value)) if isinstance(value, str): parameter["value"] = parameter["value"].replace(to_be_replaced, value) elif isinstance(value, dict) or isinstance(value, list): @@ -1543,8 +1544,7 @@ class AppBase: except json.decoder.JSONDecodeError as e: parameter["value"] = parameter["value"].replace(to_be_replaced, value) - print("VALUE: %s" % parameter["value"]) - + #print("VALUE: %s" % parameter["value"]) if parameter["variant"] == "WORKFLOW_VARIABLE": print("Handling workflow variable") @@ -2326,7 +2326,8 @@ class AppBase: raise e #break - print("\n[INFO] Returned from execution with %s of types %s" % (newres, type(newres)))#, newres) + print("\n[INFO] Returned from execution with types %s" % type(newres)) + #print("\n[INFO] Returned from execution with %s of types %s" % (newres, type(newres)))#, newres) if isinstance(newres, tuple): print("[INFO] Handling return as tuple") # Handles files. diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 79b62908..e0a95374 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,6 +1,6 @@ #!/bin/bash NAME=shuffle-app_sdk -VERSION=0.8.99 +VERSION=0.8.100 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION @@ -12,3 +12,4 @@ docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg. docker push frikky/shuffle:app_sdk docker push ghcr.io/frikky/$NAME:$VERSION +docker push ghcr.io/frikky/$NAME:nightly diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 89e77cd3..70bdb567 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -24,7 +24,7 @@ require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 // indirect github.com/elastic/go-elasticsearch/v8 v8.0.0-20210519083322-55daf7425ecb // indirect github.com/frikky/kin-openapi v0.39.0 - github.com/frikky/shuffle-shared v0.0.60 + github.com/frikky/shuffle-shared v0.0.61 github.com/fsouza/go-dockerclient v1.7.2 // indirect github.com/ghodss/yaml v1.0.0 github.com/go-git/go-billy/v5 v5.0.0 diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 6442b234..6efc8f90 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -793,8 +793,14 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) { } else { log.Printf("[DEBUG] Successfully created the default org!") + defaultEnv := os.Getenv("ORG_ID") + if len(defaultEnv) == 0 { + defaultEnv = "Shuffle" + log.Printf("[DEBUG] Setting default environment for org to %s", defaultEnv) + } + item := shuffle.Environment{ - Name: "Shuffle", + Name: defaultEnv, Type: "onprem", OrgId: orgId, Default: true, @@ -3852,6 +3858,12 @@ func runInitCloudSetup() { func runInitEs(ctx context.Context) { log.Printf("[DEBUG] Starting INIT setup (ES)") + defaultEnv := os.Getenv("ORG_ID") + if len(defaultEnv) == 0 { + defaultEnv = "Shuffle" + log.Printf("[DEBUG] Setting default environment for org to %s", defaultEnv) + } + log.Printf("[DEBUG] Getting organizations") activeOrgs, err := shuffle.GetAllOrgs(ctx) setUsers := false @@ -3894,7 +3906,7 @@ func runInitEs(ctx context.Context) { // setUsers = true // item := shuffle.Environment{ - // Name: "Shuffle", + // Name: defaultEnv, // Type: "onprem", // OrgId: orgId, // Default: true, @@ -3987,20 +3999,58 @@ func runInitEs(ctx context.Context) { log.Printf("[INFO] Trying to set up user based on environments SHUFFLE_DEFAULT_USERNAME & SHUFFLE_DEFAULT_PASSWORD") username := os.Getenv("SHUFFLE_DEFAULT_USERNAME") password := os.Getenv("SHUFFLE_DEFAULT_PASSWORD") - if len(username) == 0 || len(password) == 0 { - log.Printf("SHUFFLE_DEFAULT_USERNAME and SHUFFLE_DEFAULT_PASSWORD not defined as environments. Running without default user.") + + if len(username) == 0 || len(password) == 0 || len(activeOrgs) > 0 { + log.Printf("[DEBUG] SHUFFLE_DEFAULT_USERNAME and SHUFFLE_DEFAULT_PASSWORD not defined as environments. Running without default user.") } else { apikey := os.Getenv("SHUFFLE_DEFAULT_APIKEY") - tmpOrg := shuffle.OrgMini{ - Name: "default", + log.Printf("[DEBUG] Creating org for default user %s", username) + orgId := uuid.NewV4().String() + orgSetupName := "default" + newOrg := shuffle.Org{ + Name: orgSetupName, + Id: orgId, + Org: orgSetupName, + Users: []shuffle.User{}, + Roles: []string{"admin", "user"}, + CloudSync: false, } - err = createNewUser(username, password, "admin", apikey, tmpOrg) + err = shuffle.SetOrg(ctx, newOrg, orgId) + setUsers := false if err != nil { - log.Printf("Failed to create default user %s: %s", username, err) + log.Printf("[WARNING] Failed setting organization when creating original user: %s", err) } else { - log.Printf("Successfully created user %s", username) + log.Printf("[DEBUG] Successfully created the default org with id %s!", orgId) + setUsers = true + + item := shuffle.Environment{ + Name: defaultEnv, + Type: "onprem", + OrgId: orgId, + Default: true, + Id: uuid.NewV4().String(), + } + + err = shuffle.SetEnvironment(ctx, &item) + if err != nil { + log.Printf("[WARNING] Failed setting up new environment") + } + } + + if setUsers { + tmpOrg := shuffle.OrgMini{ + Name: orgSetupName, + Id: orgId, + } + + err = createNewUser(username, password, "admin", apikey, tmpOrg) + if err != nil { + log.Printf("[INFO] Failed to create default user %s: %s", username, err) + } else { + log.Printf("[INFO] Successfully created user %s", username) + } } } } @@ -4134,7 +4184,7 @@ func runInitEs(ctx context.Context) { log.Printf("[INFO] Finished downloading extra API samples") } - log.Printf("[INFO] Finished INIT") + log.Printf("[INFO] Finished INIT (ES)") } // Handles configuration items during Shuffle startup @@ -4206,7 +4256,7 @@ func runInit(ctx context.Context) { log.Printf("Organizations exist!") if len(activeOrgs) == 0 { - log.Printf(`No orgs. Setting org "default"`) + log.Printf(`[DEBUG] No orgs. Setting org "default"`) orgSetupName := "default" orgId := uuid.NewV4().String() newOrg := shuffle.Org{ @@ -4230,7 +4280,7 @@ func runInit(ctx context.Context) { if len(activeOrgs) == 1 { if len(activeOrgs[0].Users) == 0 { - log.Printf("ORG doesn't have any users??") + log.Printf("[WARNING] ORG doesn't have any users??") q := datastore.NewQuery("Users") var users []shuffle.User @@ -4424,8 +4474,15 @@ func runInit(ctx context.Context) { count, err := shuffle.GetEnvironmentCount() if count == 0 && err == nil && len(activeOrgs) == 1 { log.Printf("[INFO] Setting up environment with org %s", activeOrgs[0].Id) + + defaultEnv := os.Getenv("ORG_ID") + if len(defaultEnv) == 0 { + defaultEnv = "Shuffle" + log.Printf("[DEBUG] Setting default environment for org to %s", defaultEnv) + } + item := shuffle.Environment{ - Name: "Shuffle", + Name: defaultEnv, Type: "onprem", OrgId: activeOrgs[0].Id, Default: true, @@ -4761,7 +4818,7 @@ func runInit(ctx context.Context) { log.Printf("[INFO] Downloading OpenAPI data for search - EXTRA APPS") apis := "https://github.com/frikky/security-openapis" - // THis gets memory problems hahah + // FIXME: This part gets memory problems. Fix in the future to load these apps too. //apis := "https://github.com/APIs-guru/openapi-directory" fs := memfs.New() storer := memory.NewStorage() diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 480ea228..adc06f5b 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -22,12 +22,9 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/client" - scheduler "cloud.google.com/go/scheduler/apiv1" gyaml "github.com/ghodss/yaml" "github.com/h2non/filetype" uuid "github.com/satori/go.uuid" - "google.golang.org/api/cloudfunctions/v1" - schedulerpb "google.golang.org/genproto/googleapis/cloud/scheduler/v1" newscheduler "github.com/carlescere/scheduler" "github.com/frikky/kin-openapi/openapi3" @@ -48,7 +45,7 @@ var localBase = "http://localhost:5001" var baseEnvironment = "onprem" var cloudname = "cloud" -var defaultLocation = "europe-west2" + var scheduledJobs = map[string]*newscheduler.Job{} var scheduledOrgs = map[string]*newscheduler.Job{} @@ -1530,8 +1527,8 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request //workflowExecution.Stream = "tmp" //workflowExecution.WorkflowQueue = "tmp" //workflowExecution.SubscriptionNameNodestream = "testcompany-nodestream" + //workflowExecution.Locations = []string{"europe-west2"} workflowExecution.ProjectId = gceProject - workflowExecution.Locations = []string{"europe-west2"} workflowExecution.WorkflowId = workflow.ID workflowExecution.StartedAt = int64(time.Now().Unix()) workflowExecution.CompletedAt = 0 @@ -2401,26 +2398,6 @@ func deleteSchedule(ctx context.Context, id string) error { return nil } -func deleteScheduleGCP(ctx context.Context, id string) error { - c, err := scheduler.NewCloudSchedulerClient(ctx) - if err != nil { - log.Printf("%s", err) - return err - } - - req := &schedulerpb.DeleteJobRequest{ - Name: fmt.Sprintf("projects/%s/locations/europe-west2/jobs/schedule_%s", gceProject, id), - } - - err = c.DeleteJob(ctx, req) - if err != nil { - log.Printf("%s", err) - return err - } - - return nil -} - func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -2893,151 +2870,6 @@ func validateAppInput(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } -// Deploy to google cloud function :) -func deployCloudFunctionPython(ctx context.Context, name, localization, applocation string, environmentVariables map[string]string) error { - service, err := cloudfunctions.NewService(ctx) - if err != nil { - return err - } - - // ProjectsLocationsListCall - projectsLocationsFunctionsService := cloudfunctions.NewProjectsLocationsFunctionsService(service) - location := fmt.Sprintf("projects/%s/locations/%s", gceProject, localization) - functionName := fmt.Sprintf("%s/functions/%s", location, name) - - cloudFunction := &cloudfunctions.CloudFunction{ - AvailableMemoryMb: 128, - EntryPoint: "authorization", - EnvironmentVariables: environmentVariables, - HttpsTrigger: &cloudfunctions.HttpsTrigger{}, - MaxInstances: 0, - Name: functionName, - Runtime: "python37", - SourceArchiveUrl: applocation, - } - - //getCall := projectsLocationsFunctionsService.Get(fmt.Sprintf("%s/functions/function-5", location)) - //resp, err := getCall.Do() - - createCall := projectsLocationsFunctionsService.Create(location, cloudFunction) - _, err = createCall.Do() - if err != nil { - log.Printf("Failed creating new function. SKIPPING patch, as it probably already exists: %s", err) - - // FIXME - have patching code or nah? - createCall := projectsLocationsFunctionsService.Patch(fmt.Sprintf("%s/functions/%s", location, name), cloudFunction) - _, err = createCall.Do() - if err != nil { - log.Println("Failed patching function") - return err - } - - log.Printf("Successfully patched %s to %s", name, localization) - } else { - log.Printf("Successfully deployed %s to %s", name, localization) - } - - // FIXME - use response to define the HTTPS entrypoint. It's default to an easy one tho - - return nil -} - -// Deploy to google cloud function :) -func deployCloudFunctionGo(ctx context.Context, name, localization, applocation string, environmentVariables map[string]string) error { - service, err := cloudfunctions.NewService(ctx) - if err != nil { - return err - } - - // ProjectsLocationsListCall - projectsLocationsFunctionsService := cloudfunctions.NewProjectsLocationsFunctionsService(service) - location := fmt.Sprintf("projects/%s/locations/%s", gceProject, localization) - functionName := fmt.Sprintf("%s/functions/%s", location, name) - - cloudFunction := &cloudfunctions.CloudFunction{ - AvailableMemoryMb: 128, - EntryPoint: "Authorization", - EnvironmentVariables: environmentVariables, - HttpsTrigger: &cloudfunctions.HttpsTrigger{}, - MaxInstances: 1, - Name: functionName, - Runtime: "go111", - SourceArchiveUrl: applocation, - } - - //getCall := projectsLocationsFunctionsService.Get(fmt.Sprintf("%s/functions/function-5", location)) - //resp, err := getCall.Do() - - createCall := projectsLocationsFunctionsService.Create(location, cloudFunction) - _, err = createCall.Do() - if err != nil { - log.Println("Failed creating new function. Attempting patch, as it might exist already") - - createCall := projectsLocationsFunctionsService.Patch(fmt.Sprintf("%s/functions/%s", location, name), cloudFunction) - _, err = createCall.Do() - if err != nil { - log.Println("Failed patching function") - return err - } - - log.Printf("Successfully patched %s to %s", name, localization) - } else { - log.Printf("Successfully deployed %s to %s", name, localization) - } - - // FIXME - use response to define the HTTPS entrypoint. It's default to an easy one tho - - return nil -} - -// Deploy to google cloud function :) -func deployWebhookFunction(ctx context.Context, name, localization, applocation string, environmentVariables map[string]string) error { - service, err := cloudfunctions.NewService(ctx) - if err != nil { - return err - } - - // ProjectsLocationsListCall - projectsLocationsFunctionsService := cloudfunctions.NewProjectsLocationsFunctionsService(service) - location := fmt.Sprintf("projects/%s/locations/%s", gceProject, localization) - functionName := fmt.Sprintf("%s/functions/%s", location, name) - - cloudFunction := &cloudfunctions.CloudFunction{ - AvailableMemoryMb: 128, - EntryPoint: "Authorization", - EnvironmentVariables: environmentVariables, - HttpsTrigger: &cloudfunctions.HttpsTrigger{}, - MaxInstances: 1, - Name: functionName, - Runtime: "go111", - SourceArchiveUrl: applocation, - } - - //getCall := projectsLocationsFunctionsService.Get(fmt.Sprintf("%s/functions/function-5", location)) - //resp, err := getCall.Do() - - createCall := projectsLocationsFunctionsService.Create(location, cloudFunction) - _, err = createCall.Do() - if err != nil { - log.Println("Failed creating new function. Attempting patch, as it might exist already") - - createCall := projectsLocationsFunctionsService.Patch(fmt.Sprintf("%s/functions/%s", location, name), cloudFunction) - _, err = createCall.Do() - if err != nil { - log.Println("Failed patching function") - return err - } - - log.Printf("Successfully patched %s to %s", name, localization) - } else { - log.Printf("Successfully deployed %s to %s", name, localization) - } - - // FIXME - use response to define the HTTPS entrypoint. It's default to an easy one tho - - return nil -} - func loadGithubWorkflows(url, username, password, userId, branch, orgId string) error { fs := memfs.New() @@ -4083,228 +3915,6 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } -// Starts a new webhook -func handleStopHook(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - user, err := shuffle.HandleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in set new workflowhandler: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - location := strings.Split(request.URL.String(), "/") - - var fileId string - if location[1] == "api" { - if len(location) <= 4 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - if len(fileId) != 32 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Workflow ID when stopping hook is not valid"}`)) - return - } - - ctx := context.Background() - hook, err := shuffle.GetHook(ctx, fileId) - if err != nil { - log.Printf("Failed getting hook %s (stop): %s", fileId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if user.Id != hook.Owner { - log.Printf("Wrong user (%s) for workflow %s", user.Username, hook.Id) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - log.Printf("Status: %s", hook.Status) - log.Printf("Running: %t", hook.Running) - if !hook.Running { - message := fmt.Sprintf("Error: %s isn't running", hook.Id) - log.Println(message) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, message))) - return - } - - hook.Status = "stopped" - hook.Running = false - hook.Actions = []shuffle.HookAction{} - err = shuffle.SetHook(ctx, *hook) - if err != nil { - log.Printf("Failed setting hook: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - image := "webhook" - - // This is here to force stop and remove the old webhook - // FIXME - err = removeWebhookFunction(ctx, fileId) - if err != nil { - log.Printf("Container stop issue for %s-%s: %s", image, fileId, err) - } - - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true, "reason": "Stopped webhook"}`)) -} - -func removeWebhookFunction(ctx context.Context, hookid string) error { - service, err := cloudfunctions.NewService(ctx) - if err != nil { - return err - } - - // ProjectsLocationsListCall - projectsLocationsFunctionsService := cloudfunctions.NewProjectsLocationsFunctionsService(service) - location := fmt.Sprintf("projects/%s/locations/%s", gceProject, defaultLocation) - functionName := fmt.Sprintf("%s/functions/webhook_%s", location, hookid) - - deleteCall := projectsLocationsFunctionsService.Delete(functionName) - resp, err := deleteCall.Do() - if err != nil { - log.Printf("Failed to delete %s from %s: %s", hookid, defaultLocation, err) - return err - } else { - log.Printf("Successfully deleted %s from %s", hookid, defaultLocation) - } - - _ = resp - return nil -} - -func handleStartHook(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - user, err := shuffle.HandleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in set new workflowhandler: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - location := strings.Split(request.URL.String(), "/") - - var fileId string - if location[1] == "api" { - if len(location) <= 4 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - if len(fileId) != 36 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Workflow ID when starting hook is not valid"}`)) - return - } - - ctx := context.Background() - hook, err := shuffle.GetHook(ctx, fileId) - if err != nil { - log.Printf("Failed getting hook %s (start): %s", fileId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if user.Id != hook.Owner { - log.Printf("Wrong user (%s) for workflow %s", user.Username, hook.Id) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - log.Printf("Status: %s", hook.Status) - log.Printf("Running: %t", hook.Running) - if hook.Running || hook.Status == "Running" { - message := fmt.Sprintf("Error: %s is already running", hook.Id) - log.Println(message) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, message))) - return - } - - environmentVariables := map[string]string{ - "FUNCTION_APIKEY": user.ApiKey, - "CALLBACKURL": syncUrl, - "HOOKID": fileId, - } - - applocation := fmt.Sprintf("gs://%s/triggers/webhook.zip", bucketName) - hookname := fmt.Sprintf("webhook_%s", fileId) - err = deployWebhookFunction(ctx, hookname, "europe-west2", applocation, environmentVariables) - if err != nil { - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return - } - - hook.Status = "running" - hook.Running = true - err = shuffle.SetHook(ctx, *hook) - if err != nil { - log.Printf("Failed setting hook: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - log.Printf("Starting function %s?", fileId) - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true, "reason": "Started webhook"}`)) - return -} - -func removeOutlookTriggerFunction(ctx context.Context, triggerId string) error { - service, err := cloudfunctions.NewService(ctx) - if err != nil { - return err - } - - // ProjectsLocationsListCall - projectsLocationsFunctionsService := cloudfunctions.NewProjectsLocationsFunctionsService(service) - location := fmt.Sprintf("projects/%s/locations/%s", gceProject, defaultLocation) - functionName := fmt.Sprintf("%s/functions/outlooktrigger_%s", location, triggerId) - - deleteCall := projectsLocationsFunctionsService.Delete(functionName) - resp, err := deleteCall.Do() - if err != nil { - log.Printf("Failed to delete %s from %s: %s", triggerId, defaultLocation, err) - return err - } else { - log.Printf("Successfully deleted %s from %s", triggerId, defaultLocation) - } - - _ = resp - return nil -} - func handleUserInput(trigger shuffle.Trigger, organizationId string, workflowId string, referenceExecution string) error { // E.g. check email sms := "" @@ -4430,9 +4040,6 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) { return } - //workflowExecution.ProjectId = gceProject - //workflowExecution.Locations = []string{defaultLocation} - environments, err := shuffle.GetEnvironments(ctx, user.ActiveOrg.Id) environment := "Shuffle" if len(environments) >= 1 { diff --git a/docker-compose.yml b/docker-compose.yml index c0100d39..53168f1b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,7 @@ services: depends_on: - backend backend: - #build: ./backend + build: ./backend image: ghcr.io/frikky/shuffle-backend:nightly container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index d3497e4b..d8dae58c 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -92,7 +92,7 @@ export const GetIconInfo = (action) => { {"key": "list", "values": ["list", "head", "options"]}, {"key": "download", "values": ["get", "download", "return", "hello_world", "curl",]}, {"key": "add", "values": ["add"]}, - {"key": "delete", "values": ["delete", "remove"]}, + {"key": "delete", "values": ["delete", "remove", "clear", "clean",]}, {"key": "send", "values": ["send", "dispatch", "mail", "forward", "post", "submit", "mark", "set"]}, {"key": "repeat", "values": ["repeat", "retry", "pause",]}, {"key": "execute", "values": ["execute", "run", "play", "raise",]}, diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index a3b28c4e..5632e579 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -51,7 +51,8 @@ var baseimagetagsuffix = os.Getenv("SHUFFLE_BASE_IMAGE_TAG_SUFFIX") var orgId = os.Getenv("ORG_ID") var baseUrl = os.Getenv("BASE_URL") -var environment = os.Getenv("ENVIRONMENT_NAME") + +//var environment = os.Getenv("ENVIRONMENT_NAME") var dockerApiVersion = os.Getenv("DOCKER_API_VERSION") var runningMode = strings.ToLower(os.Getenv("RUNNING_MODE")) var cleanupEnv = strings.ToLower(os.Getenv("CLEANUP")) From fde84958eec25033521680db8b476855f028f449 Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 14 Jun 2021 02:08:05 +0200 Subject: [PATCH 89/96] Fixed hotloading app comparison overwrites --- backend/go-app/go.mod | 3 +-- backend/go-app/go.sum | 4 ++++ backend/go-app/main.go | 4 +--- backend/go-app/walkoff.go | 16 ++++++++-------- frontend/src/components/ParsedAction.jsx | 22 ++++++++++++---------- functions/onprem/orborus/orborus.go | 4 ++-- functions/onprem/worker/worker.go | 2 +- 7 files changed, 29 insertions(+), 26 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 70bdb567..7439deaf 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -22,9 +22,8 @@ require ( github.com/docker/go-units v0.4.0 // indirect github.com/elastic/go-elasticsearch v0.0.0 // indirect github.com/elastic/go-elasticsearch/v7 v7.12.0 // indirect - github.com/elastic/go-elasticsearch/v8 v8.0.0-20210519083322-55daf7425ecb // indirect github.com/frikky/kin-openapi v0.39.0 - github.com/frikky/shuffle-shared v0.0.61 + github.com/frikky/shuffle-shared v0.0.62 github.com/fsouza/go-dockerclient v1.7.2 // indirect github.com/ghodss/yaml v1.0.0 github.com/go-git/go-billy/v5 v5.0.0 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 67cc7a23..5bf028aa 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -115,6 +115,8 @@ github.com/elastic/go-elasticsearch/v7 v7.12.0 h1:j4tvcMrZJLp39L2NYvBb7f+lHKPqPH github.com/elastic/go-elasticsearch/v7 v7.12.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= github.com/elastic/go-elasticsearch/v8 v8.0.0-20210519083322-55daf7425ecb h1:svC8T5+v+aWpWiTt3nsGfpdqVb4NIWK/WamGXXECBXA= github.com/elastic/go-elasticsearch/v8 v8.0.0-20210519083322-55daf7425ecb/go.mod h1:xe9a/L2aeOgFKKgrO3ibQTnMdpAeL0GC+5/HpGScSa4= +github.com/elastic/go-elasticsearch/v8 v8.0.0-20210608143047-aa1301e7ba9d h1:id0CyeIuvJ9hzYhLlKsXHQ10d/k4G+CexAu53Pl3hf4= +github.com/elastic/go-elasticsearch/v8 v8.0.0-20210608143047-aa1301e7ba9d/go.mod h1:xe9a/L2aeOgFKKgrO3ibQTnMdpAeL0GC+5/HpGScSa4= github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -173,6 +175,8 @@ github.com/frikky/shuffle-shared v0.0.57 h1:YDlOVjg8bUBcinehcmorxzOoV5O55oUfG8q7 github.com/frikky/shuffle-shared v0.0.57/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= github.com/frikky/shuffle-shared v0.0.60 h1:o6/QLsu3Rbjr4+BQWs9DF4B5qZCg0gPpZWPYXmjVWqM= github.com/frikky/shuffle-shared v0.0.60/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= +github.com/frikky/shuffle-shared v0.0.62 h1:1M8y7rX8nQW7072+bUD4vgHcf65AG0kJ8m3ihmY2bPQ= +github.com/frikky/shuffle-shared v0.0.62/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= github.com/fsouza/go-dockerclient v1.7.2 h1:bBEAcqLTkpq205jooP5RVroUKiVEWgGecHyeZc4OFjo= github.com/fsouza/go-dockerclient v1.7.2/go.mod h1:+ugtMCVRwnPfY7d8/baCzZ3uwB0BrG5DB8OzbtxaRz8= github.com/getkin/kin-openapi v0.8.0 h1:a6TQjTqwkyscC4/hShJX7WhCVE+4bi9lzw61XHQW5hE= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 6efc8f90..38660417 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3881,9 +3881,7 @@ func runInitEs(ctx context.Context) { // Add all users to it if len(activeOrgs) == 1 { setUsers = true - } - - if len(activeOrgs) == 0 { + } else if len(activeOrgs) == 0 { log.Printf(`[DEBUG] No orgs. Setting NEW org "default"`) runInitCloudSetup() diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index adc06f5b..e746d7cc 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -3001,9 +3001,9 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - cacheKey := fmt.Sprintf("workflowapps-sorted-100") + cacheKey := fmt.Sprintf("workflowapps-sorted-500") shuffle.DeleteCache(ctx, cacheKey) - cacheKey = fmt.Sprintf("workflowapps-sorted-500") + cacheKey = fmt.Sprintf("workflowapps-sorted-0") shuffle.DeleteCache(ctx, cacheKey) // Just need to be logged in @@ -3535,20 +3535,20 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin appPython := fmt.Sprintf("%s/src/app.py", extra) appPythonReader, err := fs.Open(appPython) if err != nil { - log.Printf("Failed to read %s", appPython) + log.Printf("Failed to read python app %s", appPython) continue } appPythonData, err := ioutil.ReadAll(appPythonReader) if err != nil { - log.Printf("Failed reading %s: %s", appPython, err) + log.Printf("Failed reading appdata %s: %s", appPython, err) continue } dockerFp := fmt.Sprintf("%s/Dockerfile", extra) dockerfile, err := fs.Open(dockerFp) if err != nil { - log.Printf("Failed to read %s", appPython) + log.Printf("Failed to read dockerfil %s", appPython) continue } @@ -3580,9 +3580,9 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin } if len(allapps) == 0 { - allapps, err = shuffle.GetAllWorkflowApps(ctx, 500) + allapps, err = shuffle.GetAllWorkflowApps(ctx, 0) if err != nil { - log.Printf("Failed getting apps to verify: %s", err) + log.Printf("[WARNING] Failed getting apps to verify: %s", err) continue } } @@ -3659,7 +3659,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin err = checkWorkflowApp(workflowapp) if err != nil { - log.Printf("%s for app %s:%s", err, workflowapp.Name, workflowapp.AppVersion) + log.Printf("[DEBUG] %s for app %s:%s", err, workflowapp.Name, workflowapp.AppVersion) continue } diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index 6faf26d3..05cf9fb1 100644 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -96,9 +96,9 @@ const ParsedAction = (props) => { } } - // Remove self - results = results.filter(data => data.id !== action.id) - results = results.filter(data => data.type !== "TRIGGER") + // Some obscure bug made this have to be done.. zz + results = results.filter(data => data !== undefined && action !== undefined && data.id !== action.id) + results = results.filter(data => data !== undefined && data.type !== "TRIGGER") results.push({"label": "Execution Argument", "type": "INTERNAL"}) return results } @@ -767,12 +767,12 @@ const ParsedAction = (props) => { //const str = = data.value.search(submatch) //console.log("FOUND? ", n) - for (var key in keywords) { - const keyword = keywords[key] - if (data.value.includes(keyword)) { - console.log("INCLUDED: ", keyword) - } - } + //for (var key in keywords) { + // const keyword = keywords[key] + // if (data.value.includes(keyword)) { + // console.log("INCLUDED: ", keyword) + // } + //} //const keywords = ["len", "lower", "upper", "trim", "split", "length", "number", "parse", "join"] if (selectedActionParameters[count].schema !== undefined && selectedActionParameters[count].schema !== null && selectedActionParameters[count].schema.type === "file") { @@ -1485,7 +1485,9 @@ const ParsedAction = (props) => { {workflow.execution_variables !== undefined && workflow.execution_variables !== null && workflow.execution_variables.length > 0 ?
- Set execution variable (optional) + + Set execution variable (optional) + { diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 1256ec61..876b4859 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -310,7 +310,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] ShowStdout: true, } - exit := true + exit := false out, err := cli.ContainerLogs(ctx, cont.ID, logOptions) if err != nil { log.Printf("[INFO] Failed getting logs: %s", err) @@ -318,11 +318,13 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] buf := new(strings.Builder) io.Copy(buf, out) logs := buf.String() + + // FIXME: Re-add log tracking which can be sent to backend //allLogs[actionId] = logs - if stats.ContainerJSONBase.State.Status == "exited" && strings.Contains(logs, "Normal execution.") { + if stats.ContainerJSONBase.State.Status == "exited" && !strings.Contains(logs, "Normal execution.") { log.Printf("[WARNING] BAD Execution Logs for %s: %s", actionId, logs) - exit = false + exit = true } } @@ -994,20 +996,33 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } - image = images[2] - err = deployApp(dockercli, image, identifier, env, workflowExecution, action.ID) - if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { - if strings.Contains(err.Error(), "exited prematurely") { - log.Printf("[DEBUG] Shutting down (3)") - shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) - } - - log.Printf("[WARNING] Failed CLEANUP execution. Downloading image %s remotely.", image) - - err := downloadDockerImageBackend(topClient, image) - if err == nil { - log.Printf("[DEBUG] Downloaded image %s from backend (CLEANUP)", image) + err := downloadDockerImageBackend(topClient, image) + executed := false + if err == nil { + log.Printf("[DEBUG] Downloaded image %s from backend (CLEANUP)", image) + //err = deployApp(dockercli, image, identifier, env, workflow, action.ID) + err = deployApp(dockercli, image, identifier, env, workflowExecution, action.ID) + if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { + if strings.Contains(err.Error(), "exited prematurely") { + log.Printf("[DEBUG] Shutting down (41)") + shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) + } } else { + executed = true + } + } + + if !executed { + image = images[2] + err = deployApp(dockercli, image, identifier, env, workflowExecution, action.ID) + if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { + if strings.Contains(err.Error(), "exited prematurely") { + log.Printf("[DEBUG] Shutting down (3)") + shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) + } + + //log.Printf("[WARNING] Failed CLEANUP execution. Downloading image %s remotely.", image) + log.Printf("[WARNING] Failed to download image %s (CLEANUP): %s", image, err) reader, err := dockercli.ImagePull(context.Background(), image, pullOptions) @@ -1032,22 +1047,22 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { log.Printf("[INFO] Successfully downloaded %s", image) } - } - err = deployApp(dockercli, image, identifier, env, workflowExecution, action.ID) - if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { + err = deployApp(dockercli, image, identifier, env, workflowExecution, action.ID) + if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { - log.Printf("[ERROR] Failed deploying image for the FOURTH time. Aborting if the image doesn't exist") - if strings.Contains(err.Error(), "exited prematurely") { - log.Printf("[DEBUG] Shutting down (7)") - shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) - } + log.Printf("[ERROR] Failed deploying image for the FOURTH time. Aborting if the image doesn't exist") + if strings.Contains(err.Error(), "exited prematurely") { + log.Printf("[DEBUG] Shutting down (7)") + shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) + } - if strings.Contains(err.Error(), "No such image") { - //log.Printf("[WARNING] Failed deploying %s from image %s: %s", identifier, image, err) - log.Printf("[ERROR] Image doesn't exist. Shutting down") - log.Printf("[DEBUG] Shutting down (8)") - shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) + if strings.Contains(err.Error(), "No such image") { + //log.Printf("[WARNING] Failed deploying %s from image %s: %s", identifier, image, err) + log.Printf("[ERROR] Image doesn't exist. Shutting down") + log.Printf("[DEBUG] Shutting down (8)") + shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) + } } } } @@ -1071,20 +1086,34 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) } - image = images[2] - err = deployApp(dockercli, image, identifier, env, workflowExecution, action.ID) - if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { - if strings.Contains(err.Error(), "exited prematurely") { - log.Printf("[DEBUG] Shutting down (11)") - shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) - } - - log.Printf("[WARNING] Failed deploying image THREE TIMES. Attempting to download %s as last resort from backend and dockerhub.", image) - - err := downloadDockerImageBackend(topClient, image) - if err == nil { - log.Printf("[DEBUG] Downloaded image %s from backend (CLEANUP)", image) + log.Printf("[DEBUG] Failed deploy. Downloading image %s", image) + err := downloadDockerImageBackend(topClient, image) + executed := false + if err == nil { + log.Printf("[DEBUG] Downloaded image %s from backend (CLEANUP)", image) + //err = deployApp(dockercli, image, identifier, env, workflow, action.ID) + err = deployApp(dockercli, image, identifier, env, workflowExecution, action.ID) + if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { + if strings.Contains(err.Error(), "exited prematurely") { + log.Printf("[DEBUG] Shutting down (40)") + shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) + } } else { + executed = true + } + } + + if !executed { + image = images[2] + err = deployApp(dockercli, image, identifier, env, workflowExecution, action.ID) + if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { + if strings.Contains(err.Error(), "exited prematurely") { + log.Printf("[DEBUG] Shutting down (11)") + shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true) + } + + log.Printf("[WARNING] Failed deploying image THREE TIMES. Attempting to download %s as last resort from backend and dockerhub.", image) + reader, err := dockercli.ImagePull(context.Background(), image, pullOptions) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { log.Printf("[ERROR] Failed getting %s. The couldn't be find locally, AND is missing.", image) From 336928c8a3c96453bb9139bb4358bd97626a3ca3 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 22 Jun 2021 13:38:54 +0200 Subject: [PATCH 94/96] Updated version with migration built-in --- .github/install-guide.md | 2 +- backend/app_sdk/app_base.py | 5 +- backend/app_sdk/build.sh | 2 +- backend/go-app/go.mod | 2 +- backend/go-app/go.sum | 4 + backend/go-app/main.go | 295 ++++++++++++++++++------- backend/go-app/walkoff.go | 11 +- backend/tests/migrate_db.sh | 2 +- docker-compose.yml | 33 ++- frontend/src/views/AngularWorkflow.jsx | 2 +- frontend/src/views/Workflows.jsx | 18 +- shuffle-database/README.md | 12 - 12 files changed, 268 insertions(+), 120 deletions(-) delete mode 100644 shuffle-database/README.md diff --git a/.github/install-guide.md b/.github/install-guide.md index 7c0d9af0..5d745eb3 100644 --- a/.github/install-guide.md +++ b/.github/install-guide.md @@ -28,7 +28,7 @@ docker-compose up -d When you're done, skip to the "After installation" step below. -## Windows Docker setup +## Windows with WSL This step is for setting up with Docker on windows from scratch. 1. Make sure you have [Docker](https://docs.docker.com/docker-for-windows/install/) and [docker-compose](https://docs.docker.com/compose/install/) installed. WSL2 may be required. diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 57b9a64c..40f86c13 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -514,7 +514,7 @@ class AppBase: # return print("[INFO] Multiplier length: %d" % len(param_multiplier)) - tmp = "" + #tmp = "" for subparams in param_multiplier: print(f"SUBPARAMS IN MULTI: {subparams}") try: @@ -2307,12 +2307,13 @@ class AppBase: #newres = await func(**params) #print("PARAMS: %s" % params) - newres = "" + #newres = "" while True: try: newres = await func(**params) break except TypeError as e: + newres = "" errorstring = "%s" % e if "got an unexpected keyword argument" in errorstring: fieldsplit = errorstring.split("'") diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index e0a95374..9349db29 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,6 +1,6 @@ #!/bin/bash NAME=shuffle-app_sdk -VERSION=0.8.100 +VERSION=0.8.103 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 790628e4..09027148 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -23,7 +23,7 @@ require ( github.com/elastic/go-elasticsearch v0.0.0 // indirect github.com/elastic/go-elasticsearch/v7 v7.12.0 // indirect github.com/frikky/kin-openapi v0.39.0 - github.com/frikky/shuffle-shared v0.0.66 + github.com/frikky/shuffle-shared v0.0.68 github.com/fsouza/go-dockerclient v1.7.2 // indirect github.com/ghodss/yaml v1.0.0 github.com/go-git/go-billy/v5 v5.0.0 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index d7bef904..2912b94d 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -181,6 +181,10 @@ github.com/frikky/shuffle-shared v0.0.63 h1:btn7V7s98eZmx/9qyapGE3V1TyxftvjMmrzk github.com/frikky/shuffle-shared v0.0.63/go.mod h1:oPDyGFBuurPtE6UQLUrHU/JtbOJftqDljRfEdLlCTI4= github.com/frikky/shuffle-shared v0.0.65 h1:TvISX6WE1Y7M2UJ3YuMv0PtgRPWtMj5iGiArVmv6SI8= github.com/frikky/shuffle-shared v0.0.65/go.mod h1:oPDyGFBuurPtE6UQLUrHU/JtbOJftqDljRfEdLlCTI4= +github.com/frikky/shuffle-shared v0.0.66 h1:1vIcG5ZirIVQ+Li10y9PBHRUOHwpL45L0Nbmpnmpnrw= +github.com/frikky/shuffle-shared v0.0.66/go.mod h1:oPDyGFBuurPtE6UQLUrHU/JtbOJftqDljRfEdLlCTI4= +github.com/frikky/shuffle-shared v0.0.67 h1:Zl09+6rFnlpeOagDXvagC661bdhVm4DC0dylsiZ8dn4= +github.com/frikky/shuffle-shared v0.0.67/go.mod h1:oPDyGFBuurPtE6UQLUrHU/JtbOJftqDljRfEdLlCTI4= github.com/fsouza/go-dockerclient v1.7.2 h1:bBEAcqLTkpq205jooP5RVroUKiVEWgGecHyeZc4OFjo= github.com/fsouza/go-dockerclient v1.7.2/go.mod h1:+ugtMCVRwnPfY7d8/baCzZ3uwB0BrG5DB8OzbtxaRz8= github.com/getkin/kin-openapi v0.8.0 h1:a6TQjTqwkyscC4/hShJX7WhCVE+4bi9lzw61XHQW5hE= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index a5a44341..1e8a1206 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -615,13 +615,13 @@ func createNewUser(username, password, role, apikey string, org shuffle.OrgMini) // Use this for register err := shuffle.CheckPasswordStrength(password) if err != nil { - log.Printf("Bad password strength: %s", err) + log.Printf("[WARNING] Bad password strength: %s", err) return err } err = checkUsername(username) if err != nil { - log.Printf("Bad Username strength: %s", err) + log.Printf("[WARNING] Bad Username strength: %s", err) return err } @@ -5272,7 +5272,9 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { resp.Write(respBody) } -/* +// Runs DB migration from Datastore to Opensearch +// If the function has "ALL" in it, that means it's intended to be used for Orgs +// but that we've added a function to grab everything func migrateDatabase(resp http.ResponseWriter, request *http.Request) { cors := shuffle.HandleCors(resp, request) if cors { @@ -5294,72 +5296,210 @@ func migrateDatabase(resp http.ResponseWriter, request *http.Request) { return } - type dbSetup struct { - dbUrl string - } - - config := elasticsearch.Config{ - Addresses: []string{ - "https://192.168.3.8:9200", - }, - Username: "", - Password: "", - } - - //es, err := elasticsearch.NewDefaultClient() - es, err := elasticsearch.NewClient(config) - if err != nil { - log.Printf("[WARNING] Failed connecting with es7: %s", err) + if strings.ToLower(os.Getenv("SHUFFLE_ELASTIC")) != "false" { + log.Printf("[WARNING] Failed to migrate because main DB is Elastic. Set SHUFFLE_ELASTIC=false in .env") resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } - _ = es - /* - q := datastore.NewQuery(indexType) - var users []shuffle.User - _, err = dbclient.GetAll(ctx, q, &users) - if err == nil && len(users) > 0 { + ctx := context.Background() + es := getEsConfig() + _, err := shuffle.RunInit(*dbclient, *es, storage.Client{}, gceProject, "onprem", false, "") + if err != nil { + log.Printf("[WARNING] Failed to start migration because of init issues: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + log.Printf("\n\n------- STARTING MIGRATION TO OPENSEARCH --------") + users, err := shuffle.GetAllUsers(ctx) + if err != nil { + log.Printf("[ERROR] Failed getting users: %#v", err) + } else { + log.Printf("[DEBUG] Found %d user(s) to be migrated", len(users)) + } + + orgs, err := shuffle.GetAllOrgs(ctx) + if err != nil { + log.Printf("[ERROR] Failed getting orgs: %#v", err) + } else { + log.Printf("[DEBUG] Found %d org(s) to be migrated", len(orgs)) + } + + workflows, err := shuffle.GetAllWorkflows(ctx, "ALL") + if err != nil { + log.Printf("[ERROR] Failed getting workflows: %#v", err) + } else { + log.Printf("[DEBUG] Found %d workflows(s) to be migrated", len(workflows)) + } + + apps, err := shuffle.GetAllWorkflowApps(ctx, 0) + if err != nil { + log.Printf("[ERROR] Failed getting apps: %#v", err) + } else { + log.Printf("[DEBUG] Found %d app(s) to be migrated", len(apps)) + } + + openapiApps, err := shuffle.GetAllOpenApi(ctx) + if err != nil { + log.Printf("[ERROR] Failed getting openapi apps: %#v", err) + } else { + log.Printf("[DEBUG] Found %d openapi(s) to be migrated", len(openapiApps)) + } + + workflowappauth, err := shuffle.GetAllWorkflowAppAuth(ctx, "ALL") + if err != nil { + log.Printf("[ERROR] Failed getting app auth: %#v", err) + } else { + log.Printf("[DEBUG] Found %d appauth(s) to be migrated", len(workflowappauth)) + } + + environments, err := shuffle.GetEnvironments(ctx, "ALL") + if err != nil { + log.Printf("[ERROR] Failed getting environments: %#v", err) + } else { + log.Printf("[DEBUG] Found %d environment(s) to be migrated", len(environments)) + } + + hooks, err := shuffle.GetAllHooks(ctx) + if err != nil { + log.Printf("[ERROR] Failed getting hooks: %#v", err) + } else { + log.Printf("[DEBUG] Found %d hook(s) to be migrated", len(hooks)) + } + + schedules, err := shuffle.GetAllSchedules(ctx, "ALL") + if err != nil { + log.Printf("[ERROR] Failed getting schedules: %#v", err) + } else { + log.Printf("[DEBUG] Found %d schedule(s) to be migrated", len(schedules)) + } + + log.Printf("\n\n------- SWAPPING TO OPENSEARCH DB WITH ACQUIRED INFO ---------") + userSuccess := 0 + orgSuccess := 0 + workflowSuccess := 0 + appSuccess := 0 + openapiSuccess := 0 + authSuccess := 0 + envSuccess := 0 + hookSuccess := 0 + scheduleSuccess := 0 + _, err = shuffle.RunInit(*dbclient, *es, storage.Client{}, gceProject, "onprem", false, "elasticsearch") + + for _, item := range orgs { + err = shuffle.SetOrg(ctx, item, item.Id) + if err != nil { + //log.Printf("[WARNING] Failed to update org in opensearch: %s", err) } else { - log.Printf("[WARNING] Failed getting USERS: %s", err) - - for _, user := range users { - id := user.Id - log.Printf("Indexing %s (%s)", user.Username, id) - - b, err := json.Marshal(user) - if err != nil { - log.Printf("[WARNING] Failed marshalling %s - %s: %s", id, indexType, err) - return - } - - req := esapi.IndexRequest{ - Index: indexType, - DocumentID: id, - Body: strings.NewReader(string(b)), - Refresh: "true", - } - - res, err := req.Do(context.Background(), es) - if err != nil { - log.Printf("Error getting response: %s", err) - } - - defer res.Body.Close() - if res.IsError() { - log.Printf("[%s] Error indexing document ID=%d", res.Status(), id) - } else { - log.Printf("Successfully indexed %s of ID %s", indexType, id) - } - - break - } + //log.Printf("[DEBUG] Set org %s (%s) in opensearch", item.Name, item.Id) + orgSuccess += 1 } + } + log.Printf("----- ORGS FOUND: %d - success: %d - failed: %d", len(orgs), orgSuccess, len(orgs)-orgSuccess) + + for _, item := range workflowappauth { + err = shuffle.SetWorkflowAppAuthDatastore(ctx, item, item.Id) + if err != nil { + //log.Printf("[WARNING] Failed to update app auth in opensearch: %s", err) + } else { + //log.Printf("[DEBUG] Set app auth %s in opensearch", item.Id) + authSuccess += 1 + } + } + + log.Printf("----- AUTH FOUND: %d - success: %d - failed: %d", len(workflowappauth), authSuccess, len(workflowappauth)-authSuccess) + + for _, item := range environments { + err = shuffle.SetEnvironment(ctx, &item) + if err != nil { + //log.Printf("[WARNING] Failed to update env in opensearch: %s", err) + } else { + //log.Printf("[DEBUG] Set env %s in opensearch", item.Id) + envSuccess += 1 + } + } + + log.Printf("----- ENVS FOUND: %d - success: %d - failed: %d", len(environments), envSuccess, len(environments)-envSuccess) + + for _, item := range hooks { + err = shuffle.SetHook(ctx, item) + if err != nil { + //log.Printf("[WARNING] Failed to update hooks in opensearch: %s", err) + } else { + //log.Printf("[DEBUG] Set hook %s in opensearch", item.Id) + hookSuccess += 1 + } + } + + log.Printf("---- HOOKS FOUND: %d - success: %d - failed: %d", len(hooks), hookSuccess, len(hooks)-hookSuccess) + + for _, item := range schedules { + err = shuffle.SetSchedule(ctx, item) + if err != nil { + //log.Printf("[WARNING] Failed to update schedule in opensearch: %s", err) + } else { + //log.Printf("[DEBUG] Set schedule %s in opensearch", item.Id) + scheduleSuccess += 1 + } + } + + log.Printf(" SCHEDULES FOUND: %d - success: %d - failed: %d", len(schedules), scheduleSuccess, len(schedules)-scheduleSuccess) + + for _, item := range users { + err = shuffle.SetUser(ctx, &item, false) + if err != nil { + //log.Printf("[WARNING] Failed to update user in opensearch: %s", err) + } else { + //log.Printf("[DEBUG] Set user %s (%s) in opensearch", item.Username, item.Id) + userSuccess += 1 + } + } + + log.Printf("---- USERS FOUND: %d - success: %d - failed: %d", len(users), userSuccess, len(users)-userSuccess) + + for _, item := range workflows { + err = shuffle.SetWorkflow(ctx, item, item.ID) + if err != nil { + //log.Printf("[WARNING] Failed to update workflow in opensearch: %s", err) + } else { + //log.Printf("[DEBUG] Set workflow %s (%s) in opensearch", item.Name, item.ID) + workflowSuccess += 1 + } + } + + log.Printf(" WORKFLOWS FOUND: %d - success: %d - failed: %d", len(workflows), workflowSuccess, len(workflows)-workflowSuccess) + + for _, item := range openapiApps { + err = shuffle.SetOpenApiDatastore(ctx, item.ID, item) + if err != nil { + //log.Printf("[WARNING] Failed to update openapi app in opensearch: %s", err) + } else { + //log.Printf("[DEBUG] Set openapi %s in opensearch", item.ID) + openapiSuccess += 1 + } + } + + log.Printf("-- OpenAPI FOUND: %d - success: %d - failed: %d", len(openapiApps), openapiSuccess, len(openapiApps)-openapiSuccess) + + for _, item := range apps { + err = shuffle.SetWorkflowAppDatastore(ctx, item, item.ID) + if err != nil { + //log.Printf("[WARNING] Failed to update app in opensearch: %s", err) + } else { + //log.Printf("[DEBUG] Set app %s (%s) in opensearch", item.Name, item.ID) + appSuccess += 1 + } + } + + log.Printf("----- APPS FOUND: %d - success: %d - failed: %d", len(apps), appSuccess, len(apps)-appSuccess) + + // Handle users // 1. Get users // 2. Get organizations - // 3. Get files // 4. Get workflows // 5. Get apps // 6. Get workflowappauth @@ -5368,14 +5508,14 @@ func migrateDatabase(resp http.ResponseWriter, request *http.Request) { // 10. Get hooks // 11. Get openapi3 // 12. Get schedules - // 13. Get sessions - // 14. Get workflowqueue //log.Printf("[INFO] Successfully published workflow %s (%s) TO CLOUD", workflow.Name, workflow.ID) + log.Printf("\n\n[DEBUG] Successfully updated ran migration from Datastore to Opensearch!") resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) + log.Printf("[DEBUG] Panicing to force-restart Shuffle post-migration. Stop Shuffle and change database. Docs: https://shuffler.io/docs/configuration#database_migration") + os.Exit(0) } -*/ func makeWorkflowPublic(resp http.ResponseWriter, request *http.Request) { cors := shuffle.HandleCors(resp, request) @@ -5517,17 +5657,7 @@ func makeWorkflowPublic(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } -func initHandlers() { - var err error - ctx := context.Background() - - log.Printf("[DEBUG] Starting Shuffle backend - initializing database connection") - //requestCache = cache.New(5*time.Minute, 10*time.Minute) - dbclient, err = datastore.NewClient(ctx, gceProject, option.WithGRPCDialOption(grpc.WithNoProxy())) - if err != nil { - log.Fatalf("[DEBUG] Database client error during init: %s", err) - } - +func getEsConfig() *elasticsearch.Client { esUrl := os.Getenv("SHUFFLE_OPENSEARCH_URL") if len(esUrl) == 0 { esUrl = "http://shuffle-opensearch:9200" @@ -5544,7 +5674,7 @@ func initHandlers() { //config.Transport.TLSClientConfig transport := http.DefaultTransport.(*http.Transport).Clone() - transport.MaxIdleConnsPerHost = 1000 + transport.MaxIdleConnsPerHost = 100 transport.ResponseHeaderTimeout = time.Second * 10 transport.Proxy = nil @@ -5600,6 +5730,21 @@ func initHandlers() { log.Fatalf("[DEBUG] Database client for ELASTICSEARCH error during init (fatal): %s", err) } + return es +} + +func initHandlers() { + var err error + ctx := context.Background() + + log.Printf("[DEBUG] Starting Shuffle backend - initializing database connection") + //requestCache = cache.New(5*time.Minute, 10*time.Minute) + dbclient, err = datastore.NewClient(ctx, gceProject, option.WithGRPCDialOption(grpc.WithNoProxy())) + if err != nil { + log.Fatalf("[DEBUG] Database client error during init: %s", err) + } + + es := getEsConfig() elasticConfig := "elasticsearch" if strings.ToLower(os.Getenv("SHUFFLE_ELASTIC")) == "false" { elasticConfig = "" @@ -5749,7 +5894,7 @@ func initHandlers() { // Docker orborus specific - downloads an image r.HandleFunc("/api/v1/get_docker_image", getDockerImage).Methods("POST", "OPTIONS") - //r.HandleFunc("/api/v1/migrate_database", migrateDatabase).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/migrate_database", migrateDatabase).Methods("POST", "OPTIONS") // Important for email, IDS etc. Create this by: // PS: For cloud, this has to use cloud storage. diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index b302d6f0..c0f16bbf 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -1117,7 +1117,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { - log.Printf("Api authentication failed in deleting workflow: %s", err) + log.Printf("[WARNING] Api authentication failed in delete workflow: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -1201,6 +1201,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { //memcache.Delete(ctx, memcacheName) //memcacheName = fmt.Sprintf("%s_workflows", user.Username) //memcache.Delete(ctx, memcacheName) + //cacheKey := fmt.Sprintf("%s_workflows", user.Id) cacheKey := fmt.Sprintf("%s_workflows", user.Id) shuffle.DeleteCache(ctx, cacheKey) @@ -1258,7 +1259,7 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request if workflow.ID == "" || workflow.ID != id { tmpworkflow, err := shuffle.GetWorkflow(ctx, id) if err != nil { - log.Printf("Failed getting the workflow locally (execution cleanup): %s", err) + log.Printf("[WARNING] Failed getting the workflow locally (execution setup): %s", err) return shuffle.WorkflowExecution{}, "Failed getting workflow", err } @@ -2438,7 +2439,7 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() workflow, err := shuffle.GetWorkflow(ctx, fileId) if err != nil { - log.Printf("Failed getting the workflow locally (schedule workflow): %s", err) + log.Printf("[WARNING] Failed getting the workflow locally (schedule workflow): %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -3623,7 +3624,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin // Fixes (appends) authentication parameters if they're required if workflowapp.Authentication.Required { - log.Printf("[INFO] Checking authentication fields and appending for %s!", workflowapp.Name) + //log.Printf("[INFO] Checking authentication fields and appending for %s!", workflowapp.Name) // FIXME: // Might require reflection into the python code to append the fields as well for index, action := range workflowapp.Actions { @@ -3859,7 +3860,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { // Fixes (appends) authentication parameters if they're required if workflowapp.Authentication.Required { - log.Printf("[INFO] Checking authentication fields and appending for %s!", workflowapp.Name) + //log.Printf("[INFO] Checking authentication fields and appending for %s!", workflowapp.Name) // FIXME: // Might require reflection into the python code to append the fields as well for index, action := range workflowapp.Actions { diff --git a/backend/tests/migrate_db.sh b/backend/tests/migrate_db.sh index d1cbd039..23293b1e 100644 --- a/backend/tests/migrate_db.sh +++ b/backend/tests/migrate_db.sh @@ -1,2 +1,2 @@ -curl -XPOST -v localhost:5001/api/v1/migrate_database -H 'Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4' +curl -XPOST -v localhost:5001/api/v1/migrate_database -H 'Authorization: Bearer 0184d7be-33c1-4391-bf9c-dfb8508a4ea2' diff --git a/docker-compose.yml b/docker-compose.yml index c0100d39..551e6ef9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,8 +35,8 @@ services: - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps - SHUFFLE_FILE_LOCATION=/shuffle-files restart: unless-stopped - depends_on: - - opensearch + #depends_on: + #- opensearch #- database orborus: #build: ./functions/onprem/orborus @@ -92,21 +92,20 @@ services: networks: - shuffle restart: unless-stopped - # OLD DATABASE: - #database: - # #build: ./backend/database - # image: frikky/shuffle:database - # container_name: shuffle-database - # hostname: shuffle-database - # ports: - # - "8000:8000" - # networks: - # - shuffle - # environment: - # - _JAVA_OPTIONS="-Xmx2g" - # restart: unless-stopped - # volumes: - # - ${DB_LOCATION}:/etc/shuffle + database: + #build: ./backend/database + image: frikky/shuffle:database + container_name: shuffle-database + hostname: shuffle-database + ports: + - "8000:8000" + networks: + - shuffle + environment: + - _JAVA_OPTIONS="-Xmx2g" + restart: unless-stopped + volumes: + - ${DB_LOCATION}:/etc/shuffle networks: shuffle: diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 9d30eab9..3d909ed8 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -6148,7 +6148,7 @@ const AngularWorkflow = (props) => { {subworkflow.actions.map((action, index) => { //console.log(action) return ( - parent.id === action.id)} key={index} style={{backgroundColor: inputColor, color: "white"}} value={action}> + parent.id === action.id) ? "red" : "white"}} value={action}> {action.label} ) diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index b17a2388..87c4830e 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -743,7 +743,7 @@ const Workflows = (props) => { return response.json() }) - .then((responseJson) => { + .then((responseJson) => { if (!responseJson.success) { alert.error(responseJson.reason) } @@ -783,6 +783,10 @@ const Workflows = (props) => { if (trigger.parameters !== undefined && trigger.parameters !== null && trigger.parameters.length === 2) { trigger.parameters[0].value = referenceUrl+"webhook_"+trigger.id trigger.parameters[1].value = "webhook_"+trigger.id + } else if (trigger.parameters !== undefined && trigger.parameters !== null && trigger.parameters.length === 3) { + trigger.parameters[0].value = referenceUrl+"webhook_"+trigger.id + trigger.parameters[1].value = "webhook_"+trigger.id + // FIXME: Add auth here? } else { alert.info("Something is wrong with the webhook in the copy") } @@ -955,7 +959,9 @@ const Workflows = (props) => { return response.json() }) .then((responseJson) => { - getAvailableWorkflows() + setTimeout(() => { + getAvailableWorkflows() + }, 1000) }) .catch(error => { alert.error(error.toString()) @@ -983,7 +989,9 @@ const Workflows = (props) => { return response.json() }) .then((responseJson) => { - getAvailableWorkflows() + setTimeout(() => { + getAvailableWorkflows() + }, 1000) }) .catch(error => { alert.error(error.toString()) @@ -2011,7 +2019,9 @@ const Workflows = (props) => { .then((response) => { if (response.status === 200) { alert.success("Successfully loaded workflows from "+downloadUrl) - getAvailableWorkflows() + setTimeout(() => { + getAvailableWorkflows() + }, 1000) } return response.json() diff --git a/shuffle-database/README.md b/shuffle-database/README.md deleted file mode 100644 index 43246562..00000000 --- a/shuffle-database/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Database folder -This is a folder used by the Shuffle database. It has to exist with WITHOUT root permissions before starting Shuffle. - -All database files will recide here. IF you can't get Elasticsearch to work, this most likely has to do with permissions. To fix it, run this: - -``` -docker-compose down -sudo chown 1000:1000 -R shuffle-database -docker-compose up -d -``` - -This restarts the database, and assigns 1000 (Elasticsearch process) as the owner of the database folder. From ea83fa389d3e88059a8c6c552a532cef3ba274e6 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 22 Jun 2021 13:39:49 +0200 Subject: [PATCH 95/96] Removed old database from compose --- docker-compose.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 551e6ef9..78e6ae61 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -92,20 +92,20 @@ services: networks: - shuffle restart: unless-stopped - database: - #build: ./backend/database - image: frikky/shuffle:database - container_name: shuffle-database - hostname: shuffle-database - ports: - - "8000:8000" - networks: - - shuffle - environment: - - _JAVA_OPTIONS="-Xmx2g" - restart: unless-stopped - volumes: - - ${DB_LOCATION}:/etc/shuffle + #database: + # #build: ./backend/database + # image: frikky/shuffle:database + # container_name: shuffle-database + # hostname: shuffle-database + # ports: + # - "8000:8000" + # networks: + # - shuffle + # environment: + # - _JAVA_OPTIONS="-Xmx2g" + # restart: unless-stopped + # volumes: + # - ${DB_LOCATION}:/etc/shuffle networks: shuffle: From bc5151bd88da99523b6ae0765708800a6b586ae8 Mon Sep 17 00:00:00 2001 From: frikky Date: Thu, 1 Jul 2021 13:37:56 +0200 Subject: [PATCH 96/96] Merged together app SDKs --- .github/install-guide.md | 8 +- backend/app_sdk/app_base.py | 30 +- backend/app_sdk/build.sh | 24 +- backend/app_sdk_blackarch/Dockerfile | 19 - backend/app_sdk_blackarch/LICENSE | 21 - backend/app_sdk_blackarch/README.md | 16 - backend/app_sdk_blackarch/__init__.py | 0 backend/app_sdk_blackarch/app_base.py | 1169 ------------------ backend/app_sdk_blackarch/build.sh | 13 - backend/app_sdk_blackarch/requirements.txt | 2 - backend/app_sdk_blackarch/static_baseline.py | 76 -- backend/app_sdk_kali/Dockerfile | 19 - backend/app_sdk_kali/LICENSE | 21 - backend/app_sdk_kali/README.md | 16 - backend/app_sdk_kali/__init__.py | 0 backend/app_sdk_kali/app_base.py | 1169 ------------------ backend/app_sdk_kali/build.sh | 13 - backend/app_sdk_kali/requirements.txt | 2 - backend/app_sdk_kali/static_baseline.py | 76 -- backend/go-app/go.mod | 10 +- backend/go-app/go.sum | 753 ----------- backend/go-app/main.go | 84 +- frontend/src/views/AngularWorkflow.jsx | 13 + 23 files changed, 73 insertions(+), 3481 deletions(-) delete mode 100644 backend/app_sdk_blackarch/Dockerfile delete mode 100644 backend/app_sdk_blackarch/LICENSE delete mode 100644 backend/app_sdk_blackarch/README.md delete mode 100644 backend/app_sdk_blackarch/__init__.py delete mode 100644 backend/app_sdk_blackarch/app_base.py delete mode 100644 backend/app_sdk_blackarch/build.sh delete mode 100644 backend/app_sdk_blackarch/requirements.txt delete mode 100644 backend/app_sdk_blackarch/static_baseline.py delete mode 100644 backend/app_sdk_kali/Dockerfile delete mode 100644 backend/app_sdk_kali/LICENSE delete mode 100644 backend/app_sdk_kali/README.md delete mode 100644 backend/app_sdk_kali/__init__.py delete mode 100644 backend/app_sdk_kali/app_base.py delete mode 100644 backend/app_sdk_kali/build.sh delete mode 100644 backend/app_sdk_kali/requirements.txt delete mode 100644 backend/app_sdk_kali/static_baseline.py delete mode 100644 backend/go-app/go.sum diff --git a/.github/install-guide.md b/.github/install-guide.md index 5d745eb3..069b09a0 100644 --- a/.github/install-guide.md +++ b/.github/install-guide.md @@ -17,8 +17,14 @@ cd Shuffle 3. Fix prerequisites for the Opensearch database (Elasticsearch): ``` +sudo chown 1000:1000 -R shuffle-database # Required for Opensearch sudo sysctl -w vm.max_map_count=262144 # https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html -sudo chown 1000:1000 -R shuffle-database # Requires for Opensearch + +# To make the changes permanent, do: +# 1. Open the file /etc/sysctl.conf +# 2. Go to the bottom of the file +# 3. Add this line: +vm.max_map_count=262144 ``` 4. Run docker-compose. diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 40f86c13..a4da37fb 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -1213,7 +1213,7 @@ class AppBase: # Parses JSON loops and such down to the item you're looking for def recurse_json(basejson, parsersplit): - match = "#(\d+):?-?([0-9a-z]+)?#?" + match = "#([0-9a-z]+):?-?([0-9a-z]+)?#?" #print("Split: %s\n%s" % (parsersplit, basejson)) try: outercnt = 0 @@ -1225,6 +1225,7 @@ class AppBase: #print("VALUE: %s\n" % value) actualitem = re.findall(match, value, re.MULTILINE) + #print("ACTUAL RECURSE: (%s) %s" % (value, actualitem)) if value == "#": newvalue = [] for innervalue in basejson: @@ -1253,7 +1254,12 @@ class AppBase: # Means it's a single item -> continue if seconditem == "": - #print("[INFO] In first - handling %s" % firstitem) + print("[INFO] In first - handling %s. Len: %d" % (firstitem, len(basejson))) + if firstitem.lower() == "max" or firstitem.lower() == "last": + firstitem = len(basejson)-1 + if firstitem.lower() == "min" or firstitem.lower() == "first": + firstitem = 0 + tmpitem = basejson[int(firstitem)] try: newvalue, is_loop = recurse_json(tmpitem, parsersplit[outercnt+1:]) @@ -1261,16 +1267,23 @@ class AppBase: newvalue, is_loop = (tmpitem, parsersplit[outercnt+1:]) else: print("[INFO] In ELSE - handling %s and %s" % (firstitem, seconditem)) - if seconditem == "max": - seconditem = len(basejson) - if seconditem == "min": + if firstitem.lower() == "max" or firstitem.lower() == "last": + firstitem = len(basejson)-1 + if firstitem.lower() == "min" or firstitem.lower() == "first": + firstitem = 0 + if seconditem.lower() == "max" or seconditem.lower() == "last": + seconditem = len(basejson)-1 + if seconditem.lower() == "min" or seconditem.lower() == "first": seconditem = 0 newvalue = [] - for i in range(int(firstitem), int(seconditem)): + if int(seconditem) > len(basejson): + seconditem = len(basejson) + + for i in range(int(firstitem), int(seconditem)+1): # 1. Check the next item (message) # 2. Call this function again - print("Base: %s" % basejson[i]) + #print("Base: %s" % basejson[i]) try: ret, is_loop = recurse_json(basejson[i], parsersplit[outercnt+1:]) @@ -1279,10 +1292,11 @@ class AppBase: #ret = innervalue ret, is_loop = recurse_json(innervalue, parsersplit[outercnt:]) - print(ret) + #print("IN LIST: %s" % ret) #exit() newvalue.append(ret) + #print("Returning %s" % newvalue) return newvalue, is_loop else: diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 9349db29..1e52f74d 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,9 +1,12 @@ #!/bin/bash + + +### DEFAULT NAME=shuffle-app_sdk -VERSION=0.8.103 +VERSION=0.8.104 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force -docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION +docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION #docker push frikky/$NAME:$VERSION #docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION @@ -13,3 +16,20 @@ docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg. docker push frikky/shuffle:app_sdk docker push ghcr.io/frikky/$NAME:$VERSION docker push ghcr.io/frikky/$NAME:nightly + +#### BLACKARCH ### +NAME=shuffle-app_sdk_kali +docker build . -f Dockerfile_kali -t frikky/shuffle:app_sdk_kali -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION + +docker push frikky/shuffle:app_sdk_kali +docker push ghcr.io/frikky/$NAME:$VERSION +docker push ghcr.io/frikky/$NAME:nightly + +### BLACKARCH ### +NAME=shuffle-app_sdk_blackarch +docker build . -f Dockerfile_blackarch -t frikky/shuffle:app_sdk_blackarch -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION + +docker push frikky/shuffle:app_sdk_blackarch +docker push ghcr.io/frikky/$NAME:$VERSION +docker push ghcr.io/frikky/$NAME:nightly + diff --git a/backend/app_sdk_blackarch/Dockerfile b/backend/app_sdk_blackarch/Dockerfile deleted file mode 100644 index 02480aad..00000000 --- a/backend/app_sdk_blackarch/Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -FROM peterclemenko/blackarch as base - -FROM base as builder - -RUN /bin/pacman -Syu --noconfirm - -RUN /bin/pacman -Sy --noconfirm base-devel libffi musl openssl python python-pip -y - -RUN mkdir /install -WORKDIR /install - -COPY requirements.txt /requirements.txt -RUN pip install --prefix="/install" -r /requirements.txt - -FROM base - -COPY --from=builder /install /usr/local -COPY __init__.py /app/walkoff_app_sdk/__init__.py -COPY app_base.py /app/walkoff_app_sdk/app_base.py diff --git a/backend/app_sdk_blackarch/LICENSE b/backend/app_sdk_blackarch/LICENSE deleted file mode 100644 index ce11f6f3..00000000 --- a/backend/app_sdk_blackarch/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2020 Frikkylikeme - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/backend/app_sdk_blackarch/README.md b/backend/app_sdk_blackarch/README.md deleted file mode 100644 index 754392ff..00000000 --- a/backend/app_sdk_blackarch/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# app_sdk.py -This is the SDK used for apps to behave like they should. -To change it in the backend, upload it to Buckets/shuffler.appspot.com/generated_apps/baseline. - -# static_baseline.py -It's used for python code generation and should be under MIT. Has to be located here because it's used by the backend. - -## If you want to update apps.. PS: downloads from docker hub do overrides.. :) -1. Write your code & check if runtime works -2. Build app_base image -3. docker rm $(docker ps -aq) # Remove all stopped containers -4. Delete the specific app's Docker image (docker rmi frikky/shuffle:...) -5. Rebuild the Docker image (click load in GUI?) - -# LICENSE -Everything in here is MIT, not AGPLv3 as indicated by the license. diff --git a/backend/app_sdk_blackarch/__init__.py b/backend/app_sdk_blackarch/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/backend/app_sdk_blackarch/app_base.py b/backend/app_sdk_blackarch/app_base.py deleted file mode 100644 index e896f67c..00000000 --- a/backend/app_sdk_blackarch/app_base.py +++ /dev/null @@ -1,1169 +0,0 @@ -import os -import sys -import re -import time -import json -import logging -import requests -import urllib.parse - -class AppBase: - """ The base class for Python-based apps in Shuffle, handles logging and callbacks configurations""" - __version__ = None - app_name = None - - def __init__(self, redis=None, logger=None, console_logger=None):#, docker_client=None): - self.logger = logger if logger is not None else logging.getLogger("AppBaseLogger") - self.redis=redis - self.console_logger = logger if logger is not None else logging.getLogger("AppBaseLogger") - - # apikey is for the user / org - # authorization is for the specific workflow - self.url = os.getenv("CALLBACK_URL", "https://shuffler.io") - self.action = os.getenv("ACTION", "") - self.authorization = os.getenv("AUTHORIZATION", "") - self.current_execution_id = os.getenv("EXECUTIONID", "") - self.full_execution = os.getenv("FULL_EXECUTION", "") - - if isinstance(self.action, str): - self.action = json.loads(self.action) - - def send_result(self, action_result, headers, stream_path): - if action_result["status"] == "EXECUTING": - action_result["status"] = "FAILURE" - - # I wonder if this actually works - self.logger.info("Before last stream result") - try: - ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result) - self.logger.info("Result: %d" % ret.status_code) - if ret.status_code != 200: - self.logger.info(ret.text) - except requests.exceptions.ConnectionError as e: - self.logger.exception(e) - return - except TypeError as e: - self.logger.exception(e) - action_result["status"] = "FAILURE" - action_result["result"] = "POST error: %s" % e - self.logger.info("Before typeerror stream result") - ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result) - self.logger.info("Result: %d" % ret.status_code) - if ret.status_code != 200: - self.logger.info(ret.text) - - async def execute_action(self, action): - # FIXME - add request for the function STARTING here. Use "results stream" or something - # PAUSED, AWAITING_DATA, PENDING, COMPLETED, ABORTED, EXECUTING, SUCCESS, FAILURE - - # !!! Let this line stay - its used for some horrible codegeneration / stitching !!! # - #STARTCOPY - stream_path = "/api/v1/streams" - action_result = { - "action": action, - "authorization": self.authorization, - "execution_id": self.current_execution_id, - "result": "", - "started_at": int(time.time()), - "status": "EXECUTING" - } - self.logger.info("ACTION RESULT (start): %s", action_result) - - if len(self.action) == 0: - print("ACTION env not defined") - action_result["result"] = "Error in setup ENV: ACTION not defined" - self.send_result(action_result, headers, stream_path) - return - if len(self.authorization) == 0: - print("AUTHORIZATION env not defined") - action_result["result"] = "Error in setup ENV: AUTHORIZATION not defined" - self.send_result(action_result, headers, stream_path) - return - if len(self.current_execution_id) == 0: - print("EXECUTIONID env not defined") - action_result["result"] = "Error in setup ENV: EXECUTIONID not defined" - self.send_result(action_result, headers, stream_path) - return - - headers = { - "Content-Type": "application/json", - "Authorization": "Bearer %s" % self.authorization - } - - # Add async logger - # self.console_logger.handlers[0].stream.set_execution_id() - #self.logger.info("Before initial stream result") - try: - ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result) - self.logger.info("Workflow: %d" % ret.status_code) - if ret.status_code != 200: - self.logger.info(ret.text) - except requests.exceptions.ConnectionError as e: - print("Connectionerror: %s" % e) - - action_result["result"] = "Bad setup during startup: %s" % e - self.send_result(action_result, headers, stream_path) - return - - # Verify whether there are any parameters with ACTION_RESULT required - # If found, we get the full results list from backend - fullexecution = {} - if len(self.full_execution) == 0: - print("NO EXECUTION - LOADING!") - try: - tmpdata = { - "authorization": self.authorization, - "execution_id": self.current_execution_id - } - - self.logger.info("Before FULLEXEC stream result") - ret = requests.post( - "%s/api/v1/streams/results" % (self.url), - headers=headers, - json=tmpdata - ) - - if ret.status_code == 200: - fullexecution = ret.json() - else: - self.logger.info("Error: Data: ", ret.json()) - self.logger.info("Error with status code for results. Crashing because ACTION_RESULTS or WORKFLOW_VARIABLE can't be handled. Status: %d" % ret.status_code) - action_result["result"] = "Bad result from backend: %d" % ret.status_code - self.send_result(action_result, headers, stream_path) - return - except requests.exceptions.ConnectionError as e: - self.logger.info("Connectionerror: %s" % e) - action_result["result"] = "Connection error during startup: %s" % e - self.send_result(action_result, headers, stream_path) - return - else: - try: - fullexecution = json.loads(self.full_execution) - except json.decoder.JSONDecodeError as e: - print("Json decode execution error: %s" % e) - action_result["result"] = "Json error during startup: %s" % e - self.send_result(action_result, headers, stream_path) - return - - print("") - - - self.logger.info("AFTER FULLEXEC stream result") - - # Gets the value at the parenthesis level you want - def parse_nested_param(string, level): - """ - Generate strings contained in nested (), indexing i = level - """ - if len(re.findall("\(", string)) == len(re.findall("\)", string)): - LeftRightIndex = [x for x in zip( - [Left.start()+1 for Left in re.finditer('\(', string)], - reversed([Right.start() for Right in re.finditer('\)', string)]))] - - elif len(re.findall("\(", string)) > len(re.findall("\)", string)): - return parse_nested_param(string + ')', level) - elif len(re.findall("\(", string)) < len(re.findall("\)", string)): - return parse_nested_param('(' + string, level) - - else: - return 'Failed to parse params' - - try: - return [string[LeftRightIndex[level][0]:LeftRightIndex[level][1]]] - except IndexError: - return [string[LeftRightIndex[level+1][0]:LeftRightIndex[level+1][1]]] - - # Finds the deepest level parenthesis in a string - def maxDepth(S): - current_max = 0 - max = 0 - n = len(S) - - # Traverse the input string - for i in range(n): - if S[i] == '(': - current_max += 1 - - if current_max > max: - max = current_max - elif S[i] == ')': - if current_max > 0: - current_max -= 1 - else: - return -1 - - # finally check for unbalanced string - if current_max != 0: - return -1 - - return max-1 - - # Specific type parsing - def parse_type(data, thistype): - if data == None: - return "Empty" - - if "int" in thistype or "number" in thistype: - try: - return int(data) - except ValueError: - print("ValueError while casting %s" % data) - return data - if "lower" in thistype: - return data.lower() - if "upper" in thistype: - return data.upper() - if "trim" in thistype: - return data.strip() - if "strip" in thistype: - return data.strip() - if "split" in thistype: - return data.split() - if "len" in thistype or "length" in thistype: - tmp = "" - try: - tmp = json.loads(data) - except: - pass - - if isinstance(tmp, list): - return str(len(tmp)) - - return str(len(data)) - if "parse" in thistype: - splitvalues = [] - default_error = """Error. Expected syntax: parse(["hello","test1"],0:1)""" - if "," in data: - splitvalues = data.split(",") - - for item in range(len(splitvalues)): - splitvalues[item] = splitvalues[item].strip() - else: - return default_error - - lastsplit = [] - if ":" in splitvalues[-1]: - lastsplit = splitvalues[-1].split(":") - else: - try: - lastsplit = [int(splitvalues[-1])] - except ValueError: - return default_error - - try: - parsedlist = ",".join(splitvalues[0:-1]) - if len(lastsplit) > 1: - tmp = json.loads(parsedlist)[int(lastsplit[0]):int(lastsplit[1])] - else: - tmp = json.loads(parsedlist)[lastsplit[0]] - - print(tmp) - return tmp - except IndexError as e: - return default_error - - # Parses the INNER value and recurses until everything is done - def parse_wrapper(data): - try: - if "(" not in data or ")" not in data: - return data - except TypeError: - return data - - #print("Running %s" % data) - - # Look for the INNER wrapper first, then move out - wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length"] - found = False - for wrapper in wrappers: - if wrapper not in data.lower(): - continue - - found = True - break - - if not found: - return data - - # Do stuff here. - innervalue = parse_nested_param(data, maxDepth(data)-0) - outervalue = parse_nested_param(data, maxDepth(data)-1) - print("INNER: ", innervalue) - print("OUTER: ", outervalue) - - if outervalue != innervalue: - #print("Outer: ", outervalue, " inner: ", innervalue) - for key in range(len(innervalue)): - # Replace OUTERVALUE[key] with INNERVALUE[key] in data. - print("Replace %s with %s in %s" % (outervalue[key], innervalue[key], data)) - data = data.replace(outervalue[key], innervalue[key]) - else: - for thistype in wrappers: - if thistype.lower() not in data.lower(): - continue - - parsed_value = parse_type(innervalue[0], thistype.lower()) - return parsed_value - - print("DATA: %s\n" % data) - return parse_wrapper(data) - - def parse_wrapper_start(data): - newdata = [] - newstring = "" - record = True - paranCnt = 0 - for char in data: - if char == "(": - paranCnt += 1 - - if not record: - record = True - - if record: - newstring += char - - if paranCnt == 0 and char == " ": - newdata.append(newstring) - newstring = "" - record = True - - if char == ")": - paranCnt -= 1 - - if paranCnt == 0: - record = False - - if len(newstring) > 0: - newdata.append(newstring) - - #print(newdata) - parsedlist = [] - non_string = False - for item in newdata: - ret = parse_wrapper(item) - if not isinstance(ret, str): - non_string = True - - parsedlist.append(ret) - - if len(parsedlist) > 0 and not non_string: - return " ".join(parsedlist) - elif len(parsedlist) == 1 and non_string: - return parsedlist[0] - else: - #print("Casting back to string because multi: ", parsedlist) - newlist = [] - for item in parsedlist: - try: - newlist.append(str(item)) - except ValueError: - newlist.append("parsing_error") - return " ".join(newlist) - - # Parses JSON loops and such down to the item you're looking for - def recurse_json(basejson, parsersplit): - match = "#(\d+):?-?([0-9a-z]+)?#?" - print("Split: %s\n%s" % (parsersplit, basejson)) - try: - outercnt = 0 - - # Loops over split values - for value in parsersplit: - print("VALUE: %s\n" % value) - actualitem = re.findall(match, value, re.MULTILINE) - if value == "#": - newvalue = [] - for innervalue in basejson: - # 1. Check the next item (message) - # 2. Call this function again - - try: - ret, is_loop = recurse_json(innervalue, parsersplit[outercnt+1:]) - except IndexError: - # Only in here if it's the last loop without anything in it? - ret, is_loop = recurse_json(innervalue, parsersplit[outercnt:]) - - newvalue.append(ret) - - # Magical way of returning which makes app sdk identify - # it as multi execution - return newvalue, True - elif len(actualitem) > 0: - # FIXME: This is absolutely not perfect. - print("In recursion v2: ", actualitem) - - is_loop = True - newvalue = [] - firstitem = actualitem[0][0] - seconditem = actualitem[0][1] - - # Means it's a single item -> continue - if seconditem == "": - print("In first - handling %s", seconditem) - tmpitem = basejson[int(firstitem)] - try: - newvalue, is_loop = recurse_json(tmpitem, parsersplit[outercnt+1:]) - except IndexError: - newvalue, is_loop = (tmpitem, parsersplit[outercnt+1:]) - else: - if seconditem == "max": - seconditem = len(basejson) - if seconditem == "min": - seconditem = 0 - - newvalue = [] - for i in range(int(firstitem), int(seconditem)): - # 1. Check the next item (message) - # 2. Call this function again - print("Base: %s" % basejson[i]) - - try: - ret, is_loop = recurse_json(basejson[i], parsersplit[outercnt+1:]) - except IndexError: - print("INDEXERROR: ", parsersplit[outercnt]) - #ret = innervalue - ret, is_loop = recurse_json(innervalue, parsersplit[outercnt:]) - - print(ret) - #exit() - newvalue.append(ret) - - return newvalue, is_loop - - # FIXME: Add specific loop for other indexes - else: - #print("BEFORE NORMAL VALUE: ", basejson, value) - if len(value) == 0: - return basejson, False - - if isinstance(basejson[value], str): - print(f"LOADING STRING '%s' AS JSON" % basejson[value]) - try: - basejson = json.loads(basejson[value]) - except json.decoder.JSONDecodeError as e: - print("RETURNING BECAUSE '%s' IS A NORMAL STRING" % basejson[value]) - return basejson[value], False - else: - basejson = basejson[value] - - outercnt += 1 - - except KeyError as e: - print("Lower keyerror: %s" % e) - #return basejson - #return "KeyError: Couldn't find key: %s" % e - - return basejson, False - - # Takes a workflow execution as argument - # Returns a string if the result is single, or a list if it's a list - def get_json_value(execution_data, input_data): - parsersplit = input_data.split(".") - actionname = parsersplit[0][1:].replace(" ", "_", -1) - #Actionname: Start_node - - print(f"Actionname: {actionname}") - - # 1. Find the action - baseresult = "" - actionname_lower = actionname.lower() - try: - if actionname_lower == "exec" or actionname_lower == "webhook" or actionname_lower == "schedule" or actionname_lower == "userinput" or actionname_lower == "email_trigger" or actionname_lower == "trigger": - baseresult = execution_data["execution_argument"] - else: - for result in execution_data["results"]: - resultlabel = result["action"]["label"].replace(" ", "_", -1).lower() - if resultlabel.lower() == actionname_lower: - baseresult = result["result"] - break - - print("BEFORE VARIABLES!") - if len(baseresult) == 0: - try: - #print("WF Variables: %s" % execution_data["workflow"]["workflow_variables"]) - for variable in execution_data["workflow"]["workflow_variables"]: - variablename = variable["name"].replace(" ", "_", -1).lower() - - if variablename.lower() == actionname_lower: - baseresult = variable["value"] - break - except KeyError as e: - print("KeyError wf variables: %s" % e) - pass - except TypeError as e: - print("TypeError wf variables: %s" % e) - pass - - print("BEFORE EXECUTION VAR") - if len(baseresult) == 0: - try: - #print("Execution Variables: %s" % execution_data["execution_variables"]) - for variable in execution_data["execution_variables"]: - variablename = variable["name"].replace(" ", "_", -1).lower() - if variablename.lower() == actionname_lower: - baseresult = variable["value"] - break - except KeyError as e: - print("KeyError exec variables: %s" % e) - pass - except TypeError as e: - print("TypeError exec variables: %s" % e) - pass - - except KeyError as error: - print(f"KeyError in JSON: {error}") - - print(f"After first trycatch") - - # 2. Find the JSON data - if len(baseresult) == 0: - return "", False - - if len(parsersplit) == 1: - return baseresult, False - - baseresult = baseresult.replace("\'", "\"") - basejson = {} - try: - basejson = json.loads(baseresult) - except json.decoder.JSONDecodeError as e: - return baseresult, False - - data, is_loop = recurse_json(basejson, parsersplit[1:]) - parseditem = data - if is_loop: - print("DATA IS A LOOP - SHOULD WRAP") - if parsersplit[-1] == "#": - print("SET DATA WRAPPER TO NORMAL!") - parseditem = "${SHUFFLE_NO_SPLITTER%s}$" % json.dumps(data) - else: - # Return value: ${id[12345, 45678]}$ - print("SET DATA WRAPPER TO %s!" % parsersplit[-1]) - parseditem = "${%s%s}$" % (parsersplit[-1], json.dumps(data)) - - return parseditem, is_loop - - # Parses parameters sent to it and returns whether it did it successfully with the values found - def parse_params(action, fullexecution, parameter): - # Skip if it starts with $? - jsonparsevalue = "$." - is_loop = False - - # Matches with space in the first part, but not in subsequent parts. - # JSON / yaml etc shouldn't have spaces in their fields anyway. - match = ".*?([$]{1}([a-zA-Z0-9 _-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,})" - - # Regex to find all the things - if parameter["variant"] == "STATIC_VALUE": - data = parameter["value"] - actualitem = re.findall(match, data, re.MULTILINE) - #self.logger.debug(f"\n\nHandle static data with JSON: {data}\n\n") - #self.logger.info("STATIC PARSED: %s" % actualitem) - if len(actualitem) > 0: - print("ACTUAL: ", actualitem) - for replace in actualitem: - try: - to_be_replaced = replace[0] - except IndexError: - continue - - # Handles for loops etc. - value, is_loop = get_json_value(fullexecution, to_be_replaced) - - if isinstance(value, str): - parameter["value"] = parameter["value"].replace(to_be_replaced, value) - elif isinstance(value, dict): - parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value)) - else: - print("Unknown type %s" % type(value)) - try: - parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value)) - except json.decoder.JSONDecodeError as e: - parameter["value"] = parameter["value"].replace(to_be_replaced, value) - - - if parameter["variant"] == "WORKFLOW_VARIABLE": - print("Handling workflow variable") - found = False - try: - for item in fullexecution["workflow"]["workflow_variables"]: - if parameter["action_field"] == item["name"]: - found = True - parameter["value"] = item["value"] - break - except KeyError as e: - print("KeyError WF variable 1: %s" % e) - pass - except TypeError as e: - print("TypeError WF variables 1: %s" % e) - pass - - if not found: - try: - for item in fullexecution["execution_variables"]: - if parameter["action_field"] == item["name"]: - parameter["value"] = item["value"] - break - except KeyError as e: - print("KeyError WF variable 2: %s" % e) - pass - except TypeError as e: - print("TypeError WF variables 2: %s" % e) - pass - - elif parameter["variant"] == "ACTION_RESULT": - # FIXME - calculate value based on action_field and $if prominent - # FIND THE RIGHT LABEL - # GET THE LABEL'S RESULT - - tmpvalue = "" - self.logger.info("ACTION FIELD: %s" % parameter["action_field"]) - - fullname = "$" - if parameter["action_field"] == "Execution Argument": - tmpvalue = fullexecution["execution_argument"] - fullname += "exec" - else: - fullname += parameter["action_field"] - - self.logger.info("PRE Fullname: %s" % fullname) - - if parameter["value"].startswith(jsonparsevalue): - fullname += parameter["value"][1:] - #else: - # fullname = "$%s" % parameter["action_field"] - - self.logger.info("Fullname: %s" % fullname) - actualitem = re.findall(match, fullname, re.MULTILINE) - self.logger.info("ACTION PARSED: %s" % actualitem) - if len(actualitem) > 0: - for replace in actualitem: - try: - to_be_replaced = replace[0] - except IndexError: - print("Nothing to replace?: " % e) - continue - - # This will never be a loop aka multi argument - parameter["value"] = to_be_replaced - - value, is_loop = get_json_value(fullexecution, to_be_replaced) - print("Loop: %s" % is_loop) - if isinstance(value, str): - parameter["value"] = parameter["value"].replace(to_be_replaced, value) - elif isinstance(value, dict): - parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value)) - else: - print("Unknown type %s" % type(value)) - try: - parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value)) - except json.decoder.JSONDecodeError as e: - parameter["value"] = parameter["value"].replace(to_be_replaced, value) - - return "", parameter["value"], is_loop - - def run_validation(sourcevalue, check, destinationvalue): - self.logger.info("Checking %s %s %s" % (sourcevalue, check, destinationvalue)) - - if check == "=" or check.lower() == "equals": - if sourcevalue.lower() == destinationvalue.lower(): - return True - elif check == "!=" or check.lower() == "does not equal": - if sourcevalue.lower() != destinationvalue.lower(): - return True - elif check.lower() == "startswith": - if sourcevalue.lower().startswith(destinationvalue.lower()): - return True - elif check.lower() == "endswith": - if sourcevalue.lower().endswith(destinationvalue.lower()): - return True - elif check.lower() == "contains": - if destinationvalue.lower() in sourcevalue.lower(): - return True - elif check.lower() == "larger than": - try: - if sourcevalue.isdigit() and destinationvalue.isdigit(): - if int(sourcevalue) > int(destinationvalue): - return True - except AttributeError as e: - self.logger.error("Condition larger than failed with values %s and %s: %s" % (sourcevalue, destinationvalue, e)) - return False - elif check.lower() == "smaller than": - try: - if sourcevalue.isdigit() and destinationvalue.isdigit(): - if int(sourcevalue) < int(destinationvalue): - return True - except AttributeError as e: - self.logger.error("Condition smaller than failed with values %s and %s: %s" % (sourcevalue, destinationvalue, e)) - return False - else: - self.logger.info("Condition: can't handle %s yet. Setting to true" % check) - - return False - - def check_branch_conditions(action, fullexecution): - # relevantbranches = workflow.branches where destination = action - try: - if fullexecution["workflow"]["branches"] == None or len(fullexecution["workflow"]["branches"]) == 0: - return True, "" - except KeyError: - return True, "" - - relevantbranches = [] - for branch in fullexecution["workflow"]["branches"]: - if branch["destination_id"] != action["id"]: - continue - - # Remove anything without a condition - try: - if (branch["conditions"]) == 0 or branch["conditions"] == None: - continue - except KeyError: - continue - - self.logger.info("Relevant conditions: %s" % branch["conditions"]) - successful_conditions = [] - failed_conditions = [] - for condition in branch["conditions"]: - self.logger.info("Getting condition value of %s" % condition) - - # Parse all values first here - sourcevalue = condition["source"]["value"] - check, sourcevalue, is_loop = parse_params(action, fullexecution, condition["source"]) - if check: - return False, "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check) - - - #sourcevalue = sourcevalue.encode("utf-8") - sourcevalue = parse_wrapper_start(sourcevalue) - destinationvalue = condition["destination"]["value"] - - check, destinationvalue, is_loop = parse_params(action, fullexecution, condition["destination"]) - if check: - return False, "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check) - - #destinationvalue = destinationvalue.encode("utf-8") - destinationvalue = parse_wrapper_start(destinationvalue) - available_checks = [ - "=", - "equals", - "!=", - "does not equal", - ">", - "larger than", - "<", - "less than", - ">=", - "<=", - "startswith", - "endswith", - "contains", - "re", - "matches regex", - ] - - # FIXME - what should I do here? - if not condition["condition"]["value"] in available_checks: - self.logger.info("Skipping %s %s %s because %s is invalid." % (sourcevalue, condition["condition"]["value"], destinationvalue, condition["condition"]["value"])) - continue - - #print(destinationvalue) - # NEGATE - validation = run_validation(sourcevalue, condition["condition"]["value"], destinationvalue) - - # Configuration = negated because of WorkflowAppActionParam.. - try: - if condition["condition"]["configuration"]: - validation = not validation - except KeyError: - pass - - if not validation: - self.logger.info("Failed condition check for %s %s %s." % (sourcevalue, condition["condition"]["value"], destinationvalue)) - return False, "Failed condition: %s %s %s" % (sourcevalue, condition["condition"]["value"], destinationvalue) - - - # Make a general parser here, at least to get param["name"] = param["value"] in maparameter[string]string - #for condition in branch.conditons: - - return True, "" - - # Checks whether conditions are met, otherwise set - branchcheck, tmpresult = check_branch_conditions(action, fullexecution) - if not branchcheck: - self.logger.info("Failed one or more branch conditions.") - action_result["result"] = tmpresult - action_result["status"] = "FAILURE" - try: - ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result) - self.logger.info("Result: %d" % ret.status_code) - if ret.status_code != 200: - self.logger.info(ret.text) - except requests.exceptions.ConnectionError as e: - self.logger.exception(e) - - print("\n\nRETURNING BECAUSE A BRANCH FAILED\n\n") - return - - # Replace name cus there might be issues - # Not doing lower() as there might be user-made functions - actionname = action["name"] - if " " in actionname: - actionname.replace(" ", "_", -1) - #if action.generated: - # actionname = actionname.lower() - - # Runs the actual functions - try: - func = getattr(self, actionname, None) - if func == None: - self.logger.debug("Failed executing %s because func is None." % actionname) - action_result["status"] = "FAILURE" - action_result["result"] = "Function %s doesn't exist." % actionname - elif callable(func): - try: - if len(action["parameters"]) < 1: - result = await func() - else: - # Potentially parse JSON here - # FIXME - add potential authentication as first parameter(s) here - # params[parameter["name"]] = parameter["value"] - #print(fullexecution["authentication"] - # What variables are necessary here tho hmm - - params = {} - try: - for item in action["authentication"]: - print("AUTH: ", key, value) - params[item["key"]] = item["value"] - except KeyError: - print("No authentication specified!") - pass - #action["authentication"] - - # calltimes is used to handle forloops in the app itself. - # 2 kinds of loop - one in gui with one app each, and one like this, - # which is super fast, but has a bad overview (potentially good tho) - calltimes = 1 - result = "" - - all_executions = [] - - # Multi_parameter has the data for each. variable - minlength = 0 - multi_parameters = json.loads(json.dumps(params)) - multiexecution = False - multi_execution_lists = [] - for parameter in action["parameters"]: - check, value, is_loop = parse_params(action, fullexecution, parameter) - - if check: - raise "Value check error: %s" % Exception(check) - - # Custom format for ${name[0,1,2,...]}$ - #submatch = "([${]{2}([0-9a-zA-Z_-]+)(\[.*\])[}$]{2})" - submatch = "([${]{2}([0-9a-zA-Z_-]+)(\[.*\])[}$]{2})" - actualitem = re.findall(submatch, value, re.MULTILINE) - try: - if action["skip_multicheck"]: - print("Skipping multicheck") - actualitem = [] - except KeyError: - pass - - print("Return value: %s" % value) - actionname = action["name"] - #print("Multicheck ", actualitem) - print("Actual item: %s" % actualitem) - if len(actualitem) > 0: - multiexecution = True - - # Loop WITHOUT JSON variables go here. - # Loop WITH variables go in else. - if len(actualitem[0]) > 2 and actualitem[0][1] == "SHUFFLE_NO_SPLITTER": - print("Pre replacement: %s" % actualitem[0][2]) - tmpitem = value - - replacement = actualitem[0][2] - if replacement.startswith("\"") and replacement.endswith("\""): - replacement = replacement[1:len(replacement)-1] - - replacement = replacement.replace("\'", "\"", -1) - print("POST replacement: %s" % replacement) - - json_replacement = replacement - try: - json_replacement = json.loads(replacement) - except json.decoder.JSONDecodeError as e: - print("JSON error singular: %s" % e) - - if len(json_replacement) > minlength: - minlength = len(json_replacement) - - tmpitem = tmpitem.replace(actualitem[0][0], replacement, 1) - params[parameter["name"]] = tmpitem - multi_execution_lists.append(json_replacement) - multi_parameters[parameter["name"]] = json_replacement - - #print("LENGTH OF ARR: %d" % len(resultarray)) - #print("RESULTARRAY: %s" % resultarray) - print("MULTI finished: %s" % replacement) - else: - - # This is here to handle for loops within variables.. kindof - # 1. Find the length of the longest array - # 2. Build an array with the base values based on parameter["value"] - # 3. Get the n'th value of the generated list from values - # 4. Execute all n answers - replacements = {} - for replace in actualitem: - try: - to_be_replaced = replace[0] - actualitem = replace[2] - except IndexError: - continue - - try: - itemlist = json.loads(actualitem) - if len(itemlist) > minlength: - minlength = len(itemlist) - except json.decoder.JSONDecodeError as e: - print("JSON Error: %s in %s" % (e, actualitem)) - - replacements[to_be_replaced] = actualitem - - # This is a result array for JUST this value.. - # What if there are more? - resultarray = [] - for i in range(0, minlength): - tmpitem = json.loads(json.dumps(parameter["value"])) - for key, value in replacements.items(): - replacement = json.dumps(json.loads(value)[i]) - if replacement.startswith("\"") and replacement.endswith("\""): - replacement = replacement[1:len(replacement)-1] - #except json.decoder.JSONDecodeError as e: - - #print("REPLACING %s with %s" % (key, replacement)) - #replacement = parse_wrapper_start(replacement) - tmpitem = tmpitem.replace(key, replacement, -1) - - resultarray.append(tmpitem) - - # With this parameter ready, add it to... a greater list of parameters. Rofl - print("LENGTH OF ARR: %d" % len(resultarray)) - print("RESULTARRAY: %s" % resultarray) - if resultarray not in multi_execution_lists: - multi_execution_lists.append(resultarray) - - multi_parameters[parameter["name"]] = resultarray - else: - # Parses things like int(value) - self.logger.info("Parsing wrapper data for %s" % value) - value = parse_wrapper_start(value) - - params[parameter["name"]] = value - multi_parameters[parameter["name"]] = value - - # Fix lists here - print("CHECKING multi execution list!") - if len(multi_execution_lists) > 0: - print("\n Multi execution list has more data: %d" % len(multi_execution_lists)) - filteredlist = [] - for listitem in multi_execution_lists: - if listitem in filteredlist: - continue - - filteredlist.append(listitem) - - #print("New list length: %d" % len(filteredlist)) - if len(filteredlist) > 1: - print("Calculating new multi-loop length with %d lists" % len(filteredlist)) - tmplength = 1 - for innerlist in filteredlist: - print("List length: %d. %d*%d" % (len(innerlist), len(innerlist), tmplength)) - tmplength = len(innerlist)*tmplength - - minlength = tmplength - - print("New multi execution length: %d\n" % tmplength) - - # FIXME - this is horrible, but works for now - #for i in range(calltimes): - if not multiexecution: - print("APP_SDK DONE: Starting NORMAL execution of function") - newres = await func(**params) - #print("NEWRES: ", newres) - if isinstance(newres, str): - result += newres - else: - try: - result += str(newres) - except ValueError: - result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres) - print("Can't handle type %s value from function" % (type(newres))) - print("POST NEWRES RESULT: ", result) - else: - print("APP_SDK DONE: Starting MULTI execution with values %s of length %d" % (multi_parameters, minlength)) - # 1. Use number of executions based on the arrays being similar - # 2. Find the right value from the parsed multi_params - results = [] - json_object = False - for i in range(0, minlength): - # To be able to use the results as a list: - baseparams = json.loads(json.dumps(multi_parameters)) - # {'call': ['GoogleSafebrowsing_2_0', 'VirusTotal_GetReport_3_0']} - # 1. Check if list length is same as minlength - # 2. If NOT same length, duplicate based on length of array - # arraylength = 3 ["1", "2", "3"] - # arraylength = 4 ["1", "2", "3", "4"] - # minlength = 12 - 12/3 = 4 per item = ["1", "1", "1", "1", "2", "2", ...] - - try: - firstlist = True - for key, value in baseparams.items(): - - if isinstance(value, list): - try: - newvalue = value[i] - except IndexError: - pass - - if len(value) != minlength and len(value) > 0: - newarray = [] - print("VALUE: ", value) - additiontime = minlength/len(value) - print("Bad length for value: %d - should be %d. Additiontime: %d" % (len(value), minlength, additiontime)) - if firstlist: - print("Running normal list (FIRST)") - for subvalue in value: - for number in range(int(additiontime)): - newarray.append(subvalue) - else: - #print("Running secondary lists") - ## 1. Set up length of array - ## 2. Put values spread out - # FIXME: This works well, except if lists are same length - newarray = [""] * minlength - - cnt = 0 - for number in range(int(additiontime)): - for subvaluerange in range(len(value)): - # newlocation = number+(additiontime*subvaluerange) - # print("%d+(%d*%d) = %d. VAL: %s" % (number, additiontime, subvaluerange, newlocation, value[subvaluerange])) - # Reverse if same length? - if int(minlength/len(value)) == len(value): - tmp = int(len(value)-subvaluerange-1) - print("NEW: %d" % tmp) - newarray[cnt] = value[tmp] - else: - newarray[cnt] = value[subvaluerange] - cnt += 1 - - #print("Newarray =", newarray) - newvalue = newarray[i] - firstlist = False - - baseparams[key] = newvalue - except IndexError as e: - print("IndexError: %s" % e) - baseparams[key] = "IndexError: %s" % e - except KeyError as e: - print("KeyError: %s" % e) - baseparams[key] = "KeyError: %s" % e - - print("Running with params %s" % baseparams) - ret = await func(**baseparams) - if isinstance(ret, dict) or isinstance(ret, list): - results.append(ret) - json_object = True - else: - ret = ret.replace("\"", "\\\"", -1) - - try: - results.append(json.loads(ret)) - json_object = True - except json.decoder.JSONDecodeError as e: - #print("Json: %s" % e) - results.append(ret) - - #print("Inner ret parsed: %s" % ret) - - # Dump the result as a string of a list - #print("RESULTS: %s" % results) - if isinstance(results, list): - print("JSON OBJECT? ", json_object) - if json_object: - result = json.dumps(results) - else: - result = "[" - for item in results: - try: - json.loads(item) - result += item - except json.decoder.JSONDecodeError as e: - # Common nested issue which puts " around everything - try: - tmpitem = item.replace("\\\"", "\"", -1) - json.loads(tmpitem) - result += tmpitem - - except: - result += "\"%s\"" % item - - result += ", " - - result = result[:-2] - result += "]" - else: - print("Normal result?") - result = results - - print("RESULT: %s" % result) - action_result["status"] = "SUCCESS" - action_result["result"] = str(result) - if action_result["result"] == "": - action_result["result"] = result - - self.logger.debug(f"Executed {action['label']}-{action['id']} with result: {result}") - #self.logger.debug(f"Data: %s" % action_result) - except TypeError as e: - print("TypeError issue: %s" % e) - action_result["status"] = "FAILURE" - action_result["result"] = "TypeError: %s" % str(e) - else: - print("Function %s doesn't exist?" % action["name"]) - self.logger.error(f"App {self.__class__.__name__}.{action['name']} is not callable") - action_result["status"] = "FAILURE" - action_result["result"] = "Function %s is not callable." % actionname - - except Exception as e: - print(f"Failed to execute: {e}") - self.logger.exception(f"Failed to execute {e}-{action['id']}") - action_result["status"] = "FAILURE" - action_result["result"] = f"General exception: {e}" - - action_result["completed_at"] = int(time.time()) - - # Send the result :) - self.send_result(action_result, headers, stream_path) - return - - - #STOPCOPY - # !!! Let the above line stay - its used for some horrible codegeneration / stitching !!! # - - @classmethod - async def run(cls): - """ Connect to Redis and HTTP session, await actions """ - logging.basicConfig(format="{asctime} - {name} - {levelname}:{message}", style='{') - logger = logging.getLogger(f"{cls.__name__}") - logger.setLevel(logging.DEBUG) - print("Started execution!!") - - app = cls(redis=None, logger=logger, console_logger=logger) - - # Authorization for the app/function to control the workflow - # Function will crash if its wrong, which it probably should. - - await app.execute_action(app.action) diff --git a/backend/app_sdk_blackarch/build.sh b/backend/app_sdk_blackarch/build.sh deleted file mode 100644 index 029756ce..00000000 --- a/backend/app_sdk_blackarch/build.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -NAME=app_sdk_blackarch -VERSION=0.7.3 - -docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force -docker build . -t frikky/shuffle:$NAME -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION - -#docker push frikky/$NAME:$VERSION -#docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -#docker push ghcr.io/frikky/$NAME:$VERSION - -docker push frikky/shuffle:$NAME -docker push ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/app_sdk_blackarch/requirements.txt b/backend/app_sdk_blackarch/requirements.txt deleted file mode 100644 index 804abb1b..00000000 --- a/backend/app_sdk_blackarch/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -requests -urllib3 diff --git a/backend/app_sdk_blackarch/static_baseline.py b/backend/app_sdk_blackarch/static_baseline.py deleted file mode 100644 index 152131cd..00000000 --- a/backend/app_sdk_blackarch/static_baseline.py +++ /dev/null @@ -1,76 +0,0 @@ -import os -import sys -import time -import logging -import requests - -# Goal here: -# * Make an app from WALKOFF able to run without app_base.py from WALKOFF -# # How: -# * Make it rely 100% on INPUT throug HTTP invocations instead of redis READS -# # But really, how? -# * Make a WORKER that reads the queue, and reuses a function - -# Here to get it global -apikey = "" -try: - apikey = os.environ["FUNCTION_APIKEY"] -except KeyError: - pass - -# Authorize the execution -def authorization(request): - # This is basically my issue, but it enforces the use of an internal API key for execution - try: - apikey = os.environ["FUNCTION_APIKEY"] - except KeyError: - return f"Internal server error", 500 - - - # Check API key from ENV authentication - authentication = request.headers.get("Authorization") - if authentication == None: return f"Unauthorized", 401 - - apikey_split = authentication.split(" ") - if apikey_split[0] != "Bearer" or len(apikey_split) != 2: - return f"Apikey error", 401 - - if apikey != apikey_split[1]: - return f"Unauthorized", 401 - - return run(request) - -class AppBase: - """ The base class for Python-based Walkoff applications, handles Redis and logging configurations. """ - __version__ = None - app_name = None - - def __init__(self, redis=None, logger=None, console_logger=None):#, docker_client=None): - self.logger = logger if logger is not None else logging.getLogger("AppBaseLogger") - self.redis=redis - self.console_logger=console_logger - self.current_execution_id = None - self.url = "https://shuffler.io" - self.apikey = apikey - - @classmethod - async def run(cls, action): - """ Connect to Redis and HTTP session, await actions """ - logging.basicConfig(format="{asctime} - {name} - {levelname}:{message}", style='{') - logger = logging.getLogger(f"{cls.__name__}") - logger.setLevel(logging.DEBUG) - - app = cls(redis=None, logger=logger, console_logger=logger) - - # Authorization for the app/function to control the workflow - # Function will crash if its wrong, which it probably should. - - await app.execute_action(action) - - async def execute_action(self, action): - # FIXME - add request for the function STARTING here. Use "results stream" or something - # PAUSED, AWAITING_DATA, PENDING, COMPLETED, ABORTED, EXECUTING, SUCCESS, FAILURE - - self.authorization = action["authorization"] - self.execution_id = action["execution_id"] - self.current_execution_id = action["execution_id"] diff --git a/backend/app_sdk_kali/Dockerfile b/backend/app_sdk_kali/Dockerfile deleted file mode 100644 index af7fd24f..00000000 --- a/backend/app_sdk_kali/Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -FROM kalilinux/kali-rolling as base - -FROM base as builder - -RUN apt-get update -RUN apt-get dist-upgrade -y -RUN apt install build-essential libffi-dev musl-dev openssl python3 python3-pip -y - -RUN mkdir /install -WORKDIR /install - -COPY requirements.txt /requirements.txt -RUN pip install --prefix="/install" -r /requirements.txt - -FROM base - -COPY --from=builder /install /usr/local -COPY __init__.py /app/walkoff_app_sdk/__init__.py -COPY app_base.py /app/walkoff_app_sdk/app_base.py diff --git a/backend/app_sdk_kali/LICENSE b/backend/app_sdk_kali/LICENSE deleted file mode 100644 index ce11f6f3..00000000 --- a/backend/app_sdk_kali/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2020 Frikkylikeme - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/backend/app_sdk_kali/README.md b/backend/app_sdk_kali/README.md deleted file mode 100644 index 754392ff..00000000 --- a/backend/app_sdk_kali/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# app_sdk.py -This is the SDK used for apps to behave like they should. -To change it in the backend, upload it to Buckets/shuffler.appspot.com/generated_apps/baseline. - -# static_baseline.py -It's used for python code generation and should be under MIT. Has to be located here because it's used by the backend. - -## If you want to update apps.. PS: downloads from docker hub do overrides.. :) -1. Write your code & check if runtime works -2. Build app_base image -3. docker rm $(docker ps -aq) # Remove all stopped containers -4. Delete the specific app's Docker image (docker rmi frikky/shuffle:...) -5. Rebuild the Docker image (click load in GUI?) - -# LICENSE -Everything in here is MIT, not AGPLv3 as indicated by the license. diff --git a/backend/app_sdk_kali/__init__.py b/backend/app_sdk_kali/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/backend/app_sdk_kali/app_base.py b/backend/app_sdk_kali/app_base.py deleted file mode 100644 index e896f67c..00000000 --- a/backend/app_sdk_kali/app_base.py +++ /dev/null @@ -1,1169 +0,0 @@ -import os -import sys -import re -import time -import json -import logging -import requests -import urllib.parse - -class AppBase: - """ The base class for Python-based apps in Shuffle, handles logging and callbacks configurations""" - __version__ = None - app_name = None - - def __init__(self, redis=None, logger=None, console_logger=None):#, docker_client=None): - self.logger = logger if logger is not None else logging.getLogger("AppBaseLogger") - self.redis=redis - self.console_logger = logger if logger is not None else logging.getLogger("AppBaseLogger") - - # apikey is for the user / org - # authorization is for the specific workflow - self.url = os.getenv("CALLBACK_URL", "https://shuffler.io") - self.action = os.getenv("ACTION", "") - self.authorization = os.getenv("AUTHORIZATION", "") - self.current_execution_id = os.getenv("EXECUTIONID", "") - self.full_execution = os.getenv("FULL_EXECUTION", "") - - if isinstance(self.action, str): - self.action = json.loads(self.action) - - def send_result(self, action_result, headers, stream_path): - if action_result["status"] == "EXECUTING": - action_result["status"] = "FAILURE" - - # I wonder if this actually works - self.logger.info("Before last stream result") - try: - ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result) - self.logger.info("Result: %d" % ret.status_code) - if ret.status_code != 200: - self.logger.info(ret.text) - except requests.exceptions.ConnectionError as e: - self.logger.exception(e) - return - except TypeError as e: - self.logger.exception(e) - action_result["status"] = "FAILURE" - action_result["result"] = "POST error: %s" % e - self.logger.info("Before typeerror stream result") - ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result) - self.logger.info("Result: %d" % ret.status_code) - if ret.status_code != 200: - self.logger.info(ret.text) - - async def execute_action(self, action): - # FIXME - add request for the function STARTING here. Use "results stream" or something - # PAUSED, AWAITING_DATA, PENDING, COMPLETED, ABORTED, EXECUTING, SUCCESS, FAILURE - - # !!! Let this line stay - its used for some horrible codegeneration / stitching !!! # - #STARTCOPY - stream_path = "/api/v1/streams" - action_result = { - "action": action, - "authorization": self.authorization, - "execution_id": self.current_execution_id, - "result": "", - "started_at": int(time.time()), - "status": "EXECUTING" - } - self.logger.info("ACTION RESULT (start): %s", action_result) - - if len(self.action) == 0: - print("ACTION env not defined") - action_result["result"] = "Error in setup ENV: ACTION not defined" - self.send_result(action_result, headers, stream_path) - return - if len(self.authorization) == 0: - print("AUTHORIZATION env not defined") - action_result["result"] = "Error in setup ENV: AUTHORIZATION not defined" - self.send_result(action_result, headers, stream_path) - return - if len(self.current_execution_id) == 0: - print("EXECUTIONID env not defined") - action_result["result"] = "Error in setup ENV: EXECUTIONID not defined" - self.send_result(action_result, headers, stream_path) - return - - headers = { - "Content-Type": "application/json", - "Authorization": "Bearer %s" % self.authorization - } - - # Add async logger - # self.console_logger.handlers[0].stream.set_execution_id() - #self.logger.info("Before initial stream result") - try: - ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result) - self.logger.info("Workflow: %d" % ret.status_code) - if ret.status_code != 200: - self.logger.info(ret.text) - except requests.exceptions.ConnectionError as e: - print("Connectionerror: %s" % e) - - action_result["result"] = "Bad setup during startup: %s" % e - self.send_result(action_result, headers, stream_path) - return - - # Verify whether there are any parameters with ACTION_RESULT required - # If found, we get the full results list from backend - fullexecution = {} - if len(self.full_execution) == 0: - print("NO EXECUTION - LOADING!") - try: - tmpdata = { - "authorization": self.authorization, - "execution_id": self.current_execution_id - } - - self.logger.info("Before FULLEXEC stream result") - ret = requests.post( - "%s/api/v1/streams/results" % (self.url), - headers=headers, - json=tmpdata - ) - - if ret.status_code == 200: - fullexecution = ret.json() - else: - self.logger.info("Error: Data: ", ret.json()) - self.logger.info("Error with status code for results. Crashing because ACTION_RESULTS or WORKFLOW_VARIABLE can't be handled. Status: %d" % ret.status_code) - action_result["result"] = "Bad result from backend: %d" % ret.status_code - self.send_result(action_result, headers, stream_path) - return - except requests.exceptions.ConnectionError as e: - self.logger.info("Connectionerror: %s" % e) - action_result["result"] = "Connection error during startup: %s" % e - self.send_result(action_result, headers, stream_path) - return - else: - try: - fullexecution = json.loads(self.full_execution) - except json.decoder.JSONDecodeError as e: - print("Json decode execution error: %s" % e) - action_result["result"] = "Json error during startup: %s" % e - self.send_result(action_result, headers, stream_path) - return - - print("") - - - self.logger.info("AFTER FULLEXEC stream result") - - # Gets the value at the parenthesis level you want - def parse_nested_param(string, level): - """ - Generate strings contained in nested (), indexing i = level - """ - if len(re.findall("\(", string)) == len(re.findall("\)", string)): - LeftRightIndex = [x for x in zip( - [Left.start()+1 for Left in re.finditer('\(', string)], - reversed([Right.start() for Right in re.finditer('\)', string)]))] - - elif len(re.findall("\(", string)) > len(re.findall("\)", string)): - return parse_nested_param(string + ')', level) - elif len(re.findall("\(", string)) < len(re.findall("\)", string)): - return parse_nested_param('(' + string, level) - - else: - return 'Failed to parse params' - - try: - return [string[LeftRightIndex[level][0]:LeftRightIndex[level][1]]] - except IndexError: - return [string[LeftRightIndex[level+1][0]:LeftRightIndex[level+1][1]]] - - # Finds the deepest level parenthesis in a string - def maxDepth(S): - current_max = 0 - max = 0 - n = len(S) - - # Traverse the input string - for i in range(n): - if S[i] == '(': - current_max += 1 - - if current_max > max: - max = current_max - elif S[i] == ')': - if current_max > 0: - current_max -= 1 - else: - return -1 - - # finally check for unbalanced string - if current_max != 0: - return -1 - - return max-1 - - # Specific type parsing - def parse_type(data, thistype): - if data == None: - return "Empty" - - if "int" in thistype or "number" in thistype: - try: - return int(data) - except ValueError: - print("ValueError while casting %s" % data) - return data - if "lower" in thistype: - return data.lower() - if "upper" in thistype: - return data.upper() - if "trim" in thistype: - return data.strip() - if "strip" in thistype: - return data.strip() - if "split" in thistype: - return data.split() - if "len" in thistype or "length" in thistype: - tmp = "" - try: - tmp = json.loads(data) - except: - pass - - if isinstance(tmp, list): - return str(len(tmp)) - - return str(len(data)) - if "parse" in thistype: - splitvalues = [] - default_error = """Error. Expected syntax: parse(["hello","test1"],0:1)""" - if "," in data: - splitvalues = data.split(",") - - for item in range(len(splitvalues)): - splitvalues[item] = splitvalues[item].strip() - else: - return default_error - - lastsplit = [] - if ":" in splitvalues[-1]: - lastsplit = splitvalues[-1].split(":") - else: - try: - lastsplit = [int(splitvalues[-1])] - except ValueError: - return default_error - - try: - parsedlist = ",".join(splitvalues[0:-1]) - if len(lastsplit) > 1: - tmp = json.loads(parsedlist)[int(lastsplit[0]):int(lastsplit[1])] - else: - tmp = json.loads(parsedlist)[lastsplit[0]] - - print(tmp) - return tmp - except IndexError as e: - return default_error - - # Parses the INNER value and recurses until everything is done - def parse_wrapper(data): - try: - if "(" not in data or ")" not in data: - return data - except TypeError: - return data - - #print("Running %s" % data) - - # Look for the INNER wrapper first, then move out - wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length"] - found = False - for wrapper in wrappers: - if wrapper not in data.lower(): - continue - - found = True - break - - if not found: - return data - - # Do stuff here. - innervalue = parse_nested_param(data, maxDepth(data)-0) - outervalue = parse_nested_param(data, maxDepth(data)-1) - print("INNER: ", innervalue) - print("OUTER: ", outervalue) - - if outervalue != innervalue: - #print("Outer: ", outervalue, " inner: ", innervalue) - for key in range(len(innervalue)): - # Replace OUTERVALUE[key] with INNERVALUE[key] in data. - print("Replace %s with %s in %s" % (outervalue[key], innervalue[key], data)) - data = data.replace(outervalue[key], innervalue[key]) - else: - for thistype in wrappers: - if thistype.lower() not in data.lower(): - continue - - parsed_value = parse_type(innervalue[0], thistype.lower()) - return parsed_value - - print("DATA: %s\n" % data) - return parse_wrapper(data) - - def parse_wrapper_start(data): - newdata = [] - newstring = "" - record = True - paranCnt = 0 - for char in data: - if char == "(": - paranCnt += 1 - - if not record: - record = True - - if record: - newstring += char - - if paranCnt == 0 and char == " ": - newdata.append(newstring) - newstring = "" - record = True - - if char == ")": - paranCnt -= 1 - - if paranCnt == 0: - record = False - - if len(newstring) > 0: - newdata.append(newstring) - - #print(newdata) - parsedlist = [] - non_string = False - for item in newdata: - ret = parse_wrapper(item) - if not isinstance(ret, str): - non_string = True - - parsedlist.append(ret) - - if len(parsedlist) > 0 and not non_string: - return " ".join(parsedlist) - elif len(parsedlist) == 1 and non_string: - return parsedlist[0] - else: - #print("Casting back to string because multi: ", parsedlist) - newlist = [] - for item in parsedlist: - try: - newlist.append(str(item)) - except ValueError: - newlist.append("parsing_error") - return " ".join(newlist) - - # Parses JSON loops and such down to the item you're looking for - def recurse_json(basejson, parsersplit): - match = "#(\d+):?-?([0-9a-z]+)?#?" - print("Split: %s\n%s" % (parsersplit, basejson)) - try: - outercnt = 0 - - # Loops over split values - for value in parsersplit: - print("VALUE: %s\n" % value) - actualitem = re.findall(match, value, re.MULTILINE) - if value == "#": - newvalue = [] - for innervalue in basejson: - # 1. Check the next item (message) - # 2. Call this function again - - try: - ret, is_loop = recurse_json(innervalue, parsersplit[outercnt+1:]) - except IndexError: - # Only in here if it's the last loop without anything in it? - ret, is_loop = recurse_json(innervalue, parsersplit[outercnt:]) - - newvalue.append(ret) - - # Magical way of returning which makes app sdk identify - # it as multi execution - return newvalue, True - elif len(actualitem) > 0: - # FIXME: This is absolutely not perfect. - print("In recursion v2: ", actualitem) - - is_loop = True - newvalue = [] - firstitem = actualitem[0][0] - seconditem = actualitem[0][1] - - # Means it's a single item -> continue - if seconditem == "": - print("In first - handling %s", seconditem) - tmpitem = basejson[int(firstitem)] - try: - newvalue, is_loop = recurse_json(tmpitem, parsersplit[outercnt+1:]) - except IndexError: - newvalue, is_loop = (tmpitem, parsersplit[outercnt+1:]) - else: - if seconditem == "max": - seconditem = len(basejson) - if seconditem == "min": - seconditem = 0 - - newvalue = [] - for i in range(int(firstitem), int(seconditem)): - # 1. Check the next item (message) - # 2. Call this function again - print("Base: %s" % basejson[i]) - - try: - ret, is_loop = recurse_json(basejson[i], parsersplit[outercnt+1:]) - except IndexError: - print("INDEXERROR: ", parsersplit[outercnt]) - #ret = innervalue - ret, is_loop = recurse_json(innervalue, parsersplit[outercnt:]) - - print(ret) - #exit() - newvalue.append(ret) - - return newvalue, is_loop - - # FIXME: Add specific loop for other indexes - else: - #print("BEFORE NORMAL VALUE: ", basejson, value) - if len(value) == 0: - return basejson, False - - if isinstance(basejson[value], str): - print(f"LOADING STRING '%s' AS JSON" % basejson[value]) - try: - basejson = json.loads(basejson[value]) - except json.decoder.JSONDecodeError as e: - print("RETURNING BECAUSE '%s' IS A NORMAL STRING" % basejson[value]) - return basejson[value], False - else: - basejson = basejson[value] - - outercnt += 1 - - except KeyError as e: - print("Lower keyerror: %s" % e) - #return basejson - #return "KeyError: Couldn't find key: %s" % e - - return basejson, False - - # Takes a workflow execution as argument - # Returns a string if the result is single, or a list if it's a list - def get_json_value(execution_data, input_data): - parsersplit = input_data.split(".") - actionname = parsersplit[0][1:].replace(" ", "_", -1) - #Actionname: Start_node - - print(f"Actionname: {actionname}") - - # 1. Find the action - baseresult = "" - actionname_lower = actionname.lower() - try: - if actionname_lower == "exec" or actionname_lower == "webhook" or actionname_lower == "schedule" or actionname_lower == "userinput" or actionname_lower == "email_trigger" or actionname_lower == "trigger": - baseresult = execution_data["execution_argument"] - else: - for result in execution_data["results"]: - resultlabel = result["action"]["label"].replace(" ", "_", -1).lower() - if resultlabel.lower() == actionname_lower: - baseresult = result["result"] - break - - print("BEFORE VARIABLES!") - if len(baseresult) == 0: - try: - #print("WF Variables: %s" % execution_data["workflow"]["workflow_variables"]) - for variable in execution_data["workflow"]["workflow_variables"]: - variablename = variable["name"].replace(" ", "_", -1).lower() - - if variablename.lower() == actionname_lower: - baseresult = variable["value"] - break - except KeyError as e: - print("KeyError wf variables: %s" % e) - pass - except TypeError as e: - print("TypeError wf variables: %s" % e) - pass - - print("BEFORE EXECUTION VAR") - if len(baseresult) == 0: - try: - #print("Execution Variables: %s" % execution_data["execution_variables"]) - for variable in execution_data["execution_variables"]: - variablename = variable["name"].replace(" ", "_", -1).lower() - if variablename.lower() == actionname_lower: - baseresult = variable["value"] - break - except KeyError as e: - print("KeyError exec variables: %s" % e) - pass - except TypeError as e: - print("TypeError exec variables: %s" % e) - pass - - except KeyError as error: - print(f"KeyError in JSON: {error}") - - print(f"After first trycatch") - - # 2. Find the JSON data - if len(baseresult) == 0: - return "", False - - if len(parsersplit) == 1: - return baseresult, False - - baseresult = baseresult.replace("\'", "\"") - basejson = {} - try: - basejson = json.loads(baseresult) - except json.decoder.JSONDecodeError as e: - return baseresult, False - - data, is_loop = recurse_json(basejson, parsersplit[1:]) - parseditem = data - if is_loop: - print("DATA IS A LOOP - SHOULD WRAP") - if parsersplit[-1] == "#": - print("SET DATA WRAPPER TO NORMAL!") - parseditem = "${SHUFFLE_NO_SPLITTER%s}$" % json.dumps(data) - else: - # Return value: ${id[12345, 45678]}$ - print("SET DATA WRAPPER TO %s!" % parsersplit[-1]) - parseditem = "${%s%s}$" % (parsersplit[-1], json.dumps(data)) - - return parseditem, is_loop - - # Parses parameters sent to it and returns whether it did it successfully with the values found - def parse_params(action, fullexecution, parameter): - # Skip if it starts with $? - jsonparsevalue = "$." - is_loop = False - - # Matches with space in the first part, but not in subsequent parts. - # JSON / yaml etc shouldn't have spaces in their fields anyway. - match = ".*?([$]{1}([a-zA-Z0-9 _-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,})" - - # Regex to find all the things - if parameter["variant"] == "STATIC_VALUE": - data = parameter["value"] - actualitem = re.findall(match, data, re.MULTILINE) - #self.logger.debug(f"\n\nHandle static data with JSON: {data}\n\n") - #self.logger.info("STATIC PARSED: %s" % actualitem) - if len(actualitem) > 0: - print("ACTUAL: ", actualitem) - for replace in actualitem: - try: - to_be_replaced = replace[0] - except IndexError: - continue - - # Handles for loops etc. - value, is_loop = get_json_value(fullexecution, to_be_replaced) - - if isinstance(value, str): - parameter["value"] = parameter["value"].replace(to_be_replaced, value) - elif isinstance(value, dict): - parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value)) - else: - print("Unknown type %s" % type(value)) - try: - parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value)) - except json.decoder.JSONDecodeError as e: - parameter["value"] = parameter["value"].replace(to_be_replaced, value) - - - if parameter["variant"] == "WORKFLOW_VARIABLE": - print("Handling workflow variable") - found = False - try: - for item in fullexecution["workflow"]["workflow_variables"]: - if parameter["action_field"] == item["name"]: - found = True - parameter["value"] = item["value"] - break - except KeyError as e: - print("KeyError WF variable 1: %s" % e) - pass - except TypeError as e: - print("TypeError WF variables 1: %s" % e) - pass - - if not found: - try: - for item in fullexecution["execution_variables"]: - if parameter["action_field"] == item["name"]: - parameter["value"] = item["value"] - break - except KeyError as e: - print("KeyError WF variable 2: %s" % e) - pass - except TypeError as e: - print("TypeError WF variables 2: %s" % e) - pass - - elif parameter["variant"] == "ACTION_RESULT": - # FIXME - calculate value based on action_field and $if prominent - # FIND THE RIGHT LABEL - # GET THE LABEL'S RESULT - - tmpvalue = "" - self.logger.info("ACTION FIELD: %s" % parameter["action_field"]) - - fullname = "$" - if parameter["action_field"] == "Execution Argument": - tmpvalue = fullexecution["execution_argument"] - fullname += "exec" - else: - fullname += parameter["action_field"] - - self.logger.info("PRE Fullname: %s" % fullname) - - if parameter["value"].startswith(jsonparsevalue): - fullname += parameter["value"][1:] - #else: - # fullname = "$%s" % parameter["action_field"] - - self.logger.info("Fullname: %s" % fullname) - actualitem = re.findall(match, fullname, re.MULTILINE) - self.logger.info("ACTION PARSED: %s" % actualitem) - if len(actualitem) > 0: - for replace in actualitem: - try: - to_be_replaced = replace[0] - except IndexError: - print("Nothing to replace?: " % e) - continue - - # This will never be a loop aka multi argument - parameter["value"] = to_be_replaced - - value, is_loop = get_json_value(fullexecution, to_be_replaced) - print("Loop: %s" % is_loop) - if isinstance(value, str): - parameter["value"] = parameter["value"].replace(to_be_replaced, value) - elif isinstance(value, dict): - parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value)) - else: - print("Unknown type %s" % type(value)) - try: - parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value)) - except json.decoder.JSONDecodeError as e: - parameter["value"] = parameter["value"].replace(to_be_replaced, value) - - return "", parameter["value"], is_loop - - def run_validation(sourcevalue, check, destinationvalue): - self.logger.info("Checking %s %s %s" % (sourcevalue, check, destinationvalue)) - - if check == "=" or check.lower() == "equals": - if sourcevalue.lower() == destinationvalue.lower(): - return True - elif check == "!=" or check.lower() == "does not equal": - if sourcevalue.lower() != destinationvalue.lower(): - return True - elif check.lower() == "startswith": - if sourcevalue.lower().startswith(destinationvalue.lower()): - return True - elif check.lower() == "endswith": - if sourcevalue.lower().endswith(destinationvalue.lower()): - return True - elif check.lower() == "contains": - if destinationvalue.lower() in sourcevalue.lower(): - return True - elif check.lower() == "larger than": - try: - if sourcevalue.isdigit() and destinationvalue.isdigit(): - if int(sourcevalue) > int(destinationvalue): - return True - except AttributeError as e: - self.logger.error("Condition larger than failed with values %s and %s: %s" % (sourcevalue, destinationvalue, e)) - return False - elif check.lower() == "smaller than": - try: - if sourcevalue.isdigit() and destinationvalue.isdigit(): - if int(sourcevalue) < int(destinationvalue): - return True - except AttributeError as e: - self.logger.error("Condition smaller than failed with values %s and %s: %s" % (sourcevalue, destinationvalue, e)) - return False - else: - self.logger.info("Condition: can't handle %s yet. Setting to true" % check) - - return False - - def check_branch_conditions(action, fullexecution): - # relevantbranches = workflow.branches where destination = action - try: - if fullexecution["workflow"]["branches"] == None or len(fullexecution["workflow"]["branches"]) == 0: - return True, "" - except KeyError: - return True, "" - - relevantbranches = [] - for branch in fullexecution["workflow"]["branches"]: - if branch["destination_id"] != action["id"]: - continue - - # Remove anything without a condition - try: - if (branch["conditions"]) == 0 or branch["conditions"] == None: - continue - except KeyError: - continue - - self.logger.info("Relevant conditions: %s" % branch["conditions"]) - successful_conditions = [] - failed_conditions = [] - for condition in branch["conditions"]: - self.logger.info("Getting condition value of %s" % condition) - - # Parse all values first here - sourcevalue = condition["source"]["value"] - check, sourcevalue, is_loop = parse_params(action, fullexecution, condition["source"]) - if check: - return False, "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check) - - - #sourcevalue = sourcevalue.encode("utf-8") - sourcevalue = parse_wrapper_start(sourcevalue) - destinationvalue = condition["destination"]["value"] - - check, destinationvalue, is_loop = parse_params(action, fullexecution, condition["destination"]) - if check: - return False, "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check) - - #destinationvalue = destinationvalue.encode("utf-8") - destinationvalue = parse_wrapper_start(destinationvalue) - available_checks = [ - "=", - "equals", - "!=", - "does not equal", - ">", - "larger than", - "<", - "less than", - ">=", - "<=", - "startswith", - "endswith", - "contains", - "re", - "matches regex", - ] - - # FIXME - what should I do here? - if not condition["condition"]["value"] in available_checks: - self.logger.info("Skipping %s %s %s because %s is invalid." % (sourcevalue, condition["condition"]["value"], destinationvalue, condition["condition"]["value"])) - continue - - #print(destinationvalue) - # NEGATE - validation = run_validation(sourcevalue, condition["condition"]["value"], destinationvalue) - - # Configuration = negated because of WorkflowAppActionParam.. - try: - if condition["condition"]["configuration"]: - validation = not validation - except KeyError: - pass - - if not validation: - self.logger.info("Failed condition check for %s %s %s." % (sourcevalue, condition["condition"]["value"], destinationvalue)) - return False, "Failed condition: %s %s %s" % (sourcevalue, condition["condition"]["value"], destinationvalue) - - - # Make a general parser here, at least to get param["name"] = param["value"] in maparameter[string]string - #for condition in branch.conditons: - - return True, "" - - # Checks whether conditions are met, otherwise set - branchcheck, tmpresult = check_branch_conditions(action, fullexecution) - if not branchcheck: - self.logger.info("Failed one or more branch conditions.") - action_result["result"] = tmpresult - action_result["status"] = "FAILURE" - try: - ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result) - self.logger.info("Result: %d" % ret.status_code) - if ret.status_code != 200: - self.logger.info(ret.text) - except requests.exceptions.ConnectionError as e: - self.logger.exception(e) - - print("\n\nRETURNING BECAUSE A BRANCH FAILED\n\n") - return - - # Replace name cus there might be issues - # Not doing lower() as there might be user-made functions - actionname = action["name"] - if " " in actionname: - actionname.replace(" ", "_", -1) - #if action.generated: - # actionname = actionname.lower() - - # Runs the actual functions - try: - func = getattr(self, actionname, None) - if func == None: - self.logger.debug("Failed executing %s because func is None." % actionname) - action_result["status"] = "FAILURE" - action_result["result"] = "Function %s doesn't exist." % actionname - elif callable(func): - try: - if len(action["parameters"]) < 1: - result = await func() - else: - # Potentially parse JSON here - # FIXME - add potential authentication as first parameter(s) here - # params[parameter["name"]] = parameter["value"] - #print(fullexecution["authentication"] - # What variables are necessary here tho hmm - - params = {} - try: - for item in action["authentication"]: - print("AUTH: ", key, value) - params[item["key"]] = item["value"] - except KeyError: - print("No authentication specified!") - pass - #action["authentication"] - - # calltimes is used to handle forloops in the app itself. - # 2 kinds of loop - one in gui with one app each, and one like this, - # which is super fast, but has a bad overview (potentially good tho) - calltimes = 1 - result = "" - - all_executions = [] - - # Multi_parameter has the data for each. variable - minlength = 0 - multi_parameters = json.loads(json.dumps(params)) - multiexecution = False - multi_execution_lists = [] - for parameter in action["parameters"]: - check, value, is_loop = parse_params(action, fullexecution, parameter) - - if check: - raise "Value check error: %s" % Exception(check) - - # Custom format for ${name[0,1,2,...]}$ - #submatch = "([${]{2}([0-9a-zA-Z_-]+)(\[.*\])[}$]{2})" - submatch = "([${]{2}([0-9a-zA-Z_-]+)(\[.*\])[}$]{2})" - actualitem = re.findall(submatch, value, re.MULTILINE) - try: - if action["skip_multicheck"]: - print("Skipping multicheck") - actualitem = [] - except KeyError: - pass - - print("Return value: %s" % value) - actionname = action["name"] - #print("Multicheck ", actualitem) - print("Actual item: %s" % actualitem) - if len(actualitem) > 0: - multiexecution = True - - # Loop WITHOUT JSON variables go here. - # Loop WITH variables go in else. - if len(actualitem[0]) > 2 and actualitem[0][1] == "SHUFFLE_NO_SPLITTER": - print("Pre replacement: %s" % actualitem[0][2]) - tmpitem = value - - replacement = actualitem[0][2] - if replacement.startswith("\"") and replacement.endswith("\""): - replacement = replacement[1:len(replacement)-1] - - replacement = replacement.replace("\'", "\"", -1) - print("POST replacement: %s" % replacement) - - json_replacement = replacement - try: - json_replacement = json.loads(replacement) - except json.decoder.JSONDecodeError as e: - print("JSON error singular: %s" % e) - - if len(json_replacement) > minlength: - minlength = len(json_replacement) - - tmpitem = tmpitem.replace(actualitem[0][0], replacement, 1) - params[parameter["name"]] = tmpitem - multi_execution_lists.append(json_replacement) - multi_parameters[parameter["name"]] = json_replacement - - #print("LENGTH OF ARR: %d" % len(resultarray)) - #print("RESULTARRAY: %s" % resultarray) - print("MULTI finished: %s" % replacement) - else: - - # This is here to handle for loops within variables.. kindof - # 1. Find the length of the longest array - # 2. Build an array with the base values based on parameter["value"] - # 3. Get the n'th value of the generated list from values - # 4. Execute all n answers - replacements = {} - for replace in actualitem: - try: - to_be_replaced = replace[0] - actualitem = replace[2] - except IndexError: - continue - - try: - itemlist = json.loads(actualitem) - if len(itemlist) > minlength: - minlength = len(itemlist) - except json.decoder.JSONDecodeError as e: - print("JSON Error: %s in %s" % (e, actualitem)) - - replacements[to_be_replaced] = actualitem - - # This is a result array for JUST this value.. - # What if there are more? - resultarray = [] - for i in range(0, minlength): - tmpitem = json.loads(json.dumps(parameter["value"])) - for key, value in replacements.items(): - replacement = json.dumps(json.loads(value)[i]) - if replacement.startswith("\"") and replacement.endswith("\""): - replacement = replacement[1:len(replacement)-1] - #except json.decoder.JSONDecodeError as e: - - #print("REPLACING %s with %s" % (key, replacement)) - #replacement = parse_wrapper_start(replacement) - tmpitem = tmpitem.replace(key, replacement, -1) - - resultarray.append(tmpitem) - - # With this parameter ready, add it to... a greater list of parameters. Rofl - print("LENGTH OF ARR: %d" % len(resultarray)) - print("RESULTARRAY: %s" % resultarray) - if resultarray not in multi_execution_lists: - multi_execution_lists.append(resultarray) - - multi_parameters[parameter["name"]] = resultarray - else: - # Parses things like int(value) - self.logger.info("Parsing wrapper data for %s" % value) - value = parse_wrapper_start(value) - - params[parameter["name"]] = value - multi_parameters[parameter["name"]] = value - - # Fix lists here - print("CHECKING multi execution list!") - if len(multi_execution_lists) > 0: - print("\n Multi execution list has more data: %d" % len(multi_execution_lists)) - filteredlist = [] - for listitem in multi_execution_lists: - if listitem in filteredlist: - continue - - filteredlist.append(listitem) - - #print("New list length: %d" % len(filteredlist)) - if len(filteredlist) > 1: - print("Calculating new multi-loop length with %d lists" % len(filteredlist)) - tmplength = 1 - for innerlist in filteredlist: - print("List length: %d. %d*%d" % (len(innerlist), len(innerlist), tmplength)) - tmplength = len(innerlist)*tmplength - - minlength = tmplength - - print("New multi execution length: %d\n" % tmplength) - - # FIXME - this is horrible, but works for now - #for i in range(calltimes): - if not multiexecution: - print("APP_SDK DONE: Starting NORMAL execution of function") - newres = await func(**params) - #print("NEWRES: ", newres) - if isinstance(newres, str): - result += newres - else: - try: - result += str(newres) - except ValueError: - result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres) - print("Can't handle type %s value from function" % (type(newres))) - print("POST NEWRES RESULT: ", result) - else: - print("APP_SDK DONE: Starting MULTI execution with values %s of length %d" % (multi_parameters, minlength)) - # 1. Use number of executions based on the arrays being similar - # 2. Find the right value from the parsed multi_params - results = [] - json_object = False - for i in range(0, minlength): - # To be able to use the results as a list: - baseparams = json.loads(json.dumps(multi_parameters)) - # {'call': ['GoogleSafebrowsing_2_0', 'VirusTotal_GetReport_3_0']} - # 1. Check if list length is same as minlength - # 2. If NOT same length, duplicate based on length of array - # arraylength = 3 ["1", "2", "3"] - # arraylength = 4 ["1", "2", "3", "4"] - # minlength = 12 - 12/3 = 4 per item = ["1", "1", "1", "1", "2", "2", ...] - - try: - firstlist = True - for key, value in baseparams.items(): - - if isinstance(value, list): - try: - newvalue = value[i] - except IndexError: - pass - - if len(value) != minlength and len(value) > 0: - newarray = [] - print("VALUE: ", value) - additiontime = minlength/len(value) - print("Bad length for value: %d - should be %d. Additiontime: %d" % (len(value), minlength, additiontime)) - if firstlist: - print("Running normal list (FIRST)") - for subvalue in value: - for number in range(int(additiontime)): - newarray.append(subvalue) - else: - #print("Running secondary lists") - ## 1. Set up length of array - ## 2. Put values spread out - # FIXME: This works well, except if lists are same length - newarray = [""] * minlength - - cnt = 0 - for number in range(int(additiontime)): - for subvaluerange in range(len(value)): - # newlocation = number+(additiontime*subvaluerange) - # print("%d+(%d*%d) = %d. VAL: %s" % (number, additiontime, subvaluerange, newlocation, value[subvaluerange])) - # Reverse if same length? - if int(minlength/len(value)) == len(value): - tmp = int(len(value)-subvaluerange-1) - print("NEW: %d" % tmp) - newarray[cnt] = value[tmp] - else: - newarray[cnt] = value[subvaluerange] - cnt += 1 - - #print("Newarray =", newarray) - newvalue = newarray[i] - firstlist = False - - baseparams[key] = newvalue - except IndexError as e: - print("IndexError: %s" % e) - baseparams[key] = "IndexError: %s" % e - except KeyError as e: - print("KeyError: %s" % e) - baseparams[key] = "KeyError: %s" % e - - print("Running with params %s" % baseparams) - ret = await func(**baseparams) - if isinstance(ret, dict) or isinstance(ret, list): - results.append(ret) - json_object = True - else: - ret = ret.replace("\"", "\\\"", -1) - - try: - results.append(json.loads(ret)) - json_object = True - except json.decoder.JSONDecodeError as e: - #print("Json: %s" % e) - results.append(ret) - - #print("Inner ret parsed: %s" % ret) - - # Dump the result as a string of a list - #print("RESULTS: %s" % results) - if isinstance(results, list): - print("JSON OBJECT? ", json_object) - if json_object: - result = json.dumps(results) - else: - result = "[" - for item in results: - try: - json.loads(item) - result += item - except json.decoder.JSONDecodeError as e: - # Common nested issue which puts " around everything - try: - tmpitem = item.replace("\\\"", "\"", -1) - json.loads(tmpitem) - result += tmpitem - - except: - result += "\"%s\"" % item - - result += ", " - - result = result[:-2] - result += "]" - else: - print("Normal result?") - result = results - - print("RESULT: %s" % result) - action_result["status"] = "SUCCESS" - action_result["result"] = str(result) - if action_result["result"] == "": - action_result["result"] = result - - self.logger.debug(f"Executed {action['label']}-{action['id']} with result: {result}") - #self.logger.debug(f"Data: %s" % action_result) - except TypeError as e: - print("TypeError issue: %s" % e) - action_result["status"] = "FAILURE" - action_result["result"] = "TypeError: %s" % str(e) - else: - print("Function %s doesn't exist?" % action["name"]) - self.logger.error(f"App {self.__class__.__name__}.{action['name']} is not callable") - action_result["status"] = "FAILURE" - action_result["result"] = "Function %s is not callable." % actionname - - except Exception as e: - print(f"Failed to execute: {e}") - self.logger.exception(f"Failed to execute {e}-{action['id']}") - action_result["status"] = "FAILURE" - action_result["result"] = f"General exception: {e}" - - action_result["completed_at"] = int(time.time()) - - # Send the result :) - self.send_result(action_result, headers, stream_path) - return - - - #STOPCOPY - # !!! Let the above line stay - its used for some horrible codegeneration / stitching !!! # - - @classmethod - async def run(cls): - """ Connect to Redis and HTTP session, await actions """ - logging.basicConfig(format="{asctime} - {name} - {levelname}:{message}", style='{') - logger = logging.getLogger(f"{cls.__name__}") - logger.setLevel(logging.DEBUG) - print("Started execution!!") - - app = cls(redis=None, logger=logger, console_logger=logger) - - # Authorization for the app/function to control the workflow - # Function will crash if its wrong, which it probably should. - - await app.execute_action(app.action) diff --git a/backend/app_sdk_kali/build.sh b/backend/app_sdk_kali/build.sh deleted file mode 100644 index 78c76ced..00000000 --- a/backend/app_sdk_kali/build.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -NAME=app_sdk_kali -VERSION=0.7.3 - -docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force -docker build . -t frikky/shuffle:$NAME -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION - -#docker push frikky/$NAME:$VERSION -#docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -#docker push ghcr.io/frikky/$NAME:$VERSION - -docker push frikky/shuffle:$NAME -docker push ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/app_sdk_kali/requirements.txt b/backend/app_sdk_kali/requirements.txt deleted file mode 100644 index 804abb1b..00000000 --- a/backend/app_sdk_kali/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -requests -urllib3 diff --git a/backend/app_sdk_kali/static_baseline.py b/backend/app_sdk_kali/static_baseline.py deleted file mode 100644 index 152131cd..00000000 --- a/backend/app_sdk_kali/static_baseline.py +++ /dev/null @@ -1,76 +0,0 @@ -import os -import sys -import time -import logging -import requests - -# Goal here: -# * Make an app from WALKOFF able to run without app_base.py from WALKOFF -# # How: -# * Make it rely 100% on INPUT throug HTTP invocations instead of redis READS -# # But really, how? -# * Make a WORKER that reads the queue, and reuses a function - -# Here to get it global -apikey = "" -try: - apikey = os.environ["FUNCTION_APIKEY"] -except KeyError: - pass - -# Authorize the execution -def authorization(request): - # This is basically my issue, but it enforces the use of an internal API key for execution - try: - apikey = os.environ["FUNCTION_APIKEY"] - except KeyError: - return f"Internal server error", 500 - - - # Check API key from ENV authentication - authentication = request.headers.get("Authorization") - if authentication == None: return f"Unauthorized", 401 - - apikey_split = authentication.split(" ") - if apikey_split[0] != "Bearer" or len(apikey_split) != 2: - return f"Apikey error", 401 - - if apikey != apikey_split[1]: - return f"Unauthorized", 401 - - return run(request) - -class AppBase: - """ The base class for Python-based Walkoff applications, handles Redis and logging configurations. """ - __version__ = None - app_name = None - - def __init__(self, redis=None, logger=None, console_logger=None):#, docker_client=None): - self.logger = logger if logger is not None else logging.getLogger("AppBaseLogger") - self.redis=redis - self.console_logger=console_logger - self.current_execution_id = None - self.url = "https://shuffler.io" - self.apikey = apikey - - @classmethod - async def run(cls, action): - """ Connect to Redis and HTTP session, await actions """ - logging.basicConfig(format="{asctime} - {name} - {levelname}:{message}", style='{') - logger = logging.getLogger(f"{cls.__name__}") - logger.setLevel(logging.DEBUG) - - app = cls(redis=None, logger=logger, console_logger=logger) - - # Authorization for the app/function to control the workflow - # Function will crash if its wrong, which it probably should. - - await app.execute_action(action) - - async def execute_action(self, action): - # FIXME - add request for the function STARTING here. Use "results stream" or something - # PAUSED, AWAITING_DATA, PENDING, COMPLETED, ABORTED, EXECUTING, SUCCESS, FAILURE - - self.authorization = action["authorization"] - self.execution_id = action["execution_id"] - self.current_execution_id = action["execution_id"] diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 09027148..fcc25a96 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -2,7 +2,7 @@ module shuffle go 1.13 -//replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared +replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi @@ -20,11 +20,10 @@ require ( github.com/docker/docker v20.10.3-0.20210216175712-646072ed6524+incompatible github.com/docker/go-connections v0.4.0 github.com/docker/go-units v0.4.0 // indirect - github.com/elastic/go-elasticsearch v0.0.0 // indirect - github.com/elastic/go-elasticsearch/v7 v7.12.0 // indirect + github.com/elastic/go-elasticsearch/v7 v7.13.1 // indirect github.com/frikky/kin-openapi v0.39.0 - github.com/frikky/shuffle-shared v0.0.68 - github.com/fsouza/go-dockerclient v1.7.2 // indirect + github.com/frikky/shuffle-shared v0.0.69 + github.com/fsouza/go-dockerclient v1.7.2 github.com/ghodss/yaml v1.0.0 github.com/go-git/go-billy/v5 v5.0.0 github.com/go-git/go-git/v5 v5.0.0 @@ -34,6 +33,7 @@ require ( github.com/h2non/filetype v1.0.12 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/satori/go.uuid v1.2.0 + go4.org v0.0.0-20201209231011-d4a079459e60 // indirect golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 golang.org/x/oauth2 v0.0.0-20210113160501-8b1d76fa0423 google.golang.org/api v0.36.0 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum deleted file mode 100644 index 2912b94d..00000000 --- a/backend/go-app/go.sum +++ /dev/null @@ -1,753 +0,0 @@ -bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0 h1:EpMNVUorLiZIELdMZbCYX/ByTFCdoYopYAGxaGVz9ms= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.66.0/go.mod h1:dgqGAjKCDxyhGTtC9dAREQGUJpkceNm1yt590Qno0Ko= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.75.0 h1:XgtDnVJRCPEUG21gjFiRPz4zI1Mjg16R+NYQjfmU4XY= -cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.6.0/go.mod h1:hyFDG0qSGdHNz8Q6nDN8rYIkld0q/+5uBZaelxiDLfE= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/datastore v1.4.0 h1:CFDJm15RpYXeEblQ0TMDUrYtqmBmbAWTy536nA8JIc8= -cloud.google.com/go/datastore v1.4.0/go.mod h1:d18825/a9bICdAIJy2EkHs9joU4RlIZ1t6l8WDdbdY0= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1 h1:ukjixP1wl0LpnZ6LWtZJ0mX5tBmjp1f8Sqer8Z2OMUU= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.7.0 h1:DzdLPI8Em+DEk7IzA2a10ivq3mxIEASC9GeNJ6FFt5Q= -cloud.google.com/go/storage v1.7.0/go.mod h1:jGMIBwF+L/tL6WN/W5InNgYYu4HP0DvGB6rQ1mufWfs= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.12.0 h1:4y3gHptW1EHVtcPAVE0eBBlFuGqEejTTG3KdIE0lUX4= -cloud.google.com/go/storage v1.12.0/go.mod h1:fFLk2dp2oAhDz8QFKwqrjdJvxSp/W2g7nillojlL5Ho= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= -github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= -github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU= -github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= -github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= -github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= -github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= -github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= -github.com/algolia/algoliasearch-client-go v2.25.0+incompatible h1:FGQr9l++u4uQPDXrW8jM5kNJm3Iw5SxEJJtYXSFmPRY= -github.com/algolia/algoliasearch-client-go/v3 v3.18.1 h1:FP2Xtqqs/sefR5Qluygp+jVV+juXzEdJaPrZTCDLhDQ= -github.com/algolia/algoliasearch-client-go/v3 v3.18.1/go.mod h1:i7tLoP7TYDmHX3Q7vkIOL4syVse/k5VJ+k0i8WqFiJk= -github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/basgys/goxml2json v1.1.0 h1:4ln5i4rseYfXNd86lGEB+Vi652IsIXIvggKM/BhUKVw= -github.com/basgys/goxml2json v1.1.0/go.mod h1:wH7a5Np/Q4QoECFIU8zTQlZwZkrilY0itPfecMw41Dw= -github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 h1:/P9/RL0xgWE+ehnCUUN5h3RpG3dmoMCOONO1CCvq23Y= -github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013/go.mod h1:pccXHIvs3TV/TUqSNyEvF99sxjX2r4FFRIyw6TZY9+w= -github.com/carlescere/scheduler v0.0.0-20170109141437-ee74d2f83d82 h1:9bAydALqAjBfPHd/eAiJBHnMZUYov8m2PkXVr+YGQeI= -github.com/carlescere/scheduler v0.0.0-20170109141437-ee74d2f83d82/go.mod h1:tyA14J0sA3Hph4dt+AfCjPrYR13+vVodshQSM7km9qw= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= -github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= -github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.3 h1:ijQT13JedHSHrQGWFcGEwzcNKrAGIiZ+jSD5QQG07SY= -github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e h1:6JKvHHt396/qabvMhnhUZvWaHZzfVfldxE60TK8YLhg= -github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= -github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= -github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= -github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= -github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= -github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= -github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo= -github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v20.10.3-0.20210216175712-646072ed6524+incompatible h1:Yu2uGErhwEoOT/OxAFe+/SiJCqRLs+pgcS5XKrDXnG4= -github.com/docker/docker v20.10.3-0.20210216175712-646072ed6524+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v20.10.6+incompatible h1:oXI3Vas8TI8Eu/EjH4srKHJBVqraSzJybhxY7Om9faQ= -github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/elastic/go-elasticsearch v0.0.0 h1:Pd5fqOuBxKxv83b0+xOAJDAkziWYwFinWnBO0y+TZaA= -github.com/elastic/go-elasticsearch v0.0.0/go.mod h1:TkBSJBuTyFdBnrNqoPc54FN0vKf5c04IdM4zuStJ7xg= -github.com/elastic/go-elasticsearch/v7 v7.12.0 h1:j4tvcMrZJLp39L2NYvBb7f+lHKPqPHSL3nvB8+/DV+s= -github.com/elastic/go-elasticsearch/v7 v7.12.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= -github.com/elastic/go-elasticsearch/v8 v8.0.0-20210519083322-55daf7425ecb h1:svC8T5+v+aWpWiTt3nsGfpdqVb4NIWK/WamGXXECBXA= -github.com/elastic/go-elasticsearch/v8 v8.0.0-20210519083322-55daf7425ecb/go.mod h1:xe9a/L2aeOgFKKgrO3ibQTnMdpAeL0GC+5/HpGScSa4= -github.com/elastic/go-elasticsearch/v8 v8.0.0-20210608143047-aa1301e7ba9d h1:id0CyeIuvJ9hzYhLlKsXHQ10d/k4G+CexAu53Pl3hf4= -github.com/elastic/go-elasticsearch/v8 v8.0.0-20210608143047-aa1301e7ba9d/go.mod h1:xe9a/L2aeOgFKKgrO3ibQTnMdpAeL0GC+5/HpGScSa4= -github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= -github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= -github.com/frikky/kin-openapi v0.38.0 h1:V7ttwIJS8Vks4KL+mZVj1ZSqhIcQtgaG8akeqXEQgsE= -github.com/frikky/kin-openapi v0.38.0/go.mod h1:Fr28TtCHL4K0kIqtqui8HWxN1LG5uAh3z/tDfFyiA1s= -github.com/frikky/kin-openapi v0.39.0 h1:qBrbLo9XwTIgIRJFsfXrmNzQz291xiiG0IcoML21t5s= -github.com/frikky/kin-openapi v0.39.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4= -github.com/frikky/shuffle-shared v0.0.12 h1:+0EIfThmK47Po+LogPYZR4XjbS4Ds19WNMFu2YUSjhw= -github.com/frikky/shuffle-shared v0.0.12/go.mod h1:SEY432/xs4oBkOUnGwxiYKCZWZLRaheGInhAoW7N8ww= -github.com/frikky/shuffle-shared v0.0.15 h1:508ceeEHfPBMCC8/K4Zve3kwRQqiXNJSw6+BDoq9X4E= -github.com/frikky/shuffle-shared v0.0.15/go.mod h1:SEY432/xs4oBkOUnGwxiYKCZWZLRaheGInhAoW7N8ww= -github.com/frikky/shuffle-shared v0.0.20 h1:y6JlPnQDq//elICWvVfUfJyU9gH3fSpQmPy+agqZ5sA= -github.com/frikky/shuffle-shared v0.0.20/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= -github.com/frikky/shuffle-shared v0.0.21 h1:xj/XPsXTa2rx41mm4nUc7+2K9RGkq2/mpjSPtIfpjE4= -github.com/frikky/shuffle-shared v0.0.21/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= -github.com/frikky/shuffle-shared v0.0.22 h1:TFMcJCNmOOSneMMWbg5dNzp2z6m0aZLROuL+bzVToRE= -github.com/frikky/shuffle-shared v0.0.22/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= -github.com/frikky/shuffle-shared v0.0.23 h1:Pnlc2M6fHnFRLFd5K1iLTVv4/t4P04Ri1GJ5CMxzq0U= -github.com/frikky/shuffle-shared v0.0.23/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= -github.com/frikky/shuffle-shared v0.0.27 h1:BbibbAv3a5GWR/DfaoSC4D9+fh2cwSEvn9H+EVfd7BM= -github.com/frikky/shuffle-shared v0.0.27/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= -github.com/frikky/shuffle-shared v0.0.28 h1:VQqL3+ePwKSUxCOiCC8DpOEgbb2GhXI8XzFB/YlHbps= -github.com/frikky/shuffle-shared v0.0.28/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= -github.com/frikky/shuffle-shared v0.0.32 h1:Uy/zcAetSVYtRr3HEkUb7aE7Ggm0oSFxVeUNsi6q4uc= -github.com/frikky/shuffle-shared v0.0.32/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= -github.com/frikky/shuffle-shared v0.0.37 h1:6nN1Im22TBuWUCG5L619xTgEPIPXCY68QThIfNiuG8k= -github.com/frikky/shuffle-shared v0.0.37/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= -github.com/frikky/shuffle-shared v0.0.38 h1:OZSwU1HDOaPzdlG1s77svgXJKzlNewM6GjeH1/8EIUM= -github.com/frikky/shuffle-shared v0.0.38/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= -github.com/frikky/shuffle-shared v0.0.40 h1:H0au2np5xSy9mZEUWN+a29IORk+YP95FipENBD7iJRw= -github.com/frikky/shuffle-shared v0.0.40/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ= -github.com/frikky/shuffle-shared v0.0.46 h1:L52pyEVKZujM136qzD7LnarJIq0iXyfpDqhGrJkE7ik= -github.com/frikky/shuffle-shared v0.0.46/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= -github.com/frikky/shuffle-shared v0.0.47 h1:fywEmbJGbD/VT9LdkHvLWPGh9HEt8ipFDzoGHRe+QgA= -github.com/frikky/shuffle-shared v0.0.47/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= -github.com/frikky/shuffle-shared v0.0.49 h1:fChF0Nh/bMuXZg67Pt9XXn9+mH4IlKgB3dAzhqwQF5o= -github.com/frikky/shuffle-shared v0.0.49/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= -github.com/frikky/shuffle-shared v0.0.50 h1:dQIXf4mwUHuEVsXiMtZaSznz6vWt+C0KjyTAsAgMs3s= -github.com/frikky/shuffle-shared v0.0.50/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= -github.com/frikky/shuffle-shared v0.0.51 h1:JrCGoRNj/LkAvSlkyx7tLv7ToNtJDIAqKqiA/poO+G4= -github.com/frikky/shuffle-shared v0.0.51/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= -github.com/frikky/shuffle-shared v0.0.52 h1:sCSJl6WakYit32UjaRn0wsy4YhTBE6QZbCofYKtuATg= -github.com/frikky/shuffle-shared v0.0.52/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= -github.com/frikky/shuffle-shared v0.0.53 h1:TszF/PoJ3JrfEf7qCGdN0So3bDbofNxh4Fqqz1KlH7Q= -github.com/frikky/shuffle-shared v0.0.53/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= -github.com/frikky/shuffle-shared v0.0.54 h1:rc8JcavY6uDxaIkXFwLVotfv3/epXUzuCWzRCozZNGg= -github.com/frikky/shuffle-shared v0.0.54/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= -github.com/frikky/shuffle-shared v0.0.56 h1:stC793SdQeBh98yJqCL74aXFo9YYhVIg8CQJ7hm4d6o= -github.com/frikky/shuffle-shared v0.0.56/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= -github.com/frikky/shuffle-shared v0.0.57 h1:YDlOVjg8bUBcinehcmorxzOoV5O55oUfG8q7TOiBOeU= -github.com/frikky/shuffle-shared v0.0.57/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= -github.com/frikky/shuffle-shared v0.0.60 h1:o6/QLsu3Rbjr4+BQWs9DF4B5qZCg0gPpZWPYXmjVWqM= -github.com/frikky/shuffle-shared v0.0.60/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= -github.com/frikky/shuffle-shared v0.0.62 h1:1M8y7rX8nQW7072+bUD4vgHcf65AG0kJ8m3ihmY2bPQ= -github.com/frikky/shuffle-shared v0.0.62/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= -github.com/frikky/shuffle-shared v0.0.63 h1:btn7V7s98eZmx/9qyapGE3V1TyxftvjMmrzkEcUvu+c= -github.com/frikky/shuffle-shared v0.0.63/go.mod h1:oPDyGFBuurPtE6UQLUrHU/JtbOJftqDljRfEdLlCTI4= -github.com/frikky/shuffle-shared v0.0.65 h1:TvISX6WE1Y7M2UJ3YuMv0PtgRPWtMj5iGiArVmv6SI8= -github.com/frikky/shuffle-shared v0.0.65/go.mod h1:oPDyGFBuurPtE6UQLUrHU/JtbOJftqDljRfEdLlCTI4= -github.com/frikky/shuffle-shared v0.0.66 h1:1vIcG5ZirIVQ+Li10y9PBHRUOHwpL45L0Nbmpnmpnrw= -github.com/frikky/shuffle-shared v0.0.66/go.mod h1:oPDyGFBuurPtE6UQLUrHU/JtbOJftqDljRfEdLlCTI4= -github.com/frikky/shuffle-shared v0.0.67 h1:Zl09+6rFnlpeOagDXvagC661bdhVm4DC0dylsiZ8dn4= -github.com/frikky/shuffle-shared v0.0.67/go.mod h1:oPDyGFBuurPtE6UQLUrHU/JtbOJftqDljRfEdLlCTI4= -github.com/fsouza/go-dockerclient v1.7.2 h1:bBEAcqLTkpq205jooP5RVroUKiVEWgGecHyeZc4OFjo= -github.com/fsouza/go-dockerclient v1.7.2/go.mod h1:+ugtMCVRwnPfY7d8/baCzZ3uwB0BrG5DB8OzbtxaRz8= -github.com/getkin/kin-openapi v0.8.0 h1:a6TQjTqwkyscC4/hShJX7WhCVE+4bi9lzw61XHQW5hE= -github.com/getkin/kin-openapi v0.8.0/go.mod h1:zZQMFkVgRHCdhgb6ihCTIo9dyDZFvX0k/xAKqw1FhPw= -github.com/getkin/kin-openapi v0.52.0 h1:6WqsF5d6PfJ8AscdD+9Rtb2RP2iBWyC7V6GcjssWg7M= -github.com/getkin/kin-openapi v0.52.0/go.mod h1:fRpo2Nw4Czgy0QnrIesRrEXs5+15N1F9mGZLP/aIomE= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= -github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= -github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= -github.com/go-git/go-billy v4.2.0+incompatible h1:Z6QtVXd5tjxUtcODLugkJg4WaZnGg13CD8qB9pr+7q0= -github.com/go-git/go-billy/v5 v5.0.0 h1:7NQHvd9FVid8VL4qVUMm8XifBK+2xCoZ2lSk0agRrHM= -github.com/go-git/go-billy/v5 v5.0.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= -github.com/go-git/go-git v4.7.0+incompatible h1:+W9rgGY4DOKKdX2x6HxSR7HNeTxqiKrOvKnuittYVdA= -github.com/go-git/go-git-fixtures/v4 v4.0.1/go.mod h1:m+ICp2rF3jDhFgEZ/8yziagdT1C+ZpZcrJjappBCDSw= -github.com/go-git/go-git/v5 v5.0.0 h1:k5RWPm4iJwYtfWoxIJy4wJX9ON7ihPeZZYC1fLYDnpg= -github.com/go-git/go-git/v5 v5.0.0/go.mod h1:oYD8y9kWsGINPFJoLdaScGCN6dlKg23blmClfZwtUVA= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= -github.com/google/go-github/v28 v28.1.1 h1:kORf5ekX5qwXO2mGzXXOjMe/g6ap8ahVe0sBEulhSxo= -github.com/google/go-github/v28 v28.1.1/go.mod h1:bsqJWQX05omyWVmc00nEUql9mhQyv38lDZ8kPZcQVoM= -github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200905233945-acf8798be1f7/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gorilla/handlers v1.4.2 h1:0QniY0USkHQ1RGCLfKxeNHK9bkDHGRYGNDFBCS+YARg= -github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= -github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/h2non/filetype v1.0.12 h1:yHCsIe0y2cvbDARtJhGBTD2ecvqMSTvlIcph9En/Zao= -github.com/h2non/filetype v1.0.12/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY= -github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/moby/sys/mount v0.2.0 h1:WhCW5B355jtxndN5ovugJlMFJawbUODuW8fSnEH6SSM= -github.com/moby/sys/mount v0.2.0/go.mod h1:aAivFE2LB3W4bACsUXChRHQ0qKWsetY4Y9V7sxOougM= -github.com/moby/sys/mountinfo v0.4.0 h1:1KInV3Huv18akCu58V7lzNlt+jFmqlu1EaErnEHE/VM= -github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= -github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk= -github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= -github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= -github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJGY8Y= -github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= -github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= -github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= -github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79 h1:IaQbIIB2X/Mp/DKctl6ROxz1KyMlKp4uyvL6+kQ7C88= -golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5 h1:WQ8q63x+f/zpC8Ac1s9wLElVoHhm32p6tudrU72n1QA= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b h1:iFwSg7t5GZmB/Q5TjiEAsdoLDrdJRC1RiF2WhuV29Qw= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210113160501-8b1d76fa0423 h1:/hEknzWkMPCjTo7StMHRrBRa8YBbXuBWfck8680k3RE= -golang.org/x/oauth2 v0.0.0-20210113160501-8b1d76fa0423/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200409092240-59c9f1ba88fa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e h1:hq86ru83GdWTlfQFZGO4nZJTU4Bs2wfHl8oFHRaXsfc= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3 h1:kzM6+9dur93BcC2kVlYl34cHU+TYZLanmpSJHVMmL64= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210216224549-f992740a1bac h1:9glrpwtNjBYgRpb67AZJKHfzj1stG/8BL5H7In2oTC4= -golang.org/x/sys v0.0.0-20210216224549-f992740a1bac/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/term v0.0.0-20201113234701-d7a72108b828/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200409170454-77362c5149f0/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d h1:lzLdP95xJmMpwQ6LUHwrc5V7js93hTiY7gkznu0BgmY= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20200915173823-2db8f0ff891c/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= -golang.org/x/tools v0.0.0-20200918232735-d647fc253266/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.21.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.23.0 h1:YlvGEOq2NA2my8cZ/9V8BcEO9okD48FlJcdqN0xJL3s= -google.golang.org/api v0.23.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.31.0/go.mod h1:CL+9IBCa2WWU6gRuBWaKqGWLFFwbEUXkfeMkHLQWYWo= -google.golang.org/api v0.32.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0 h1:l2Nfbl2GPXdWorv+dT2XfinX2jOOw4zv1VhLstx+6rE= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200409111301-baae70f3302d/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31 h1:Bz1qTn2YRWV+9OKJtxHJiQKCiXIdf+kwuKXdt9cBxyU= -google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200831141814-d751682dd103/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200914193844-75d14daec038/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200921151605-7abf4a1a14d5/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210113195801-ae06605f4595 h1:x7nk+/4+SvuTDI4wnzQUlhvi+DTpyfncXBo3QWTFs7U= -google.golang.org/genproto v0.0.0-20210113195801-ae06605f4595/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.28.1/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.34.1 h1:ugq+9++ZQPFzM2pKUMCIK8gj9M0pFyuUWO9Q8kwEDQw= -google.golang.org/grpc v1.34.1/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg= -gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= -gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= -gopkg.in/src-d/go-git.v4 v4.13.1 h1:SRtFyV8Kxc0UP7aCHcijOMQGPxHSmMOPrzulQWolkYE= -gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8= -gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200506231410-2ff61e1afc86 h1:OfFoIUYv/me30yv7XlMy4F9RJw8DEm8WQ6QG1Ph4bH0= -gopkg.in/yaml.v3 v3.0.0-20200506231410-2ff61e1afc86/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 1e8a1206..825a20bb 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -7,7 +7,6 @@ import ( "bytes" "context" "crypto/md5" - "crypto/tls" //"crypto/x509" "encoding/hex" "encoding/json" @@ -31,7 +30,7 @@ import ( "cloud.google.com/go/storage" "google.golang.org/appengine/mail" - "github.com/elastic/go-elasticsearch/v8" + //"github.com/elastic/go-elasticsearch/v7" //"github.com/elastic/go-elasticsearch/v8/esapi" "github.com/frikky/kin-openapi/openapi2" @@ -4081,6 +4080,7 @@ func runInitEs(ctx context.Context) { } // Getting apps to see if we should initialize a test + // FIXME: Isn't this a little backwards? workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000) log.Printf("[INFO] Getting and validating workflowapps. Got %d with err %#v", len(workflowapps), err) if err != nil && len(workflowapps) == 0 { @@ -5304,7 +5304,7 @@ func migrateDatabase(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - es := getEsConfig() + es := shuffle.GetEsConfig() _, err := shuffle.RunInit(*dbclient, *es, storage.Client{}, gceProject, "onprem", false, "") if err != nil { log.Printf("[WARNING] Failed to start migration because of init issues: %s", err) @@ -5657,82 +5657,6 @@ func makeWorkflowPublic(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } -func getEsConfig() *elasticsearch.Client { - esUrl := os.Getenv("SHUFFLE_OPENSEARCH_URL") - if len(esUrl) == 0 { - esUrl = "http://shuffle-opensearch:9200" - } - - // https://github.com/elastic/go-elasticsearch/blob/f741c073f324c15d3d401d945ee05b0c410bd06d/elasticsearch.go#L98 - config := elasticsearch.Config{ - Addresses: []string{esUrl}, - Username: os.Getenv("SHUFFLE_OPENSEARCH_USERNAME"), - Password: os.Getenv("SHUFFLE_OPENSEARCH_PASSWORD"), - APIKey: os.Getenv("SHUFFLE_OPENSEARCH_APIKEY"), - CloudID: os.Getenv("SHUFFLE_OPENSEARCH_CLOUDID"), - } - - //config.Transport.TLSClientConfig - transport := http.DefaultTransport.(*http.Transport).Clone() - transport.MaxIdleConnsPerHost = 100 - transport.ResponseHeaderTimeout = time.Second * 10 - transport.Proxy = nil - - if len(os.Getenv("SHUFFLE_OPENSEARCH_PROXY")) > 0 { - httpProxy := os.Getenv("SHUFFLE_OPENSEARCH_PROXY") - - url_i := url.URL{} - url_proxy, err := url_i.Parse(httpProxy) - if err == nil { - log.Printf("[DEBUG] Setting Opensearch proxy to %s", httpProxy) - transport.Proxy = http.ProxyURL(url_proxy) - } else { - log.Printf("[ERROR] Failed setting proxy for %s", httpProxy) - } - } - - skipSSLVerify := false - if strings.ToLower(os.Getenv("SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY")) == "true" { - log.Printf("[DEBUG] SKIPPING SSL verification with Opensearch") - skipSSLVerify = true - } - - transport.TLSClientConfig = &tls.Config{ - MinVersion: tls.VersionTLS11, - InsecureSkipVerify: skipSSLVerify, - } - - //https://github.com/elastic/go-elasticsearch/blob/master/_examples/security/elasticsearch-cluster.yml - certificateLocation := os.Getenv("SHUFFLE_OPENSEARCH_CERTIFICATE_FILE") - if len(certificateLocation) > 0 { - cert, err := ioutil.ReadFile(certificateLocation) - if err != nil { - log.Fatalf("[WARNING] Failed configuring certificates: %s not found", err) - } else { - config.CACert = cert - - //if transport.TLSClientConfig.RootCAs, err = x509.SystemCertPool(); err != nil { - // log.Fatalf("[ERROR] Problem adding system CA: %s", err) - //} - - //// --> Add the custom certificate authority - //if ok := transport.TLSClientConfig.RootCAs.AppendCertsFromPEM(cert); !ok { - // log.Fatalf("[ERROR] Problem adding CA from file %q", *cert) - //} - } - - log.Printf("[INFO] Added certificate %#v elastic client.", certificateLocation) - } - config.Transport = transport - - es, err := elasticsearch.NewClient(config) - if err != nil { - log.Fatalf("[DEBUG] Database client for ELASTICSEARCH error during init (fatal): %s", err) - } - - return es -} - func initHandlers() { var err error ctx := context.Background() @@ -5744,7 +5668,7 @@ func initHandlers() { log.Fatalf("[DEBUG] Database client error during init: %s", err) } - es := getEsConfig() + es := shuffle.GetEsConfig() elasticConfig := "elasticsearch" if strings.ToLower(os.Getenv("SHUFFLE_ELASTIC")) == "false" { elasticConfig = "" diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 3d909ed8..a40590d9 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -2286,6 +2286,11 @@ const AngularWorkflow = (props) => { } } + console.log("TARGET: ", event.target.target().data()) + if (event.target.target().data("isButton") === true || event.target.target().data("isDescriptor") === true) { + event.target.remove() + return + } targetnode = -1 var sourcenode = workflow.triggers.findIndex(data => data.id === edge.source) @@ -3373,6 +3378,11 @@ const AngularWorkflow = (props) => { if (workflow.visual_branches !== undefined && workflow.visual_branches !== null && workflow.visual_branches.length > 0) { const visualedges = workflow.visual_branches.map((branch, index) => { const edge = { }; + + if (workflow.branches[index] === undefined) { + return {} + } + var conditions = workflow.branches[index].conditions if (conditions === undefined || conditions === null) { conditions = [] @@ -3400,6 +3410,9 @@ const AngularWorkflow = (props) => { var newedges = [] for (var key in edges) { var item = edges[key] + if (item.data === undefined) { + continue + } const sourcecheck = insertedNodes.find(data => data.data.id === item.data.source) const destcheck = insertedNodes.find(data => data.data.id === item.data.target)