From 722d832d881de4bfeafc2424d8145c37f779002c Mon Sep 17 00:00:00 2001 From: frikky Date: Thu, 15 Oct 2020 17:12:13 +0200 Subject: [PATCH] Fixed App SDK issue for the n'th time --- backend/app_sdk/app_base.py | 14 +++- backend/app_sdk/build.sh | 2 +- backend/go-app/walkoff.go | 114 ++++------------------------ functions/onprem/orborus/orborus.go | 12 ++- 4 files changed, 32 insertions(+), 110 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index e896f67c..701a8b3f 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -875,12 +875,14 @@ class AppBase: print("Return value: %s" % value) actionname = action["name"] #print("Multicheck ", actualitem) - print("Actual item: %s" % actualitem) + print("ITEM LENGTH: %d, Actual item: %s" % (len(actualitem), actualitem)) if len(actualitem) > 0: multiexecution = True # Loop WITHOUT JSON variables go here. # Loop WITH variables go in else. + print("Before first part in multiexec!") + handled = False if len(actualitem[0]) > 2 and actualitem[0][1] == "SHUFFLE_NO_SPLITTER": print("Pre replacement: %s" % actualitem[0][2]) tmpitem = value @@ -910,13 +912,13 @@ class AppBase: #print("RESULTARRAY: %s" % resultarray) print("MULTI finished: %s" % replacement) else: - # This is here to handle for loops within variables.. kindof # 1. Find the length of the longest array # 2. Build an array with the base values based on parameter["value"] # 3. Get the n'th value of the generated list from values # 4. Execute all n answers replacements = {} + curminlength = 0 for replace in actualitem: try: to_be_replaced = replace[0] @@ -928,6 +930,9 @@ class AppBase: itemlist = json.loads(actualitem) if len(itemlist) > minlength: minlength = len(itemlist) + + if len(itemlist) > curminlength: + curminlength = len(itemlist) except json.decoder.JSONDecodeError as e: print("JSON Error: %s in %s" % (e, actualitem)) @@ -935,8 +940,9 @@ class AppBase: # This is a result array for JUST this value.. # What if there are more? + print("LENGTH: %d. In second part of else: %s" % (len(itemlist), replacements)) resultarray = [] - for i in range(0, minlength): + for i in range(0, curminlength): tmpitem = json.loads(json.dumps(parameter["value"])) for key, value in replacements.items(): replacement = json.dumps(json.loads(value)[i]) @@ -959,7 +965,7 @@ class AppBase: multi_parameters[parameter["name"]] = resultarray else: # Parses things like int(value) - self.logger.info("Parsing wrapper data for %s" % value) + print("Normal parsing (not looping) with data %s" % value) value = parse_wrapper_start(value) params[parameter["name"]] = value diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index b1ac5113..2f622fcd 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,6 +1,6 @@ #!/bin/bash NAME=app_sdk -VERSION=0.7.3 +VERSION=0.7.5 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -t frikky/shuffle:$NAME -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 95f54058..96734540 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -28,7 +28,6 @@ import ( "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/storage/memory" http2 "gopkg.in/src-d/go-git.v4/plumbing/transport/http" @@ -1552,12 +1551,6 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { log.Printf("Failed to increase total workflows: %s", err) } - // recalculate authenticators stats - err = recalculateAppAuthentications() - if err != nil { - log.Printf("Authentications recalculation failed: %s", err) - } - //memcacheName := fmt.Sprintf("%s_%s", user.Username, fileId) //memcache.Delete(ctx, memcacheName) //memcacheName = fmt.Sprintf("%s_workflows", user.Username) @@ -1721,6 +1714,11 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { for _, action := range workflow.Actions { allNodes = append(allNodes, action.ID) + if len(action.Errors) > 0 { + action.IsValid = true + action.Errors = []string{} + } + if action.Environment == "" { resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "An environment for %s is required"}`, action.Label))) @@ -4403,7 +4401,6 @@ func deployWebhookFunction(ctx context.Context, name, localization, applocation func loadGithubWorkflows(url, username, password, userId, branch string) error { fs := memfs.New() - // FIXME: add more git options lol if strings.Contains(url, "github") || strings.Contains(url, "gitlab") || strings.Contains(url, "bitbucket") { cloneOptions := &git.CloneOptions{ URL: url, @@ -4412,11 +4409,16 @@ func loadGithubWorkflows(url, username, password, userId, branch string) error { // FIXME: Better auth. if len(username) > 0 && len(password) > 0 { cloneOptions.Auth = &http2.BasicAuth{ + Username: username, Password: password, } } + if len(branch) > 0 { + cloneOptions.ReferenceName = plumbing.ReferenceName(branch) + } + storer := memory.NewStorage() r, err := git.Clone(storer, fs, cloneOptions) if err != nil { @@ -4424,31 +4426,9 @@ func loadGithubWorkflows(url, username, password, userId, branch string) error { return err } - if len(branch) > 0 { - log.Printf("Checkout to branch: %s", branch) - - w, _ := r.Worktree() - - err := r.Fetch(&git.FetchOptions{ - RefSpecs: []config.RefSpec{"refs/*:refs/*", "HEAD:refs/heads/HEAD"}, - }) - if err != nil { - log.Printf("Failed fetch for git repo: %s", err) - } - - err = w.Checkout(&git.CheckoutOptions{ - Branch: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branch)), - Force: true, - }) - if err != nil { - log.Printf("Failed checkout for git repo: %s", err) - return errors.New(fmt.Sprintf("Failed checking out to branch %s - does it exist?", branch)) - } - } - dir, err := fs.ReadDir("/") if err != nil { - log.Printf("Failed reading folder: %s", err) + log.Printf("FAiled reading folder: %s", err) } _ = r @@ -4523,7 +4503,7 @@ func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) { if err != nil { log.Printf("Error with unmarshal tmpBody: %s", err) resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "json decode error"}`)) + resp.Write([]byte(`{"success": false}`)) return } @@ -4532,82 +4512,14 @@ func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) { if err != nil { log.Printf("Failed to update workflows: %s", err) resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + resp.Write([]byte(`{"success": false}`)) return } - // recalculate authenticators stats - err = recalculateAppAuthentications() - if err != nil { - log.Printf("Authentications recalculation failed: %s", err) - } - resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } -func recalculateAppAuthentications() error { - // create context - ctx := context.Background() - - // form workflows list - workflows, err := getAllWorkflows(ctx) - if err != nil { - log.Printf("Error: Failed getting workflows: %s", err) - return err - } - - // form authenticators list - auths, err := getAllWorkflowAppAuth(ctx) - if err != nil { - log.Printf("Error: Failed getting auths: %s", err) - return err - } - - // iterate through auths - for _, auth := range auths { - // reset calculated values - auth.WorkflowCount = 0 - auth.NodeCount = 0 - auth.Usage = []AuthenticationUsage{} - - // iterate through workflows to find which uses this auth - for _, workflow := range workflows { - hasCurrentAuth := false - usageItem := AuthenticationUsage{ - WorkflowId: workflow.ID, - Nodes: []string{}, - } - - // iterate through actions - for _, action := range workflow.Actions { - if action.AuthenticationId == auth.Id { - // this workflow should be added to "usage" field - hasCurrentAuth = true - - // add this action to list - usageItem.Nodes = append(usageItem.Nodes, action.ID) - } - } - - // update current auth with found workflow - if hasCurrentAuth { - auth.WorkflowCount += 1 - auth.NodeCount += int64(len(usageItem.Nodes)) - auth.Usage = append(auth.Usage, usageItem) - } - } - - // update record in database - err := setWorkflowAppAuthDatastore(ctx, auth, auth.Id) - if err != nil { - log.Printf("Failed setting up app auth %s: %s", auth.Id, err) - } - } - - return nil -} - func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index b18a2793..2520377e 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -126,7 +126,7 @@ func deployWorker(image string, identifier string, env []string) { log.Printf("[INFO] Found container ID %s", containerId) hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId)) } else { - log.Printf("[INFO] Empty self container id, continue without NetworkMode") + //log.Printf("[INFO] Empty self container id, continue without NetworkMode") } config := &container.Config{ @@ -144,7 +144,7 @@ func deployWorker(image string, identifier string, env []string) { ) if err != nil { - log.Println(err) + log.Printf("[ERROR] Container create error: %s", err) return } @@ -218,9 +218,11 @@ func initializeImages() { if baseimageregistry == "" { baseimageregistry = "docker.io" + log.Printf("Setting baseimageregistry") } if baseimagename == "" { baseimagename = "frikky/shuffle" + log.Printf("Setting baseimagename") } // check whether they are the same first @@ -290,13 +292,15 @@ func main() { // FIXME - during init, BUILD and/or LOAD worker and app_sdk // Build/load app_sdk so it can be loaded as 127.0.0.1:5000/walkoff_app_sdk log.Printf("[INFO] Setting up Docker environment. Downloading worker and App SDK!") - go initializeImages() + + initializeImages() //workerName := "worker" //workerVersion := "0.1.0" //workerImage := fmt.Sprintf("docker.pkg.github.com/frikky/shuffle/%s:%s", workerName, workerVersion) //workerImage := fmt.Sprintf("%s/worker:%s", baseimagename, workerVersion) // workerImage := fmt.Sprintf("docker.io/%s:worker", baseimagename) + // fmt.Sprintf("%s/%s:app_sdk%s", baseimageregistry, baseimagename, baseimagetagsuffix), workerImage := fmt.Sprintf("%s/%s:worker%s", baseimageregistry, baseimagename, baseimagetagsuffix) log.Printf("[INFO] Finished configuring docker environment") @@ -387,7 +391,7 @@ func main() { } if hasStarted && len(executionRequests.Data) > 0 { - log.Printf("[INFO] Body: %s", string(body)) + //log.Printf("[INFO] Body: %s", string(body)) // Type string `json:"type"` }