From b0c82db41b88e2b6a9a1400ab44520abcaa66aea Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 4 Jun 2023 16:48:11 +0200 Subject: [PATCH] Fixed some app loading prioritization and position issues --- backend/go-app/go.mod | 2 +- backend/go-app/main.go | 16 ++++++++----- backend/go-app/walkoff.go | 47 +++++++++++++++++++++++++++++++++------ 3 files changed, 51 insertions(+), 14 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 3bb19fce..eb328296 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -2,7 +2,7 @@ module main go 1.19 -//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared +replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared require ( cloud.google.com/go/datastore v1.10.0 diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 3fc9f9fa..8f77bd19 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3905,6 +3905,11 @@ func runInitEs(ctx context.Context) { } } + if strings.Contains(os.Getenv("SHUFFLE_OPENSEARCH_URL"), "https") { + log.Printf("[INFO] Waiting 10 seconds during init to make sure the opensearch instance is up and running with security features properly") + time.Sleep(10 * time.Second) + } + _ = setUsers schedules, err := shuffle.GetAllSchedules(ctx, "ALL") if err != nil { @@ -4188,7 +4193,8 @@ func runInitEs(ctx context.Context) { url := os.Getenv("SHUFFLE_APP_DOWNLOAD_LOCATION") if len(url) == 0 { - log.Printf("[INFO] Skipping download of apps since no URL is set. Default would be https://github.com/frikky/shuffle-apps") + log.Printf("[INFO] Skipping download of apps since no URL is set. Default would be https://github.com/shuffle/shuffle-apps") + url = "https://github.com/shuffle/shuffle-apps" //url = "" //return } @@ -4215,7 +4221,6 @@ func runInitEs(ctx context.Context) { log.Printf("[DEBUG] Getting apps from url '%s'", url) r, err := git.Clone(storer, fs, cloneOptions) - if err != nil { log.Printf("[WARNING] Failed loading repo into memory (init): %s", err) } @@ -4227,7 +4232,6 @@ func runInitEs(ctx context.Context) { _ = r //iterateAppGithubFolders(fs, dir, "", "testing") - // FIXME: Get all the apps? _, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate) if err != nil { log.Printf("[WARNING] Error from app load in init: %s", err) @@ -4242,7 +4246,7 @@ func runInitEs(ctx context.Context) { } log.Printf("[INFO] Downloading OpenAPI data for search - EXTRA APPS") - apis := "https://github.com/frikky/security-openapis" + apis := "https://github.com/shuffle/security-openapis" // THis gets memory problems hahah //apis := "https://github.com/APIs-guru/openapi-directory" @@ -4850,7 +4854,7 @@ func runInit(ctx context.Context) { url := os.Getenv("SHUFFLE_APP_DOWNLOAD_LOCATION") if len(url) == 0 { - url = "https://github.com/frikky/shuffle-apps" + url = "https://github.com/shuffle/shuffle-apps" } username := os.Getenv("SHUFFLE_DOWNLOAD_AUTH_USERNAME") @@ -4901,7 +4905,7 @@ func runInit(ctx context.Context) { } log.Printf("[INFO] Downloading OpenAPI data for search - EXTRA APPS") - apis := "https://github.com/frikky/security-openapis" + apis := "https://github.com/shuffle/security-openapis" // FIXME: This part gets memory problems. Fix in the future to load these apps too. //apis := "https://github.com/APIs-guru/openapi-directory" diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 02f3bd9e..d26cce0a 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -814,7 +814,7 @@ func JSONCheck(str string) bool { func handleExecutionStatistics(execution shuffle.WorkflowExecution) { // FIXME: CLEAN UP THE JSON THAT'S SAVED. - // https://github.com/frikky/Shuffle/issues/172 + // https://github.com/shuffle/Shuffle/issues/172 appResults := []shuffle.AppExecutionExample{} for _, result := range execution.Results { resultCheck := JSONCheck(result.Result) @@ -2370,7 +2370,7 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, if !found { err = shuffle.SetWorkflowAppDatastore(ctx, api, api.ID) if err != nil { - log.Printf("[WARNING] Failed setting workflowapp in loop: %s", err) + log.Printf("[WARNING] Failed setting workflowapp %s (%s) in loop: %s", api.Name, api.ID, err) continue } else { appCounter += 1 @@ -2599,7 +2599,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { // Might require reflection into the python code to append the fields as well for index, action := range workflowapp.Actions { if action.AuthNotRequired { - log.Printf("Skipping auth setup: %s", action.Name) + log.Printf("[WARNING] Skipping auth setup for: %s", action.Name) continue } @@ -2647,7 +2647,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { err = shuffle.SetWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) if err != nil { - log.Printf("Failed setting workflowapp: %s", err) + log.Printf("[WARNING] Failed setting workflowapp: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -2876,6 +2876,11 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. // Folder? switch mode := file.Mode(); { case mode.IsDir(): + // Specific folder for skipping + if file.Name() == "unsupported" { + continue + } + tmpExtra := fmt.Sprintf("%s%s/", extra, file.Name()) dir, err := fs.ReadDir(tmpExtra) if err != nil { @@ -2899,7 +2904,8 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. //buildFirst, buildLast, err := IterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate) if !forceUpdate { - return buildLaterFirst, buildLaterList, err + continue + //return buildLaterFirst, buildLaterList, err } } @@ -2973,7 +2979,8 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. err = gyaml.Unmarshal(appfileData, &workflowapp) if err != nil { log.Printf("[WARNING] Failed building workflowapp %s: %s", extra, err) - return buildLaterFirst, buildLaterList, errors.New(fmt.Sprintf("Failed building %s: %s", extra, err)) + continue + //return buildLaterFirst, buildLaterList, errors.New(fmt.Sprintf("Failed building %s: %s", extra, err)) //continue } @@ -3021,7 +3028,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. } } - workflowapp.ReferenceInfo.GithubUrl = fmt.Sprintf("https://github.com/frikky/shuffle-apps/tree/master/%s/%s", strings.ToLower(newName), workflowapp.AppVersion) + workflowapp.ReferenceInfo.GithubUrl = fmt.Sprintf("https://github.com/shuffle/shuffle-apps/tree/master/%s/%s", strings.ToLower(newName), workflowapp.AppVersion) tags := []string{ fmt.Sprintf("%s:%s_%s", baseDockerName, strings.ToLower(newName), workflowapp.AppVersion), @@ -3188,6 +3195,32 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. cacheKey = fmt.Sprintf("workflowapps-sorted-1000") shuffle.DeleteCache(ctx, cacheKey) + newSortedList := []shuffle.BuildLaterStruct{} + initApps := []string{ + "tools", + "http", + "email", + } + for _, buildLater := range buildLaterFirst { + found := false + for _, appname := range initApps { + for _, tag := range buildLater.Tags { + if strings.Contains(strings.ToLower(tag), appname) { + newSortedList = append(newSortedList, buildLater) + found = true + break + } + } + + if found { + break + } + } + } + + // Prepend newSortedList to buildLaterFirst + buildLaterFirst = append(newSortedList, buildLaterFirst...) + if len(extra) == 0 { log.Printf("[INFO] Starting build of %d containers (FIRST)", len(buildLaterFirst)) for _, item := range buildLaterFirst {