diff --git a/backend/go-app/main.go b/backend/go-app/main.go index a0e4cba3..720ca901 100755 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -321,7 +321,7 @@ func checkError(cmdName string, cmdArgs []string) error { scanner := bufio.NewScanner(cmdReader) go func() { for scanner.Scan() { - fmt.Printf("Out: %s\n", scanner.Text()) + log.Printf("Out: %s\n", scanner.Text()) } }() @@ -1402,7 +1402,7 @@ func parseWorkflowParameters(resp http.ResponseWriter, request *http.Request) (m return t, err } - //fmt.Println(curjson.String()) + //log.Println(curjson.String()) //log.Printf("Parsing json a second time: %s", string(curjson.String())) err = json.Unmarshal(curjson.Bytes(), &t) @@ -2556,7 +2556,7 @@ func execSubprocess(cmdName string, cmdArgs []string) error { scanner := bufio.NewScanner(cmdReader) go func() { for scanner.Scan() { - fmt.Printf("Out: %s\n", scanner.Text()) + log.Printf("Out: %s\n", scanner.Text()) } }() @@ -3528,7 +3528,7 @@ func handleAppHotload(ctx context.Context, location string, forceUpdate bool) er return err } - _, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate) + _, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate, false) if err != nil { log.Printf("[WARNING] Githubfolders error: %s", err) return err @@ -3981,7 +3981,7 @@ func runInitCloudSetup() { if err != nil { log.Printf("[INFO] Failed initial setup: %s", err) } else { - log.Printf("[INFO] Ran initial setup!") + log.Printf("[INFO] Finished initial cloudsync setup!") } } @@ -4419,20 +4419,8 @@ func runInitEs(ctx context.Context) { } // Getting apps to see if we should initialize a test - // FIXME: Isn't this a little backwards? workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000, 0) log.Printf("[INFO] Getting and validating workflowapps. Got %d with err %#v", len(workflowapps), err) - - // accept any certificate (might be useful for testing) - //customGitClient := &http.Client{ - // Transport: &http.Transport{ - // TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - // }, - // Timeout: 15 * time.Second, - //} - //client.InstallProtocol("http", githttp.NewClient(customGitClient)) - //client.InstallProtocol("https", githttp.NewClient(customGitClient)) - if err != nil && len(workflowapps) == 0 { log.Printf("[WARNING] Failed getting apps (runInit): %s", err) } else if err == nil && len(workflowapps) < 10 { @@ -4442,8 +4430,9 @@ func runInitEs(ctx context.Context) { url := os.Getenv("SHUFFLE_APP_DOWNLOAD_LOCATION") if len(url) == 0 { - log.Printf("[INFO] Skipping download of apps since no URL is set. Default would be https://github.com/shuffle/shuffle-apps") - url = "https://github.com/shuffle/shuffle-apps" + log.Printf("[INFO] Skipping download of apps since no URL is set. Default would be https://github.com/shuffle/python-apps") + + url = "https://github.com/shuffle/python-apps" //url = "" //return } @@ -4483,7 +4472,7 @@ func runInitEs(ctx context.Context) { _ = r //iterateAppGithubFolders(fs, dir, "", "testing") - _, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate) + _, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate, true) if err != nil { log.Printf("[WARNING] Error from app load in init: %s", err) } @@ -5266,6 +5255,7 @@ func initHandlers() { // App specific. Partially Singul. r.HandleFunc("/api/v1/apps/categories", shuffle.GetActiveCategories).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/apps/categories/run", singul.RunCategoryAction).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/apps/{key}/execute", executeSingleAction).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/{key}/run", executeSingleAction).Methods("POST", "OPTIONS") diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 578bb585..c8a16dba 100755 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -2016,14 +2016,14 @@ func validateAppInput(resp http.ResponseWriter, request *http.Request) { //fmt.Printf("File type: %s. MIME: %s\n", kind.Extension, kind.MIME.Value) if kind == filetype.Unknown { - fmt.Println("Unknown file type") + log.Println("Unknown file type") resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } if kind.MIME.Value != "application/zip" { - fmt.Println("Not zip, can't unzip") + log.Println("Not zip, can't unzip") resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -3005,7 +3005,7 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) { } // Onlyname is used to -func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate bool) ([]shuffle.BuildLaterStruct, []shuffle.BuildLaterStruct, error) { +func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate, duringStartup bool) ([]shuffle.BuildLaterStruct, []shuffle.BuildLaterStruct, error) { var err error allapps := []shuffle.WorkflowApp{} @@ -3017,7 +3017,13 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. "YARA", "ATTACK-PREDICTOR", } - //if strings.ToUpper(workflowapp.Name) == strings.ToUpper(appname) { + + startupNames := []string{ + "shuffle-tools", + "http", + "email", + "shuffle-ai", + } // It's here to prevent getting them in every iteration buildLaterFirst := []shuffle.BuildLaterStruct{} @@ -3027,6 +3033,19 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. continue } + //duringStartup + if duringStartup { + // Look for names: shuffle tools, http, email, shuffle ai + if shuffle.ArrayContains(startupNames, strings.ToLower(file.Name())) { + // Allowed to build during startup + + //log.Printf("\n\n\nFOUND MATCHING APP: %s\n\n\n", file.Name()) + } else { + //log.Printf("\n\n\nWRONG APP (2): %s\n\n\n", file.Name()) + continue + } + } + // Folder? switch mode := file.Mode(); { case mode.IsDir(): @@ -3043,7 +3062,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. } // Go routine? Hmm, this can be super quick I guess - buildFirst, buildLast, err := IterateAppGithubFolders(ctx, fs, dir, tmpExtra, "", forceUpdate) + buildFirst, buildLast, err := IterateAppGithubFolders(ctx, fs, dir, tmpExtra, "", forceUpdate, false) for _, item := range buildFirst { buildLaterFirst = append(buildLaterFirst, item) @@ -3055,7 +3074,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. if err != nil { log.Printf("[WARNING] Error reading folder: %s", err) - //buildFirst, buildLast, err := IterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate) + //buildFirst, buildLast, err := IterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate, false) if !forceUpdate { continue @@ -3355,6 +3374,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. "http", "email", } + for _, buildLater := range buildLaterFirst { found := false for _, appname := range initApps { @@ -3373,12 +3393,19 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. } // Prepend newSortedList to buildLaterFirst + handledImages := []string{} buildLaterFirst = append(newSortedList, buildLaterFirst...) - if len(extra) == 0 { log.Printf("[INFO] Starting build of %d containers (FIRST)", len(buildLaterFirst)) for _, item := range buildLaterFirst { + + if len(item.Tags) > 0 && shuffle.ArrayContains(handledImages, item.Tags[0]) { + continue + } + + handledImages = append(handledImages, item.Tags[0]) err = buildImageMemory(fs, item.Tags, item.Extra, true) + if err != nil { orgId := "" @@ -3408,6 +3435,12 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. if len(buildLaterList) > 0 { log.Printf("[INFO] Starting build of %d skipped docker images", len(buildLaterList)) for _, item := range buildLaterList { + if len(item.Tags) > 0 && shuffle.ArrayContains(handledImages, item.Tags[0]) { + continue + } + + handledImages = append(handledImages, item.Tags[0]) + err = buildImageMemory(fs, item.Tags, item.Extra, true) if err != nil { log.Printf("[INFO] Failed image build memory: %s", err) @@ -3541,7 +3574,7 @@ func LoadSpecificApps(resp http.ResponseWriter, request *http.Request) { } } - IterateAppGithubFolders(ctx, fs, dir, "", "", tmpBody.ForceUpdate) + IterateAppGithubFolders(ctx, fs, dir, "", "", tmpBody.ForceUpdate, false) } else if strings.Contains(tmpBody.URL, "s3") { //https://docs.aws.amazon.com/sdk-for-go/api/service/s3/ diff --git a/docker-compose.yml b/docker-compose.yml index 1715535a..9ff6cc59 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -65,7 +65,7 @@ services: hostname: shuffle-opensearch container_name: shuffle-opensearch environment: - - "OPENSEARCH_JAVA_OPTS=-Xms2048m -Xmx2048m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM + - "OPENSEARCH_JAVA_OPTS=-Xms3072m -Xmx3072m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM - bootstrap.memory_lock=true - DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI=true - cluster.initial_master_nodes=shuffle-opensearch diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index a90b799e..e1adeb09 100755 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -12044,7 +12044,6 @@ const AngularWorkflow = (defaultprops) => { prioritizedApps, Array.prototype.concat.apply( filteredApps.filter((innerapp) => !internalIds.includes(innerapp.id.toLowerCase())), - triggers ) ) ) @@ -12820,9 +12819,9 @@ const AngularWorkflow = (defaultprops) => { : -