Opensearch and app optimisations

backend/go-app/main.go

@@ -321,7 +321,7 @@ func checkError(cmdName string, cmdArgs []string) error {
 	scanner := bufio.NewScanner(cmdReader)
 	go func() {
 		for scanner.Scan() {
-			fmt.Printf("Out: %s\n", scanner.Text())
+			log.Printf("Out: %s\n", scanner.Text())
 		}
 	}()
 
@@ -1402,7 +1402,7 @@ func parseWorkflowParameters(resp http.ResponseWriter, request *http.Request) (m
 		return t, err
 	}
 
-	//fmt.Println(curjson.String())
+	//log.Println(curjson.String())
 	//log.Printf("Parsing json a second time: %s", string(curjson.String()))
 
 	err = json.Unmarshal(curjson.Bytes(), &t)
@@ -2556,7 +2556,7 @@ func execSubprocess(cmdName string, cmdArgs []string) error {
 	scanner := bufio.NewScanner(cmdReader)
 	go func() {
 		for scanner.Scan() {
-			fmt.Printf("Out: %s\n", scanner.Text())
+			log.Printf("Out: %s\n", scanner.Text())
 		}
 	}()
 
@@ -3528,7 +3528,7 @@ func handleAppHotload(ctx context.Context, location string, forceUpdate bool) er
 		return err
 	}
 
-	_, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate)
+	_, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate, false)
 	if err != nil {
 		log.Printf("[WARNING] Githubfolders error: %s", err)
 		return err
@@ -3981,7 +3981,7 @@ func runInitCloudSetup() {
 	if err != nil {
 		log.Printf("[INFO] Failed initial setup: %s", err)
 	} else {
-		log.Printf("[INFO] Ran initial setup!")
+		log.Printf("[INFO] Finished initial cloudsync setup!")
 	}
 }
 
@@ -4419,20 +4419,8 @@ func runInitEs(ctx context.Context) {
 	}
 
 	// Getting apps to see if we should initialize a test
-	// FIXME: Isn't this a little backwards?
 	workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000, 0)
 	log.Printf("[INFO] Getting and validating workflowapps. Got %d with err %#v", len(workflowapps), err)
-
-	// accept any certificate (might be useful for testing)
-	//customGitClient := &http.Client{
-	//	Transport: &http.Transport{
-	//		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-	//	},
-	//	Timeout: 15 * time.Second,
-	//}
-	//client.InstallProtocol("http", githttp.NewClient(customGitClient))
-	//client.InstallProtocol("https", githttp.NewClient(customGitClient))
-
 	if err != nil && len(workflowapps) == 0 {
 		log.Printf("[WARNING] Failed getting apps (runInit): %s", err)
 	} else if err == nil && len(workflowapps) < 10 {
@@ -4442,8 +4430,9 @@ func runInitEs(ctx context.Context) {
 
 		url := os.Getenv("SHUFFLE_APP_DOWNLOAD_LOCATION")
 		if len(url) == 0 {
-			log.Printf("[INFO] Skipping download of apps since no URL is set. Default would be https://github.com/shuffle/shuffle-apps")
+			log.Printf("[INFO] Skipping download of apps since no URL is set. Default would be https://github.com/shuffle/python-apps")
-			url = "https://github.com/shuffle/shuffle-apps"
+			
+			url = "https://github.com/shuffle/python-apps"
 			//url = ""
 			//return
 		}
@@ -4483,7 +4472,7 @@ func runInitEs(ctx context.Context) {
 		_ = r
 		//iterateAppGithubFolders(fs, dir, "", "testing")
 
-		_, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate)
+		_, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate, true)
 		if err != nil {
 			log.Printf("[WARNING] Error from app load in init: %s", err)
 		}
@@ -5266,6 +5255,7 @@ func initHandlers() {
 	// App specific. Partially Singul.
 	r.HandleFunc("/api/v1/apps/categories", shuffle.GetActiveCategories).Methods("GET", "OPTIONS")
 	r.HandleFunc("/api/v1/apps/categories/run", singul.RunCategoryAction).Methods("POST", "OPTIONS")
+
 	r.HandleFunc("/api/v1/apps/{key}/execute", executeSingleAction).Methods("POST", "OPTIONS")
 	r.HandleFunc("/api/v1/apps/{key}/run", executeSingleAction).Methods("POST", "OPTIONS")
 

backend/go-app/walkoff.go

@@ -2016,14 +2016,14 @@ func validateAppInput(resp http.ResponseWriter, request *http.Request) {
 
 	//fmt.Printf("File type: %s. MIME: %s\n", kind.Extension, kind.MIME.Value)
 	if kind == filetype.Unknown {
-		fmt.Println("Unknown file type")
+		log.Println("Unknown file type")
 		resp.WriteHeader(401)
 		resp.Write([]byte(`{"success": false}`))
 		return
 	}
 
 	if kind.MIME.Value != "application/zip" {
-		fmt.Println("Not zip, can't unzip")
+		log.Println("Not zip, can't unzip")
 		resp.WriteHeader(401)
 		resp.Write([]byte(`{"success": false}`))
 		return
@@ -3005,7 +3005,7 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) {
 }
 
 // Onlyname is used to
-func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate bool) ([]shuffle.BuildLaterStruct, []shuffle.BuildLaterStruct, error) {
+func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate, duringStartup bool) ([]shuffle.BuildLaterStruct, []shuffle.BuildLaterStruct, error) {
 	var err error
 
 	allapps := []shuffle.WorkflowApp{}
@@ -3017,7 +3017,13 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
 		"YARA",
 		"ATTACK-PREDICTOR",
 	}
-	//if strings.ToUpper(workflowapp.Name) == strings.ToUpper(appname) {
+
+	startupNames := []string{
+		"shuffle-tools",
+		"http",
+		"email",
+		"shuffle-ai",
+	}
 
 	// It's here to prevent getting them in every iteration
 	buildLaterFirst := []shuffle.BuildLaterStruct{}
@@ -3027,6 +3033,19 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
 			continue
 		}
 
+		//duringStartup 
+		if duringStartup {
+			// Look for names: shuffle tools, http, email, shuffle ai
+			if shuffle.ArrayContains(startupNames, strings.ToLower(file.Name())) {
+				// Allowed to build during startup
+
+				//log.Printf("\n\n\nFOUND MATCHING APP: %s\n\n\n", file.Name())
+			} else {
+				//log.Printf("\n\n\nWRONG APP (2): %s\n\n\n", file.Name())
+				continue
+			}
+		}
+
 		// Folder?
 		switch mode := file.Mode(); {
 		case mode.IsDir():
@@ -3043,7 +3062,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
 			}
 
 			// Go routine? Hmm, this can be super quick I guess
-			buildFirst, buildLast, err := IterateAppGithubFolders(ctx, fs, dir, tmpExtra, "", forceUpdate)
+			buildFirst, buildLast, err := IterateAppGithubFolders(ctx, fs, dir, tmpExtra, "", forceUpdate, false)
 
 			for _, item := range buildFirst {
 				buildLaterFirst = append(buildLaterFirst, item)
@@ -3055,7 +3074,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
 
 			if err != nil {
 				log.Printf("[WARNING] Error reading folder: %s", err)
-				//buildFirst, buildLast, err := IterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate)
+				//buildFirst, buildLast, err := IterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate, false)
 
 				if !forceUpdate {
 					continue
@@ -3355,6 +3374,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
 		"http",
 		"email",
 	}
+
 	for _, buildLater := range buildLaterFirst {
 		found := false
 		for _, appname := range initApps {
@@ -3373,12 +3393,19 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
 	}
 
 	// Prepend newSortedList to buildLaterFirst
+	handledImages := []string{}
 	buildLaterFirst = append(newSortedList, buildLaterFirst...)
-
 	if len(extra) == 0 {
 		log.Printf("[INFO] Starting build of %d containers (FIRST)", len(buildLaterFirst))
 		for _, item := range buildLaterFirst {
+
+			if len(item.Tags) > 0 && shuffle.ArrayContains(handledImages, item.Tags[0]) {
+				continue
+			} 
+
+			handledImages = append(handledImages, item.Tags[0])
 			err = buildImageMemory(fs, item.Tags, item.Extra, true)
+
 			if err != nil {
 				orgId := ""
 
@@ -3408,6 +3435,12 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
 		if len(buildLaterList) > 0 {
 			log.Printf("[INFO] Starting build of %d skipped docker images", len(buildLaterList))
 			for _, item := range buildLaterList {
+				if len(item.Tags) > 0 && shuffle.ArrayContains(handledImages, item.Tags[0]) {
+					continue
+				} 
+
+				handledImages = append(handledImages, item.Tags[0])
+
 				err = buildImageMemory(fs, item.Tags, item.Extra, true)
 				if err != nil {
 					log.Printf("[INFO] Failed image build memory: %s", err)
@@ -3541,7 +3574,7 @@ func LoadSpecificApps(resp http.ResponseWriter, request *http.Request) {
 			}
 		}
 
-		IterateAppGithubFolders(ctx, fs, dir, "", "", tmpBody.ForceUpdate)
+		IterateAppGithubFolders(ctx, fs, dir, "", "", tmpBody.ForceUpdate, false)
 
 	} else if strings.Contains(tmpBody.URL, "s3") {
 		//https://docs.aws.amazon.com/sdk-for-go/api/service/s3/

docker-compose.yml

@@ -65,7 +65,7 @@ services:
     hostname: shuffle-opensearch
     container_name: shuffle-opensearch
     environment:
-      - "OPENSEARCH_JAVA_OPTS=-Xms2048m -Xmx2048m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
+      - "OPENSEARCH_JAVA_OPTS=-Xms3072m -Xmx3072m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
       - bootstrap.memory_lock=true
       - DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI=true
       - cluster.initial_master_nodes=shuffle-opensearch

frontend/src/views/AngularWorkflow.jsx

@@ -12044,7 +12044,6 @@ const AngularWorkflow = (defaultprops) => {
         prioritizedApps,
         Array.prototype.concat.apply(
           filteredApps.filter((innerapp) => !internalIds.includes(innerapp.id.toLowerCase())),
-          triggers
         )
       )
     )
@@ -12820,9 +12819,9 @@ const AngularWorkflow = (defaultprops) => {
                   </InstantSearch>
                 </div>
                 :
-                <div style={{ marginLeft: 10, marginTop: 10, marginBottom: 100, }}>
+                <div style={{ marginLeft: 10, marginTop: 15, marginBottom: 100, }}>
                   <Typography variant="body1" color="textSecondary">
-                    Apps need to be activated before they can be used. Search from our 2500+ apps to activate them for your organisation.
+                    <b>Apps need to be activated</b> before they can be used. Search in the search bar from our 2500+ apps to activate them for.
                   </Typography>
                 </div>
               }

functions/onprem/orborus/orborus.go

@@ -497,7 +497,7 @@ func deployServiceWorkers(image string) {
 	}
 
 	// Running 2 by default instead of 1. Higher scale mechanisms - es
-	replicas := uint64(2)
+	replicas := uint64(1)
 	scaleReplicas := os.Getenv("SHUFFLE_SCALE_REPLICAS")
 	if len(scaleReplicas) > 0 {
 		tmpInt, err := strconv.Atoi(scaleReplicas)

functions/onprem/worker/worker.go

@@ -612,7 +612,7 @@ func deployk8sApp(image string, identifier string, env []string) error {
 	// use deployment instead of pod
 	// then expose a service similarly.
 	// number of replicas can be set to os.Getenv("SHUFFLE_SCALE_REPLICAS")
-	replicaNumber := 2
+	replicaNumber := 1
 	replicaNumberStr := os.Getenv("SHUFFLE_SCALE_REPLICAS")
 	if len(replicaNumberStr) > 0 {
 		tmpInt, err := strconv.Atoi(replicaNumberStr)
@@ -3222,9 +3222,9 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl
 
 	// Apps used a lot should have 2 replicas (default)
 	replicas := uint64(1)
-	if (strings.Contains(strings.ToLower(name), "shuffle") && strings.Contains(strings.ToLower(name), "tools")) || strings.Contains(strings.ToLower(name), "http") {
+	//if (strings.Contains(strings.ToLower(name), "shuffle") && strings.Contains(strings.ToLower(name), "tools")) || strings.Contains(strings.ToLower(name), "http") {
-		replicas = 2
+	//	replicas = 2
-	}
+	//}
 
 	// Sent from Orborus
 	// Should be equal to