marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Opensearch and app optimisations

Browse Source
This commit is contained in:
Frikky
2025-08-27 12:54:50 +02:00
parent 19c07f0f07
commit 4fe311cc99
6 changed files with 59 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/go-app/main.go
+10 -20
View File
@@ -321,7 +321,7 @@ func checkError(cmdName string, cmdArgs []string) error {
scanner := bufio.NewScanner(cmdReader)
go func() {
for scanner.Scan() {
fmt.Printf("Out: %s\n", scanner.Text())
log.Printf("Out: %s\n", scanner.Text())
}
}()
@@ -1402,7 +1402,7 @@ func parseWorkflowParameters(resp http.ResponseWriter, request *http.Request) (m
return t, err
}
//fmt.Println(curjson.String())
//log.Println(curjson.String())
//log.Printf("Parsing json a second time: %s", string(curjson.String()))
err = json.Unmarshal(curjson.Bytes(), &t)
@@ -2556,7 +2556,7 @@ func execSubprocess(cmdName string, cmdArgs []string) error {
scanner := bufio.NewScanner(cmdReader)
go func() {
for scanner.Scan() {
fmt.Printf("Out: %s\n", scanner.Text())
log.Printf("Out: %s\n", scanner.Text())
}
}()
@@ -3528,7 +3528,7 @@ func handleAppHotload(ctx context.Context, location string, forceUpdate bool) er
return err
}
_, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate)
_, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate, false)
if err != nil {
log.Printf("[WARNING] Githubfolders error: %s", err)
return err
@@ -3981,7 +3981,7 @@ func runInitCloudSetup() {
if err != nil {
log.Printf("[INFO] Failed initial setup: %s", err)
} else {
log.Printf("[INFO] Ran initial setup!")
log.Printf("[INFO] Finished initial cloudsync setup!")
}
}
@@ -4419,20 +4419,8 @@ func runInitEs(ctx context.Context) {
}
// Getting apps to see if we should initialize a test
// FIXME: Isn't this a little backwards?
workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000, 0)
log.Printf("[INFO] Getting and validating workflowapps. Got %d with err %#v", len(workflowapps), err)
// accept any certificate (might be useful for testing)
//customGitClient := &http.Client{
// Transport: &http.Transport{
// TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
// },
// Timeout: 15 * time.Second,
//}
//client.InstallProtocol("http", githttp.NewClient(customGitClient))
//client.InstallProtocol("https", githttp.NewClient(customGitClient))
if err != nil && len(workflowapps) == 0 {
log.Printf("[WARNING] Failed getting apps (runInit): %s", err)
} else if err == nil && len(workflowapps) < 10 {
@@ -4442,8 +4430,9 @@ func runInitEs(ctx context.Context) {
url := os.Getenv("SHUFFLE_APP_DOWNLOAD_LOCATION")
if len(url) == 0 {
log.Printf("[INFO] Skipping download of apps since no URL is set. Default would be https://github.com/shuffle/shuffle-apps")
url = "https://github.com/shuffle/shuffle-apps"
log.Printf("[INFO] Skipping download of apps since no URL is set. Default would be https://github.com/shuffle/python-apps")
url = "https://github.com/shuffle/python-apps"
//url = ""
//return
}
@@ -4483,7 +4472,7 @@ func runInitEs(ctx context.Context) {
_ = r
//iterateAppGithubFolders(fs, dir, "", "testing")
_, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate)
_, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate, true)
if err != nil {
log.Printf("[WARNING] Error from app load in init: %s", err)
}
@@ -5266,6 +5255,7 @@ func initHandlers() {
// App specific. Partially Singul.
r.HandleFunc("/api/v1/apps/categories", shuffle.GetActiveCategories).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/apps/categories/run", singul.RunCategoryAction).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/{key}/execute", executeSingleAction).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/{key}/run", executeSingleAction).Methods("POST", "OPTIONS")
backend/go-app/walkoff.go
+41 -8
View File
@@ -2016,14 +2016,14 @@ func validateAppInput(resp http.ResponseWriter, request *http.Request) {
//fmt.Printf("File type: %s. MIME: %s\n", kind.Extension, kind.MIME.Value)
if kind == filetype.Unknown {
fmt.Println("Unknown file type")
log.Println("Unknown file type")
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if kind.MIME.Value != "application/zip" {
fmt.Println("Not zip, can't unzip")
log.Println("Not zip, can't unzip")
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
@@ -3005,7 +3005,7 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) {
}
// Onlyname is used to
func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate bool) ([]shuffle.BuildLaterStruct, []shuffle.BuildLaterStruct, error) {
func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate, duringStartup bool) ([]shuffle.BuildLaterStruct, []shuffle.BuildLaterStruct, error) {
var err error
allapps := []shuffle.WorkflowApp{}
@@ -3017,7 +3017,13 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
"YARA",
"ATTACK-PREDICTOR",
}
//if strings.ToUpper(workflowapp.Name) == strings.ToUpper(appname) {
startupNames := []string{
"shuffle-tools",
"http",
"email",
"shuffle-ai",
}
// It's here to prevent getting them in every iteration
buildLaterFirst := []shuffle.BuildLaterStruct{}
@@ -3027,6 +3033,19 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
continue
}
//duringStartup
if duringStartup {
// Look for names: shuffle tools, http, email, shuffle ai
if shuffle.ArrayContains(startupNames, strings.ToLower(file.Name())) {
// Allowed to build during startup
//log.Printf("\n\n\nFOUND MATCHING APP: %s\n\n\n", file.Name())
} else {
//log.Printf("\n\n\nWRONG APP (2): %s\n\n\n", file.Name())
continue
}
}
// Folder?
switch mode := file.Mode(); {
case mode.IsDir():
@@ -3043,7 +3062,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
}
// Go routine? Hmm, this can be super quick I guess
buildFirst, buildLast, err := IterateAppGithubFolders(ctx, fs, dir, tmpExtra, "", forceUpdate)
buildFirst, buildLast, err := IterateAppGithubFolders(ctx, fs, dir, tmpExtra, "", forceUpdate, false)
for _, item := range buildFirst {
buildLaterFirst = append(buildLaterFirst, item)
@@ -3055,7 +3074,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
if err != nil {
log.Printf("[WARNING] Error reading folder: %s", err)
//buildFirst, buildLast, err := IterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate)
//buildFirst, buildLast, err := IterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate, false)
if !forceUpdate {
continue
@@ -3355,6 +3374,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
"http",
"email",
}
for _, buildLater := range buildLaterFirst {
found := false
for _, appname := range initApps {
@@ -3373,12 +3393,19 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
}
// Prepend newSortedList to buildLaterFirst
handledImages := []string{}
buildLaterFirst = append(newSortedList, buildLaterFirst...)
if len(extra) == 0 {
log.Printf("[INFO] Starting build of %d containers (FIRST)", len(buildLaterFirst))
for _, item := range buildLaterFirst {
if len(item.Tags) > 0 && shuffle.ArrayContains(handledImages, item.Tags[0]) {
continue
}
handledImages = append(handledImages, item.Tags[0])
err = buildImageMemory(fs, item.Tags, item.Extra, true)
if err != nil {
orgId := ""
@@ -3408,6 +3435,12 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
if len(buildLaterList) > 0 {
log.Printf("[INFO] Starting build of %d skipped docker images", len(buildLaterList))
for _, item := range buildLaterList {
if len(item.Tags) > 0 && shuffle.ArrayContains(handledImages, item.Tags[0]) {
continue
}
handledImages = append(handledImages, item.Tags[0])
err = buildImageMemory(fs, item.Tags, item.Extra, true)
if err != nil {
log.Printf("[INFO] Failed image build memory: %s", err)
@@ -3541,7 +3574,7 @@ func LoadSpecificApps(resp http.ResponseWriter, request *http.Request) {
}
}
IterateAppGithubFolders(ctx, fs, dir, "", "", tmpBody.ForceUpdate)
IterateAppGithubFolders(ctx, fs, dir, "", "", tmpBody.ForceUpdate, false)
} else if strings.Contains(tmpBody.URL, "s3") {
//https://docs.aws.amazon.com/sdk-for-go/api/service/s3/
docker-compose.yml
+1 -1
View File
@@ -65,7 +65,7 @@ services:
hostname: shuffle-opensearch
container_name: shuffle-opensearch
environment:
- "OPENSEARCH_JAVA_OPTS=-Xms2048m -Xmx2048m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
- "OPENSEARCH_JAVA_OPTS=-Xms3072m -Xmx3072m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
- bootstrap.memory_lock=true
- DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI=true
- cluster.initial_master_nodes=shuffle-opensearch
frontend/src/views/AngularWorkflow.jsx
+2 -3
View File
@@ -12044,7 +12044,6 @@ const AngularWorkflow = (defaultprops) => {
prioritizedApps,
Array.prototype.concat.apply(
filteredApps.filter((innerapp) => !internalIds.includes(innerapp.id.toLowerCase())),
triggers
)
)
)
@@ -12820,9 +12819,9 @@ const AngularWorkflow = (defaultprops) => {
</InstantSearch>
</div>
:
<div style={{ marginLeft: 10, marginTop: 10, marginBottom: 100, }}>
<div style={{ marginLeft: 10, marginTop: 15, marginBottom: 100, }}>
<Typography variant="body1" color="textSecondary">
Apps need to be activated before they can be used. Search from our 2500+ apps to activate them for your organisation.
<b>Apps need to be activated</b> before they can be used. Search in the search bar from our 2500+ apps to activate them for.
</Typography>
</div>
}
functions/onprem/orborus/orborus.go
+1 -1
View File
@@ -497,7 +497,7 @@ func deployServiceWorkers(image string) {
}
// Running 2 by default instead of 1. Higher scale mechanisms - es
replicas := uint64(2)
replicas := uint64(1)
scaleReplicas := os.Getenv("SHUFFLE_SCALE_REPLICAS")
if len(scaleReplicas) > 0 {
tmpInt, err := strconv.Atoi(scaleReplicas)
functions/onprem/worker/worker.go
+4 -4
View File
@@ -612,7 +612,7 @@ func deployk8sApp(image string, identifier string, env []string) error {
// use deployment instead of pod
// then expose a service similarly.
// number of replicas can be set to os.Getenv("SHUFFLE_SCALE_REPLICAS")
replicaNumber := 2
replicaNumber := 1
replicaNumberStr := os.Getenv("SHUFFLE_SCALE_REPLICAS")
if len(replicaNumberStr) > 0 {
tmpInt, err := strconv.Atoi(replicaNumberStr)
@@ -3222,9 +3222,9 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl
// Apps used a lot should have 2 replicas (default)
replicas := uint64(1)
if (strings.Contains(strings.ToLower(name), "shuffle") && strings.Contains(strings.ToLower(name), "tools")) || strings.Contains(strings.ToLower(name), "http") {
replicas = 2
}
//if (strings.Contains(strings.ToLower(name), "shuffle") && strings.Contains(strings.ToLower(name), "tools")) || strings.Contains(strings.ToLower(name), "http") {
// replicas = 2
//}
// Sent from Orborus
// Should be equal to