marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed some app loading prioritization and position issues

Browse Source
This commit is contained in:
frikky
2023-06-04 16:48:11 +02:00
parent c3a10956de
commit b0c82db41b
3 changed files with 51 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/go-app/go.mod
+1 -1
View File
@@ -2,7 +2,7 @@ module main
go 1.19
//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared
replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared
require (
cloud.google.com/go/datastore v1.10.0
backend/go-app/main.go
+10 -6
View File
@@ -3905,6 +3905,11 @@ func runInitEs(ctx context.Context) {
}
}
if strings.Contains(os.Getenv("SHUFFLE_OPENSEARCH_URL"), "https") {
log.Printf("[INFO] Waiting 10 seconds during init to make sure the opensearch instance is up and running with security features properly")
time.Sleep(10 * time.Second)
}
_ = setUsers
schedules, err := shuffle.GetAllSchedules(ctx, "ALL")
if err != nil {
@@ -4188,7 +4193,8 @@ func runInitEs(ctx context.Context) {
url := os.Getenv("SHUFFLE_APP_DOWNLOAD_LOCATION")
if len(url) == 0 {
log.Printf("[INFO] Skipping download of apps since no URL is set. Default would be https://github.com/frikky/shuffle-apps")
log.Printf("[INFO] Skipping download of apps since no URL is set. Default would be https://github.com/shuffle/shuffle-apps")
url = "https://github.com/shuffle/shuffle-apps"
//url = ""
//return
}
@@ -4215,7 +4221,6 @@ func runInitEs(ctx context.Context) {
log.Printf("[DEBUG] Getting apps from url '%s'", url)
r, err := git.Clone(storer, fs, cloneOptions)
if err != nil {
log.Printf("[WARNING] Failed loading repo into memory (init): %s", err)
}
@@ -4227,7 +4232,6 @@ func runInitEs(ctx context.Context) {
_ = r
//iterateAppGithubFolders(fs, dir, "", "testing")
// FIXME: Get all the apps?
_, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate)
if err != nil {
log.Printf("[WARNING] Error from app load in init: %s", err)
@@ -4242,7 +4246,7 @@ func runInitEs(ctx context.Context) {
}
log.Printf("[INFO] Downloading OpenAPI data for search - EXTRA APPS")
apis := "https://github.com/frikky/security-openapis"
apis := "https://github.com/shuffle/security-openapis"
// THis gets memory problems hahah
//apis := "https://github.com/APIs-guru/openapi-directory"
@@ -4850,7 +4854,7 @@ func runInit(ctx context.Context) {
url := os.Getenv("SHUFFLE_APP_DOWNLOAD_LOCATION")
if len(url) == 0 {
url = "https://github.com/frikky/shuffle-apps"
url = "https://github.com/shuffle/shuffle-apps"
}
username := os.Getenv("SHUFFLE_DOWNLOAD_AUTH_USERNAME")
@@ -4901,7 +4905,7 @@ func runInit(ctx context.Context) {
}
log.Printf("[INFO] Downloading OpenAPI data for search - EXTRA APPS")
apis := "https://github.com/frikky/security-openapis"
apis := "https://github.com/shuffle/security-openapis"
// FIXME: This part gets memory problems. Fix in the future to load these apps too.
//apis := "https://github.com/APIs-guru/openapi-directory"
backend/go-app/walkoff.go
+40 -7
View File
@@ -814,7 +814,7 @@ func JSONCheck(str string) bool {
func handleExecutionStatistics(execution shuffle.WorkflowExecution) {
// FIXME: CLEAN UP THE JSON THAT'S SAVED.
// https://github.com/frikky/Shuffle/issues/172
// https://github.com/shuffle/Shuffle/issues/172
appResults := []shuffle.AppExecutionExample{}
for _, result := range execution.Results {
resultCheck := JSONCheck(result.Result)
@@ -2370,7 +2370,7 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string,
if !found {
err = shuffle.SetWorkflowAppDatastore(ctx, api, api.ID)
if err != nil {
log.Printf("[WARNING] Failed setting workflowapp in loop: %s", err)
log.Printf("[WARNING] Failed setting workflowapp %s (%s) in loop: %s", api.Name, api.ID, err)
continue
} else {
appCounter += 1
@@ -2599,7 +2599,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) {
// Might require reflection into the python code to append the fields as well
for index, action := range workflowapp.Actions {
if action.AuthNotRequired {
log.Printf("Skipping auth setup: %s", action.Name)
log.Printf("[WARNING] Skipping auth setup for: %s", action.Name)
continue
}
@@ -2647,7 +2647,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) {
err = shuffle.SetWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID)
if err != nil {
log.Printf("Failed setting workflowapp: %s", err)
log.Printf("[WARNING] Failed setting workflowapp: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
@@ -2876,6 +2876,11 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
// Folder?
switch mode := file.Mode(); {
case mode.IsDir():
// Specific folder for skipping
if file.Name() == "unsupported" {
continue
}
tmpExtra := fmt.Sprintf("%s%s/", extra, file.Name())
dir, err := fs.ReadDir(tmpExtra)
if err != nil {
@@ -2899,7 +2904,8 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
//buildFirst, buildLast, err := IterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate)
if !forceUpdate {
return buildLaterFirst, buildLaterList, err
continue
//return buildLaterFirst, buildLaterList, err
}
}
@@ -2973,7 +2979,8 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
err = gyaml.Unmarshal(appfileData, &workflowapp)
if err != nil {
log.Printf("[WARNING] Failed building workflowapp %s: %s", extra, err)
return buildLaterFirst, buildLaterList, errors.New(fmt.Sprintf("Failed building %s: %s", extra, err))
continue
//return buildLaterFirst, buildLaterList, errors.New(fmt.Sprintf("Failed building %s: %s", extra, err))
//continue
}
@@ -3021,7 +3028,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
}
}
workflowapp.ReferenceInfo.GithubUrl = fmt.Sprintf("https://github.com/frikky/shuffle-apps/tree/master/%s/%s", strings.ToLower(newName), workflowapp.AppVersion)
workflowapp.ReferenceInfo.GithubUrl = fmt.Sprintf("https://github.com/shuffle/shuffle-apps/tree/master/%s/%s", strings.ToLower(newName), workflowapp.AppVersion)
tags := []string{
fmt.Sprintf("%s:%s_%s", baseDockerName, strings.ToLower(newName), workflowapp.AppVersion),
@@ -3188,6 +3195,32 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
cacheKey = fmt.Sprintf("workflowapps-sorted-1000")
shuffle.DeleteCache(ctx, cacheKey)
newSortedList := []shuffle.BuildLaterStruct{}
initApps := []string{
"tools",
"http",
"email",
}
for _, buildLater := range buildLaterFirst {
found := false
for _, appname := range initApps {
for _, tag := range buildLater.Tags {
if strings.Contains(strings.ToLower(tag), appname) {
newSortedList = append(newSortedList, buildLater)
found = true
break
}
}
if found {
break
}
}
}
// Prepend newSortedList to buildLaterFirst
buildLaterFirst = append(newSortedList, buildLaterFirst...)
if len(extra) == 0 {
log.Printf("[INFO] Starting build of %d containers (FIRST)", len(buildLaterFirst))
for _, item := range buildLaterFirst {