diff --git a/.github/install-guide.md b/.github/install-guide.md index 165490b7..7c0d9af0 100644 --- a/.github/install-guide.md +++ b/.github/install-guide.md @@ -1,6 +1,8 @@ # Installation guide Installation of Shuffle is currently only available in docker. Looking for how to update Shuffle? Check the [updating guide](https://shuffler.io/docs/configuration#updating_shuffle) +This document outlines a an introduction environment which is not scalable. [Read here](https://shuffler.io/docs/configuration#production_readiness) for information on production readiness. + # Docker - *nix The Docker setup is done with docker-compose diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 8c587b6c..57b9a64c 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -514,6 +514,7 @@ class AppBase: # return print("[INFO] Multiplier length: %d" % len(param_multiplier)) + tmp = "" for subparams in param_multiplier: print(f"SUBPARAMS IN MULTI: {subparams}") try: @@ -2306,6 +2307,7 @@ class AppBase: #newres = await func(**params) #print("PARAMS: %s" % params) + newres = "" while True: try: newres = await func(**params) diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index 51a842a2..a4101b61 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -697,7 +697,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { // REBUILDS THE APP if len(img.ID) == 0 { if len(img2.ID) == 0 { - workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500) + workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 0) log.Printf("[INFO] Getting workflowapps for a rebuild. Got %d with err %#v", len(workflowapps), err) if err == nil { imageName := "" @@ -720,16 +720,22 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { } if len(imageName) > 0 && len(imageVersion) > 0 { - log.Printf("Looking for appname %s with version %s", imageName, imageVersion) foundApp := shuffle.WorkflowApp{} + imageName = strings.ToLower(imageName) + imageVersion = strings.ToLower(imageVersion) + log.Printf("[DEBUG] Looking for appname %s with version %s", imageName, imageVersion) + for _, app := range workflowapps { if strings.ToLower(strings.Replace(app.Name, " ", "_", -1)) == imageName && app.AppVersion == imageVersion { if app.Generated { + log.Printf("[DEBUG] Found matching app %s:%s - %s", imageName, imageVersion, app.ID) foundApp = app break + } else { + log.Printf("[WARNING] Trying to rebuild app that isn't generated - not allowed. Looking further.") } - break + //break } } @@ -767,7 +773,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { } //log.Printf("[INFO] Img found (%s): %#v", tagFound, img) - log.Printf("[INFO] Img found to be downloaded: %s", tagFound) + log.Printf("[INFO] Img found to be downloaded by client: %s", tagFound) newClient, err := newdockerclient.NewClientFromEnv() if err != nil { diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 47a4753f..790628e4 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -23,7 +23,7 @@ require ( github.com/elastic/go-elasticsearch v0.0.0 // indirect github.com/elastic/go-elasticsearch/v7 v7.12.0 // indirect github.com/frikky/kin-openapi v0.39.0 - github.com/frikky/shuffle-shared v0.0.63 + github.com/frikky/shuffle-shared v0.0.66 github.com/fsouza/go-dockerclient v1.7.2 // indirect github.com/ghodss/yaml v1.0.0 github.com/go-git/go-billy/v5 v5.0.0 diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index d04a31a1..d7bef904 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -179,6 +179,8 @@ github.com/frikky/shuffle-shared v0.0.62 h1:1M8y7rX8nQW7072+bUD4vgHcf65AG0kJ8m3i github.com/frikky/shuffle-shared v0.0.62/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc= github.com/frikky/shuffle-shared v0.0.63 h1:btn7V7s98eZmx/9qyapGE3V1TyxftvjMmrzkEcUvu+c= github.com/frikky/shuffle-shared v0.0.63/go.mod h1:oPDyGFBuurPtE6UQLUrHU/JtbOJftqDljRfEdLlCTI4= +github.com/frikky/shuffle-shared v0.0.65 h1:TvISX6WE1Y7M2UJ3YuMv0PtgRPWtMj5iGiArVmv6SI8= +github.com/frikky/shuffle-shared v0.0.65/go.mod h1:oPDyGFBuurPtE6UQLUrHU/JtbOJftqDljRfEdLlCTI4= github.com/fsouza/go-dockerclient v1.7.2 h1:bBEAcqLTkpq205jooP5RVroUKiVEWgGecHyeZc4OFjo= github.com/fsouza/go-dockerclient v1.7.2/go.mod h1:+ugtMCVRwnPfY7d8/baCzZ3uwB0BrG5DB8OzbtxaRz8= github.com/getkin/kin-openapi v0.8.0 h1:a6TQjTqwkyscC4/hShJX7WhCVE+4bi9lzw61XHQW5hE= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 75fab7ca..a5a44341 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3091,7 +3091,7 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) { var test Test err := json.Unmarshal(body, &test) if err != nil { - log.Printf("Failed unmarshalling test: %s", err) + log.Printf("[WARNING] Failed unmarshalling test: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -3107,7 +3107,7 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) { ctx := context.Background() app, err := shuffle.GetApp(ctx, test.Id, user) if err != nil { - log.Printf("Error getting app when editing: %s", app.Name) + log.Printf("[WARNING] Error getting app when editing: %s", app.Name) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -3325,15 +3325,19 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) { // FIXME: Might cause versioning issues if we re-use the same!! // FIXME: Need a way to track different versions of the same app properly. // Hint: Save API.id somewhere, and use newmd5 to save latest version - err = shuffle.SetOpenApiDatastore(ctx, newmd5, parsed) - if err != nil { - log.Printf("[ERROR] Failed saving app %s to database: %s", newmd5, err) - resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%"}`, err))) + + if len(user.Id) > 0 { + err = shuffle.SetOpenApiDatastore(ctx, newmd5, parsed) + if err != nil { + log.Printf("[ERROR] Failed saving app %s to database: %s", newmd5, err) + resp.WriteHeader(500) + resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%"}`, err))) + } + + shuffle.SetOpenApiDatastore(ctx, api.ID, parsed) } // Backup every single one - shuffle.SetOpenApiDatastore(ctx, api.ID, parsed) /* err = increaseStatisticsField(ctx, "total_apps_created", newmd5, 1, user.ActiveOrg.Id) @@ -3351,6 +3355,8 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) { shuffle.DeleteCache(ctx, cacheKey) cacheKey = fmt.Sprintf("workflowapps-sorted-500") shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-1000") + shuffle.DeleteCache(ctx, cacheKey) shuffle.DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id)) if len(user.Id) > 0 { @@ -3475,6 +3481,8 @@ func handleAppHotload(ctx context.Context, location string, forceUpdate bool) er shuffle.DeleteCache(ctx, cacheKey) cacheKey = fmt.Sprintf("workflowapps-sorted-500") shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-1000") + shuffle.DeleteCache(ctx, cacheKey) //shuffle.DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id)) return nil @@ -4073,7 +4081,7 @@ func runInitEs(ctx context.Context) { } // Getting apps to see if we should initialize a test - workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500) + workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000) log.Printf("[INFO] Getting and validating workflowapps. Got %d with err %#v", len(workflowapps), err) if err != nil && len(workflowapps) == 0 { log.Printf("[WARNING] Failed getting apps (runInit): %s", err) @@ -4713,7 +4721,7 @@ func runInit(ctx context.Context) { } // Getting apps to see if we should initialize a test - workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500) + workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000) log.Printf("[INFO] Getting and validating workflowapps. Got %d with err %s", len(workflowapps), err) if err != nil && len(workflowapps) == 0 { log.Printf("[WARNING] Failed getting apps (runInit): %s", err) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 1febda07..b302d6f0 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -2666,7 +2666,7 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { //return } - workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500) + workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000) if err != nil { log.Printf("Failed getting apps (getworkflowapps): %s", err) resp.WriteHeader(401) @@ -2787,7 +2787,7 @@ func getSpecificApps(resp http.ResponseWriter, request *http.Request) { // FIXME - continue the search here with github repos etc. // Caching might be smart :D ctx := context.Background() - workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500) + workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000) if err != nil { log.Printf("Error: Failed getting workflowapps: %s", err) resp.WriteHeader(401) @@ -3005,7 +3005,9 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - cacheKey := fmt.Sprintf("workflowapps-sorted-500") + cacheKey := fmt.Sprintf("workflowapps-sorted-1000") + shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-500") shuffle.DeleteCache(ctx, cacheKey) cacheKey = fmt.Sprintf("workflowapps-sorted-0") shuffle.DeleteCache(ctx, cacheKey) @@ -3046,6 +3048,8 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { shuffle.DeleteCache(ctx, cacheKey) cacheKey = fmt.Sprintf("workflowapps-sorted-500") shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-1000") + shuffle.DeleteCache(ctx, cacheKey) resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) @@ -3178,6 +3182,8 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) { shuffle.DeleteCache(ctx, cacheKey) cacheKey = fmt.Sprintf("workflowapps-sorted-500") shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-1000") + shuffle.DeleteCache(ctx, cacheKey) resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) @@ -3186,7 +3192,7 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) { func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error { ctx := context.Background() - workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500) + workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000) appCounter := 0 if err != nil { log.Printf("Failed to get existing generated apps") @@ -3317,6 +3323,8 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, shuffle.DeleteCache(ctx, cacheKey) cacheKey = fmt.Sprintf("workflowapps-sorted-500") shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-1000") + shuffle.DeleteCache(ctx, cacheKey) } } else { //log.Printf("Skipped upload of %s (%s)", api.Name, api.ID) @@ -3741,6 +3749,8 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin shuffle.DeleteCache(ctx, cacheKey) cacheKey = fmt.Sprintf("workflowapps-sorted-500") shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-1000") + shuffle.DeleteCache(ctx, cacheKey) //log.Printf("BUILDLATERFIRST: %d, BUILDLATERLIST: %d", len(buildLaterFirst), len(buildLaterList)) if len(extra) == 0 { @@ -3811,7 +3821,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - allapps, err := shuffle.GetAllWorkflowApps(ctx, 500) + allapps, err := shuffle.GetAllWorkflowApps(ctx, 1000) if err != nil { log.Printf("Failed getting apps to verify: %s", err) resp.WriteHeader(401) @@ -3914,6 +3924,8 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { shuffle.DeleteCache(ctx, cacheKey) cacheKey = fmt.Sprintf("workflowapps-sorted-500") shuffle.DeleteCache(ctx, cacheKey) + cacheKey = fmt.Sprintf("workflowapps-sorted-1000") + shuffle.DeleteCache(ctx, cacheKey) resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 829b97da..9d30eab9 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -2742,6 +2742,15 @@ const AngularWorkflow = (props) => { for (var key in filteredelements) { } + + var copyText = document.getElementById("copy_element_shuffle") + if (copyText !== undefined && copyText !== null) { + const clipboard = navigator.clipboard + if (clipboard === undefined) { + alert.error("Can only copy over HTTPS (port 3443)") + return + } + } console.log("FILTERED: ", filteredelements) } break; diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index d5d2b828..b4b98103 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -665,7 +665,7 @@ const Apps = (props) => { - {selectedApp.versions !== null && selectedApp.versions !== undefined && selectedApp.versions.length > 0 ? + {selectedApp.versions !== null && selectedApp.versions !== undefined && selectedApp.versions.length > 1 ?