Fixed issues with semantic versioning for apps

.github/install-guide.md

@@ -1,6 +1,8 @@
 # Installation guide
 Installation of Shuffle is currently only available in docker. Looking for how to update Shuffle? Check the [updating guide](https://shuffler.io/docs/configuration#updating_shuffle)
 
+This document outlines a an introduction environment which is not scalable. [Read here](https://shuffler.io/docs/configuration#production_readiness) for information on production readiness. 
+
 # Docker - *nix
 The Docker setup is done with docker-compose 
 

backend/app_sdk/app_base.py

@@ -514,6 +514,7 @@ class AppBase:
             #    return
 
             print("[INFO] Multiplier length: %d" % len(param_multiplier))
+            tmp = ""
             for subparams in param_multiplier:
                 print(f"SUBPARAMS IN MULTI: {subparams}")
                 try:
@@ -2306,6 +2307,7 @@ class AppBase:
 
                             #newres = await func(**params)
                             #print("PARAMS: %s" % params)
+                            newres = ""
                             while True:
                                 try:
                                     newres = await func(**params)

backend/go-app/docker.go

@@ -697,7 +697,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) {
 	// REBUILDS THE APP
 	if len(img.ID) == 0 {
 		if len(img2.ID) == 0 {
-			workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500)
+			workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 0)
 			log.Printf("[INFO] Getting workflowapps for a rebuild. Got %d with err %#v", len(workflowapps), err)
 			if err == nil {
 				imageName := ""
@@ -720,16 +720,22 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) {
 				}
 
 				if len(imageName) > 0 && len(imageVersion) > 0 {
-					log.Printf("Looking for appname %s with version %s", imageName, imageVersion)
 					foundApp := shuffle.WorkflowApp{}
+					imageName = strings.ToLower(imageName)
+					imageVersion = strings.ToLower(imageVersion)
+					log.Printf("[DEBUG] Looking for appname %s with version %s", imageName, imageVersion)
+
 					for _, app := range workflowapps {
 						if strings.ToLower(strings.Replace(app.Name, " ", "_", -1)) == imageName && app.AppVersion == imageVersion {
 							if app.Generated {
+								log.Printf("[DEBUG] Found matching app %s:%s - %s", imageName, imageVersion, app.ID)
 								foundApp = app
 								break
+							} else {
+								log.Printf("[WARNING] Trying to rebuild app that isn't generated - not allowed. Looking further.")
 							}
 
-							break
+							//break
 						}
 					}
 
@@ -767,7 +773,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) {
 	}
 
 	//log.Printf("[INFO] Img found (%s): %#v", tagFound, img)
-	log.Printf("[INFO] Img found to be downloaded: %s", tagFound)
+	log.Printf("[INFO] Img found to be downloaded by client: %s", tagFound)
 
 	newClient, err := newdockerclient.NewClientFromEnv()
 	if err != nil {

backend/go-app/go.mod

@@ -23,7 +23,7 @@ require (
 	github.com/elastic/go-elasticsearch v0.0.0 // indirect
 	github.com/elastic/go-elasticsearch/v7 v7.12.0 // indirect
 	github.com/frikky/kin-openapi v0.39.0
-	github.com/frikky/shuffle-shared v0.0.63
+	github.com/frikky/shuffle-shared v0.0.66
 	github.com/fsouza/go-dockerclient v1.7.2 // indirect
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-git/go-billy/v5 v5.0.0

backend/go-app/go.sum

@@ -179,6 +179,8 @@ github.com/frikky/shuffle-shared v0.0.62 h1:1M8y7rX8nQW7072+bUD4vgHcf65AG0kJ8m3i
 github.com/frikky/shuffle-shared v0.0.62/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc=
 github.com/frikky/shuffle-shared v0.0.63 h1:btn7V7s98eZmx/9qyapGE3V1TyxftvjMmrzkEcUvu+c=
 github.com/frikky/shuffle-shared v0.0.63/go.mod h1:oPDyGFBuurPtE6UQLUrHU/JtbOJftqDljRfEdLlCTI4=
+github.com/frikky/shuffle-shared v0.0.65 h1:TvISX6WE1Y7M2UJ3YuMv0PtgRPWtMj5iGiArVmv6SI8=
+github.com/frikky/shuffle-shared v0.0.65/go.mod h1:oPDyGFBuurPtE6UQLUrHU/JtbOJftqDljRfEdLlCTI4=
 github.com/fsouza/go-dockerclient v1.7.2 h1:bBEAcqLTkpq205jooP5RVroUKiVEWgGecHyeZc4OFjo=
 github.com/fsouza/go-dockerclient v1.7.2/go.mod h1:+ugtMCVRwnPfY7d8/baCzZ3uwB0BrG5DB8OzbtxaRz8=
 github.com/getkin/kin-openapi v0.8.0 h1:a6TQjTqwkyscC4/hShJX7WhCVE+4bi9lzw61XHQW5hE=

backend/go-app/main.go

@@ -3091,7 +3091,7 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) {
 	var test Test
 	err := json.Unmarshal(body, &test)
 	if err != nil {
-		log.Printf("Failed unmarshalling test: %s", err)
+		log.Printf("[WARNING] Failed unmarshalling test: %s", err)
 		resp.WriteHeader(401)
 		resp.Write([]byte(`{"success": false}`))
 		return
@@ -3107,7 +3107,7 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) {
 		ctx := context.Background()
 		app, err := shuffle.GetApp(ctx, test.Id, user)
 		if err != nil {
-			log.Printf("Error getting app when editing: %s", app.Name)
+			log.Printf("[WARNING] Error getting app when editing: %s", app.Name)
 			resp.WriteHeader(401)
 			resp.Write([]byte(`{"success": false}`))
 			return
@@ -3325,15 +3325,19 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) {
 	// FIXME: Might cause versioning issues if we re-use the same!!
 	// FIXME: Need a way to track different versions of the same app properly.
 	// Hint: Save API.id somewhere, and use newmd5 to save latest version
-	err = shuffle.SetOpenApiDatastore(ctx, newmd5, parsed)
+
-	if err != nil {
+	if len(user.Id) > 0 {
-		log.Printf("[ERROR] Failed saving app %s to database: %s", newmd5, err)
+		err = shuffle.SetOpenApiDatastore(ctx, newmd5, parsed)
-		resp.WriteHeader(500)
+		if err != nil {
-		resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%"}`, err)))
+			log.Printf("[ERROR] Failed saving app %s to database: %s", newmd5, err)
+			resp.WriteHeader(500)
+			resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%"}`, err)))
+		}
+
+		shuffle.SetOpenApiDatastore(ctx, api.ID, parsed)
 	}
 
 	// Backup every single one
-	shuffle.SetOpenApiDatastore(ctx, api.ID, parsed)
 
 	/*
 		err = increaseStatisticsField(ctx, "total_apps_created", newmd5, 1, user.ActiveOrg.Id)
@@ -3351,6 +3355,8 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) {
 	shuffle.DeleteCache(ctx, cacheKey)
 	cacheKey = fmt.Sprintf("workflowapps-sorted-500")
 	shuffle.DeleteCache(ctx, cacheKey)
+	cacheKey = fmt.Sprintf("workflowapps-sorted-1000")
+	shuffle.DeleteCache(ctx, cacheKey)
 	shuffle.DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id))
 
 	if len(user.Id) > 0 {
@@ -3475,6 +3481,8 @@ func handleAppHotload(ctx context.Context, location string, forceUpdate bool) er
 	shuffle.DeleteCache(ctx, cacheKey)
 	cacheKey = fmt.Sprintf("workflowapps-sorted-500")
 	shuffle.DeleteCache(ctx, cacheKey)
+	cacheKey = fmt.Sprintf("workflowapps-sorted-1000")
+	shuffle.DeleteCache(ctx, cacheKey)
 	//shuffle.DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id))
 
 	return nil
@@ -4073,7 +4081,7 @@ func runInitEs(ctx context.Context) {
 	}
 
 	// Getting apps to see if we should initialize a test
-	workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500)
+	workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000)
 	log.Printf("[INFO] Getting and validating workflowapps. Got %d with err %#v", len(workflowapps), err)
 	if err != nil && len(workflowapps) == 0 {
 		log.Printf("[WARNING] Failed getting apps (runInit): %s", err)
@@ -4713,7 +4721,7 @@ func runInit(ctx context.Context) {
 	}
 
 	// Getting apps to see if we should initialize a test
-	workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500)
+	workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000)
 	log.Printf("[INFO] Getting and validating workflowapps. Got %d with err %s", len(workflowapps), err)
 	if err != nil && len(workflowapps) == 0 {
 		log.Printf("[WARNING] Failed getting apps (runInit): %s", err)

backend/go-app/walkoff.go

@@ -2666,7 +2666,7 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) {
 		//return
 	}
 
-	workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500)
+	workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000)
 	if err != nil {
 		log.Printf("Failed getting apps (getworkflowapps): %s", err)
 		resp.WriteHeader(401)
@@ -2787,7 +2787,7 @@ func getSpecificApps(resp http.ResponseWriter, request *http.Request) {
 	// FIXME - continue the search here with github repos etc.
 	// Caching might be smart :D
 	ctx := context.Background()
-	workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500)
+	workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000)
 	if err != nil {
 		log.Printf("Error: Failed getting workflowapps: %s", err)
 		resp.WriteHeader(401)
@@ -3005,7 +3005,9 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) {
 	}
 
 	ctx := context.Background()
-	cacheKey := fmt.Sprintf("workflowapps-sorted-500")
+	cacheKey := fmt.Sprintf("workflowapps-sorted-1000")
+	shuffle.DeleteCache(ctx, cacheKey)
+	cacheKey = fmt.Sprintf("workflowapps-sorted-500")
 	shuffle.DeleteCache(ctx, cacheKey)
 	cacheKey = fmt.Sprintf("workflowapps-sorted-0")
 	shuffle.DeleteCache(ctx, cacheKey)
@@ -3046,6 +3048,8 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) {
 	shuffle.DeleteCache(ctx, cacheKey)
 	cacheKey = fmt.Sprintf("workflowapps-sorted-500")
 	shuffle.DeleteCache(ctx, cacheKey)
+	cacheKey = fmt.Sprintf("workflowapps-sorted-1000")
+	shuffle.DeleteCache(ctx, cacheKey)
 
 	resp.WriteHeader(200)
 	resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
@@ -3178,6 +3182,8 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) {
 	shuffle.DeleteCache(ctx, cacheKey)
 	cacheKey = fmt.Sprintf("workflowapps-sorted-500")
 	shuffle.DeleteCache(ctx, cacheKey)
+	cacheKey = fmt.Sprintf("workflowapps-sorted-1000")
+	shuffle.DeleteCache(ctx, cacheKey)
 
 	resp.WriteHeader(200)
 	resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
@@ -3186,7 +3192,7 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) {
 func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error {
 
 	ctx := context.Background()
-	workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500)
+	workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000)
 	appCounter := 0
 	if err != nil {
 		log.Printf("Failed to get existing generated apps")
@@ -3317,6 +3323,8 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string,
 						shuffle.DeleteCache(ctx, cacheKey)
 						cacheKey = fmt.Sprintf("workflowapps-sorted-500")
 						shuffle.DeleteCache(ctx, cacheKey)
+						cacheKey = fmt.Sprintf("workflowapps-sorted-1000")
+						shuffle.DeleteCache(ctx, cacheKey)
 					}
 				} else {
 					//log.Printf("Skipped upload of %s (%s)", api.Name, api.ID)
@@ -3741,6 +3749,8 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
 	shuffle.DeleteCache(ctx, cacheKey)
 	cacheKey = fmt.Sprintf("workflowapps-sorted-500")
 	shuffle.DeleteCache(ctx, cacheKey)
+	cacheKey = fmt.Sprintf("workflowapps-sorted-1000")
+	shuffle.DeleteCache(ctx, cacheKey)
 
 	//log.Printf("BUILDLATERFIRST: %d, BUILDLATERLIST: %d", len(buildLaterFirst), len(buildLaterList))
 	if len(extra) == 0 {
@@ -3811,7 +3821,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) {
 	}
 
 	ctx := context.Background()
-	allapps, err := shuffle.GetAllWorkflowApps(ctx, 500)
+	allapps, err := shuffle.GetAllWorkflowApps(ctx, 1000)
 	if err != nil {
 		log.Printf("Failed getting apps to verify: %s", err)
 		resp.WriteHeader(401)
@@ -3914,6 +3924,8 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) {
 	shuffle.DeleteCache(ctx, cacheKey)
 	cacheKey = fmt.Sprintf("workflowapps-sorted-500")
 	shuffle.DeleteCache(ctx, cacheKey)
+	cacheKey = fmt.Sprintf("workflowapps-sorted-1000")
+	shuffle.DeleteCache(ctx, cacheKey)
 
 	resp.WriteHeader(200)
 	resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))

frontend/src/views/AngularWorkflow.jsx

@@ -2742,6 +2742,15 @@ const AngularWorkflow = (props) => {
 
 					for (var key in filteredelements) {
 					}
+
+					var copyText = document.getElementById("copy_element_shuffle")
+					if (copyText !== undefined && copyText !== null) {
+						const clipboard = navigator.clipboard
+						if (clipboard === undefined) {
+							alert.error("Can only copy over HTTPS (port 3443)")
+							return
+						}
+					}
 					console.log("FILTERED: ", filteredelements)
 				}
 	      break;

frontend/src/views/Apps.jsx

@@ -665,7 +665,7 @@ const Apps = (props) => {
 						</Typography>	
 					</div>
 				</div>
-				{selectedApp.versions !== null && selectedApp.versions !== undefined && selectedApp.versions.length > 0 ? 
+				{selectedApp.versions !== null && selectedApp.versions !== undefined && selectedApp.versions.length > 1 ? 
 					<Select
 						defaultValue={selectedApp.app_version}
 						onChange={(event) => {

functions/onprem/worker/worker.go

@@ -310,7 +310,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env []
 				ShowStdout: true,
 			}
 
-			exit := true
+			exit := false
 			out, err := cli.ContainerLogs(ctx, cont.ID, logOptions)
 			if err != nil {
 				log.Printf("[INFO] Failed getting logs: %s", err)
@@ -318,11 +318,13 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env []
 				buf := new(strings.Builder)
 				io.Copy(buf, out)
 				logs := buf.String()
+
+				// FIXME: Re-add log tracking which can be sent to backend
 				//allLogs[actionId] = logs
 
-				if stats.ContainerJSONBase.State.Status == "exited" && strings.Contains(logs, "Normal execution.") {
+				if stats.ContainerJSONBase.State.Status == "exited" && !strings.Contains(logs, "Normal execution.") {
 					log.Printf("[WARNING] BAD Execution Logs for %s: %s", actionId, logs)
-					exit = false
+					exit = true
 				}
 			}
 
@@ -994,20 +996,33 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) {
 					shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true)
 				}
 
-				image = images[2]
+				err := downloadDockerImageBackend(topClient, image)
-				err = deployApp(dockercli, image, identifier, env, workflowExecution, action.ID)
+				executed := false
-				if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") {
+				if err == nil {
-					if strings.Contains(err.Error(), "exited prematurely") {
+					log.Printf("[DEBUG] Downloaded image %s from backend (CLEANUP)", image)
-						log.Printf("[DEBUG] Shutting down (3)")
+					//err = deployApp(dockercli, image, identifier, env, workflow, action.ID)
-						shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true)
+					err = deployApp(dockercli, image, identifier, env, workflowExecution, action.ID)
-					}
+					if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") {
-
+						if strings.Contains(err.Error(), "exited prematurely") {
-					log.Printf("[WARNING] Failed CLEANUP execution. Downloading image %s remotely.", image)
+							log.Printf("[DEBUG] Shutting down (41)")
-
+							shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true)
-					err := downloadDockerImageBackend(topClient, image)
+						}
-					if err == nil {
-						log.Printf("[DEBUG] Downloaded image %s from backend (CLEANUP)", image)
 					} else {
+						executed = true
+					}
+				}
+
+				if !executed {
+					image = images[2]
+					err = deployApp(dockercli, image, identifier, env, workflowExecution, action.ID)
+					if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") {
+						if strings.Contains(err.Error(), "exited prematurely") {
+							log.Printf("[DEBUG] Shutting down (3)")
+							shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true)
+						}
+
+						//log.Printf("[WARNING] Failed CLEANUP execution. Downloading image %s remotely.", image)
+
 						log.Printf("[WARNING] Failed to download image %s (CLEANUP): %s", image, err)
 
 						reader, err := dockercli.ImagePull(context.Background(), image, pullOptions)
@@ -1032,22 +1047,22 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) {
 
 							log.Printf("[INFO] Successfully downloaded %s", image)
 						}
-					}
 
-					err = deployApp(dockercli, image, identifier, env, workflowExecution, action.ID)
+						err = deployApp(dockercli, image, identifier, env, workflowExecution, action.ID)
-					if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") {
+						if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") {
 
-						log.Printf("[ERROR] Failed deploying image for the FOURTH time. Aborting if the image doesn't exist")
+							log.Printf("[ERROR] Failed deploying image for the FOURTH time. Aborting if the image doesn't exist")
-						if strings.Contains(err.Error(), "exited prematurely") {
+							if strings.Contains(err.Error(), "exited prematurely") {
-							log.Printf("[DEBUG] Shutting down (7)")
+								log.Printf("[DEBUG] Shutting down (7)")
-							shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true)
+								shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true)
-						}
+							}
 
-						if strings.Contains(err.Error(), "No such image") {
+							if strings.Contains(err.Error(), "No such image") {
-							//log.Printf("[WARNING] Failed deploying %s from image %s: %s", identifier, image, err)
+								//log.Printf("[WARNING] Failed deploying %s from image %s: %s", identifier, image, err)
-							log.Printf("[ERROR] Image doesn't exist. Shutting down")
+								log.Printf("[ERROR] Image doesn't exist. Shutting down")
-							log.Printf("[DEBUG] Shutting down (8)")
+								log.Printf("[DEBUG] Shutting down (8)")
-							shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true)
+								shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true)
+							}
 						}
 					}
 				}
@@ -1071,20 +1086,34 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) {
 						shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true)
 					}
 
-					image = images[2]
+					log.Printf("[DEBUG] Failed deploy. Downloading image %s", image)
-					err = deployApp(dockercli, image, identifier, env, workflowExecution, action.ID)
+					err := downloadDockerImageBackend(topClient, image)
-					if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") {
+					executed := false
-						if strings.Contains(err.Error(), "exited prematurely") {
+					if err == nil {
-							log.Printf("[DEBUG] Shutting down (11)")
+						log.Printf("[DEBUG] Downloaded image %s from backend (CLEANUP)", image)
-							shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true)
+						//err = deployApp(dockercli, image, identifier, env, workflow, action.ID)
-						}
+						err = deployApp(dockercli, image, identifier, env, workflowExecution, action.ID)
-
+						if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") {
-						log.Printf("[WARNING] Failed deploying image THREE TIMES. Attempting to download %s as last resort from backend and dockerhub.", image)
+							if strings.Contains(err.Error(), "exited prematurely") {
-
+								log.Printf("[DEBUG] Shutting down (40)")
-						err := downloadDockerImageBackend(topClient, image)
+								shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true)
-						if err == nil {
+							}
-							log.Printf("[DEBUG] Downloaded image %s from backend (CLEANUP)", image)
 						} else {
+							executed = true
+						}
+					}
+
+					if !executed {
+						image = images[2]
+						err = deployApp(dockercli, image, identifier, env, workflowExecution, action.ID)
+						if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") {
+							if strings.Contains(err.Error(), "exited prematurely") {
+								log.Printf("[DEBUG] Shutting down (11)")
+								shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true)
+							}
+
+							log.Printf("[WARNING] Failed deploying image THREE TIMES. Attempting to download %s as last resort from backend and dockerhub.", image)
+
 							reader, err := dockercli.ImagePull(context.Background(), image, pullOptions)
 							if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") {
 								log.Printf("[ERROR] Failed getting %s. The couldn't be find locally, AND is missing.", image)