Merging changes from 1.2.0 for cleanup

backend/go-app/docker.go

@@ -698,7 +698,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) {
 	// REBUILDS THE APP
 	if len(img.ID) == 0 {
 		if len(img2.ID) == 0 {
-			workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 0)
+			workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 0, 0)
 			log.Printf("[INFO] Getting workflowapps for a rebuild. Got %d with err %#v", len(workflowapps), err)
 			if err == nil {
 				imageName := ""

backend/go-app/go.mod

@@ -7,44 +7,29 @@ replace github.com/shuffle/shuffle-shared => ../../../../git/shuffle-shared
 //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi
 
 require (
-	cloud.google.com/go v0.75.0
 	cloud.google.com/go/datastore v1.4.0
 	cloud.google.com/go/pubsub v1.3.1
 	cloud.google.com/go/storage v1.12.0
-	github.com/Masterminds/semver v1.5.0 // indirect
-	github.com/RobotsAndPencils/go-saml v0.0.0-20170520135329-fb13cb52a46b // indirect
-	github.com/algolia/algoliasearch-client-go/v3 v3.18.1 // indirect
 	github.com/basgys/goxml2json v1.1.0
-	github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 // indirect
+	github.com/bitly/go-simplejson v0.5.1 // indirect
 	github.com/carlescere/scheduler v0.0.0-20170109141437-ee74d2f83d82
 	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/docker v20.10.3-0.20210216175712-646072ed6524+incompatible
-	github.com/docker/go-connections v0.4.0
+	github.com/frikky/kin-openapi v0.41.0
-	github.com/docker/go-units v0.4.0 // indirect
-	github.com/elastic/go-elasticsearch/v7 v7.13.1 // indirect
-	github.com/frikky/kin-openapi v0.39.0
-	github.com/frikky/shuffle-shared v0.1.15
 	github.com/fsouza/go-dockerclient v1.7.2
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-git/go-billy/v5 v5.0.0
 	github.com/go-git/go-git/v5 v5.0.0
-	github.com/google/go-github/v28 v28.1.1
-	github.com/gorilla/handlers v1.4.2 // indirect
 	github.com/gorilla/mux v1.8.0
 	github.com/h2non/filetype v1.0.12
-	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
+	github.com/opensearch-project/opensearch-go v1.1.0 // indirect
-	github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d // indirect
+	github.com/opensearch-project/opensearch-go/v2 v2.3.0 // indirect
-	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/satori/go.uuid v1.2.0
 	github.com/shuffle/shuffle-shared v0.1.15
-	go4.org v0.0.0-20201209231011-d4a079459e60 // indirect
+	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
-	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
-	golang.org/x/oauth2 v0.0.0-20210113160501-8b1d76fa0423
 	google.golang.org/api v0.36.0
 	google.golang.org/appengine v1.6.7
-	google.golang.org/genproto v0.0.0-20210113195801-ae06605f4595
 	google.golang.org/grpc v1.34.1
 	gopkg.in/src-d/go-git.v4 v4.13.1
-	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/yaml.v3 v3.0.1
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )

backend/go-app/main.go

@@ -3028,7 +3028,7 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) {
 	if test.Editing && len(user.Id) > 0 {
 		// Quick verification test
 		ctx := context.Background()
-		app, err := shuffle.GetApp(ctx, test.Id, user)
+		app, err := shuffle.GetApp(ctx, test.Id, user, false)
 		if err != nil {
 			log.Printf("[WARNING] Error getting app when editing: %s", app.Name)
 			resp.WriteHeader(401)
@@ -4019,7 +4019,7 @@ func runInitEs(ctx context.Context) {
 
 	// Getting apps to see if we should initialize a test
 	// FIXME: Isn't this a little backwards?
-	workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000)
+	workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000, 0)
 	log.Printf("[INFO] Getting and validating workflowapps. Got %d with err %#v", len(workflowapps), err)
 
 	// accept any certificate (might be useful for testing)
@@ -4671,7 +4671,7 @@ func runInit(ctx context.Context) {
 	}
 
 	// Getting apps to see if we should initialize a test
-	workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000)
+	workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000,0 )
 	log.Printf("[INFO] Getting and validating workflowapps. Got %d with err %s", len(workflowapps), err)
 	if err != nil && len(workflowapps) == 0 {
 		log.Printf("[WARNING] Failed getting apps (runInit): %s", err)
@@ -5286,7 +5286,7 @@ func migrateDatabase(resp http.ResponseWriter, request *http.Request) {
 		log.Printf("[DEBUG] Found %d workflows(s) to be migrated", len(workflows))
 	}
 
-	apps, err := shuffle.GetAllWorkflowApps(ctx, 0)
+	apps, err := shuffle.GetAllWorkflowApps(ctx, 0, 0)
 	if err != nil {
 		log.Printf("[ERROR] Failed getting apps: %#v", err)
 	} else {
@@ -5729,6 +5729,7 @@ func initHandlers() {
 	r.HandleFunc("/api/v1/workflows/{key}", deleteWorkflow).Methods("DELETE", "OPTIONS")
 	r.HandleFunc("/api/v1/workflows/{key}", shuffle.SaveWorkflow).Methods("PUT", "OPTIONS")
 	r.HandleFunc("/api/v1/workflows/{key}", shuffle.GetSpecificWorkflow).Methods("GET", "OPTIONS")
+	r.HandleFunc("/api/v1/workflows/{key}/revisions", shuffle.GetWorkflowRevisions).Methods("GET", "OPTIONS")
 
 	// Triggers
 	r.HandleFunc("/api/v1/hooks/new", shuffle.HandleNewHook).Methods("POST", "OPTIONS")

backend/go-app/walkoff.go

@@ -577,7 +577,7 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque
 
 	//setWorkflowqueuetest(id)
 	ctx := context.Background()
-	executionRequests, err := shuffle.GetWorkflowQueue(ctx, id)
+	executionRequests, err := shuffle.GetWorkflowQueue(ctx, id, 10)
 	if err != nil {
 		log.Printf("(1) Failed reading body for workflowqueue: %s", err)
 		resp.WriteHeader(401)
@@ -676,7 +676,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
 	}
 
 	ctx := context.Background()
-	executionRequests, err := shuffle.GetWorkflowQueue(ctx, id)
+	executionRequests, err := shuffle.GetWorkflowQueue(ctx, id, 10)
 	if err != nil {
 		// Skipping as this comes up over and over
 		//log.Printf("(2) Failed reading body for workflowqueue: %s", err)
@@ -782,7 +782,8 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
 
 	//log.Printf("Actionresult unmarshal: %s", string(body))
 	log.Printf("[DEBUG] Got workflow result from %s of length %d", request.RemoteAddr, len(body))
-	err = shuffle.ValidateNewWorkerExecution(body)
+	ctx := context.Background()
+	err = shuffle.ValidateNewWorkerExecution(ctx, body)
 	if err == nil {
 		resp.WriteHeader(200)
 		resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "success"}`)))
@@ -808,7 +809,6 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
 	// 4. Push to db
 	// IF FAIL: Set executionstatus: abort or cancel
 
-	ctx := context.Background()
 	workflowExecution, err := shuffle.GetWorkflowExecution(ctx, actionResult.ExecutionId)
 	if err != nil {
 		log.Printf("[ERROR] Failed getting execution (workflowqueue) %s: %s", actionResult.ExecutionId, err)
@@ -910,7 +910,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl
 	}
 
 	//log.Printf("BASE LENGTH: %d", len(workflowExecution.Results))
-	workflowExecution, dbSave, err := shuffle.ParsedExecutionResult(ctx, *workflowExecution, actionResult, false)
+	workflowExecution, dbSave, err := shuffle.ParsedExecutionResult(ctx, *workflowExecution, actionResult, false, 0)
 	if err != nil {
 		log.Printf("[ERROR] Failed running of parsedexecution: %s", err)
 		resp.WriteHeader(401)
@@ -1514,7 +1514,7 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request
 		}
 	}
 
-	childNodes := shuffle.FindChildNodes(workflowExecution, workflowExecution.Start)
+	childNodes := shuffle.FindChildNodes(workflowExecution, workflowExecution.Start, []string{}, []string{})
 
 	topic := "workflows"
 	startFound := false
@@ -1561,14 +1561,14 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request
 				newFields := []shuffle.AuthenticationStore{}
 				for _, field := range curAuth.Fields {
 					parsedKey := fmt.Sprintf("%s_%d_%s_%s", curAuth.OrgId, curAuth.Created, curAuth.Label, field.Key)
-					newValue, err := shuffle.HandleKeyDecryption(field.Value, parsedKey)
+					newValue, err := shuffle.HandleKeyDecryption([]byte(field.Value), parsedKey)
 					if err != nil {
 						log.Printf("[WARNING] Failed decryption for %s: %s", field.Key, err)
 						setField = false
 						break
 					}
 
-					field.Value = newValue
+					field.Value = string(newValue)
 					newFields = append(newFields, field)
 				}
 
@@ -2655,7 +2655,7 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) {
 		return
 	}
 
-	workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000)
+	workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000, 0)
 	if err != nil {
 		log.Printf("{WARNING] Failed getting apps (getworkflowapps): %s", err)
 		resp.WriteHeader(401)
@@ -2753,7 +2753,7 @@ func getSpecificApps(resp http.ResponseWriter, request *http.Request) {
 	// FIXME - continue the search here with github repos etc.
 	// Caching might be smart :D
 	ctx := context.Background()
-	workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000)
+	workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000, 0)
 	if err != nil {
 		log.Printf("Error: Failed getting workflowapps: %s", err)
 		resp.WriteHeader(401)
@@ -3023,7 +3023,7 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) {
 func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error {
 
 	ctx := context.Background()
-	workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000)
+	workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000, 0)
 	appCounter := 0
 	if err != nil {
 		log.Printf("Failed to get existing generated apps")
@@ -3316,7 +3316,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) {
 	}
 
 	ctx := context.Background()
-	allapps, err := shuffle.GetAllWorkflowApps(ctx, 1000)
+	allapps, err := shuffle.GetAllWorkflowApps(ctx, 1000, 0)
 	if err != nil {
 		log.Printf("Failed getting apps to verify: %s", err)
 		resp.WriteHeader(401)
@@ -3582,10 +3582,14 @@ func executeSingleAction(resp http.ResponseWriter, request *http.Request) {
 	time.Sleep(2 * time.Second)
 	log.Printf("[INFO] Starting validation of execution %s", workflowExecution.ExecutionId)
 
-	returnBytes := shuffle.HandleRetValidation(ctx, workflowExecution)
+	returnBody := shuffle.HandleRetValidation(ctx, workflowExecution, 1)
+	returnBytes, err := json.Marshal(returnBody)
+	if err != nil {
+		log.Printf("[ERROR] Failed to marshal retStruct in single execution: %s", err)
+	}
 
 	resp.WriteHeader(200)
-	resp.Write(returnBytes)
+	resp.Write([]byte(returnBytes))
 }
 
 // Onlyname is used to
@@ -3766,7 +3770,7 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
 				}
 
 				if len(allapps) == 0 {
-					allapps, err = shuffle.GetAllWorkflowApps(ctx, 0)
+					allapps, err = shuffle.GetAllWorkflowApps(ctx, 0, 0)
 					if err != nil {
 						log.Printf("[WARNING] Failed getting apps to verify: %s", err)
 						continue

docker-compose.yml

@@ -66,7 +66,7 @@ services:
       - CLEANUP=${SHUFFLE_CONTAINER_AUTO_CLEANUP}
     restart: unless-stopped
   opensearch:
-    image: opensearchproject/opensearch:1.1.0
+    image: opensearchproject/opensearch:2.5.0
     hostname: shuffle-opensearch
     container_name: shuffle-opensearch
     environment: