Added github download redirect handler for raw.githubusercontent.com

backend/app_sdk/app_base.py

@@ -73,13 +73,17 @@ class AppBase:
             self.logger.info("[DEBUG] Already JSON-like. Returning from magic")
             return input_data
 
+        if len(input_data) < 3:
+            self.logger.info("[DEBUG] Too short input data")
+            return input_data
+
         # Don't touch large data.
         if len(input_data) > 100000:
             self.logger.info("[DEBUG] Value too large. Returning from magic")
             return input_data
 
         if not "\n" in input_data and not "," in input_data: 
-            self.logger.info("[DEBUG] No data to autoparse")
+            self.logger.info("[DEBUG] No data to autoparse - requires newline or comma")
             return input_data
 
         new_input = input_data

backend/app_sdk/build.sh

@@ -3,7 +3,7 @@
 
 ### DEFAULT 
 NAME=shuffle-app_sdk
-VERSION=0.9.40
+VERSION=0.9.44
 
 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force
 docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly

backend/go-app/go.mod

@@ -2,7 +2,7 @@ module main
 
 go 1.15
 
-//replace github.com/shuffle/shuffle-shared => ../../../../git/shuffle-shared
+replace github.com/shuffle/shuffle-shared => ../../../../git/shuffle-shared
 
 //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi
 //replace github.com/frikky/go-elasticsearch => ../../../../git/go-elasticsearch

backend/go-app/main.go

@@ -2846,82 +2846,6 @@ func getOpenapi(resp http.ResponseWriter, request *http.Request) {
 	resp.Write(data)
 }
 
-func echoOpenapiData(resp http.ResponseWriter, request *http.Request) {
-	cors := handleCors(resp, request)
-	if cors {
-		return
-	}
-
-	// Just here to verify that the user is logged in
-	user, err := shuffle.HandleApiAuthentication(resp, request)
-	if err != nil {
-		log.Printf("Api authentication failed in validate swagger: %s", err)
-		resp.WriteHeader(401)
-		resp.Write([]byte(`{"success": false, "reason": "Failed authentication"}`))
-		return
-	}
-
-	if user.Role == "org-reader" {
-		log.Printf("[WARNING] Org-reader doesn't have access to echo OpenAPI data: %s (%s)", user.Username, user.Id)
-		resp.WriteHeader(401)
-		resp.Write([]byte(`{"success": false, "reason": "Read only user"}`))
-		return
-	}
-
-	body, err := ioutil.ReadAll(request.Body)
-	if err != nil {
-		log.Printf("Bodyreader err: %s", err)
-		resp.WriteHeader(401)
-		resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`))
-		return
-	}
-
-	newbody := string(body)
-	newbody = strings.TrimSpace(newbody)
-	if strings.HasPrefix(newbody, "\"") {
-		newbody = newbody[1:len(newbody)]
-	}
-
-	if strings.HasSuffix(newbody, "\"") {
-		newbody = newbody[0 : len(newbody)-1]
-	}
-
-	req, err := http.NewRequest("GET", newbody, nil)
-	if err != nil {
-		log.Printf("[ERROR] Requestbuilder err: %s", err)
-		resp.WriteHeader(500)
-		resp.Write([]byte(`{"success": false, "reason": "Failed building request"}`))
-		return
-	}
-
-	httpClient := &http.Client{}
-	newresp, err := httpClient.Do(req)
-	if err != nil {
-		log.Printf("[ERROR] Grabbing error: %s", err)
-		resp.WriteHeader(500)
-		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed making remote request to get the data"}`)))
-		return
-	}
-	defer newresp.Body.Close()
-
-	urlbody, err := ioutil.ReadAll(newresp.Body)
-	if err != nil {
-		log.Printf("[ERROR] URLbody error: %s", err)
-		resp.WriteHeader(500)
-		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't get data from selected uri"}`)))
-		return
-	}
-
-	if newresp.StatusCode >= 400 {
-		resp.WriteHeader(201)
-		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, urlbody)))
-		return
-	}
-
-	resp.WriteHeader(200)
-	resp.Write(urlbody)
-}
-
 func handleSwaggerValidation(body []byte) (shuffle.ParsedOpenApi, error) {
 	type versionCheck struct {
 		Swagger        string `datastore:"swagger" json:"swagger" yaml:"swagger"`
@@ -5888,7 +5812,7 @@ func initHandlers() {
 	// OpenAPI configuration
 	r.HandleFunc("/api/v1/verify_swagger", verifySwagger).Methods("POST", "OPTIONS")
 	r.HandleFunc("/api/v1/verify_openapi", verifySwagger).Methods("POST", "OPTIONS")
-	r.HandleFunc("/api/v1/get_openapi_uri", echoOpenapiData).Methods("POST", "OPTIONS")
+	r.HandleFunc("/api/v1/get_openapi_uri", shuffle.EchoOpenapiData).Methods("POST", "OPTIONS")
 	r.HandleFunc("/api/v1/validate_openapi", shuffle.ValidateSwagger).Methods("POST", "OPTIONS")
 	r.HandleFunc("/api/v1/get_openapi/{key}", getOpenapi).Methods("GET", "OPTIONS")
 

frontend/src/views/AngularWorkflow.jsx

@@ -2059,7 +2059,7 @@ const AngularWorkflow = (props) => {
         const parentNode = cy.getElementById(data.attachedTo);
         if (parentNode !== null && parentNode !== undefined) {
           var newNodeData = JSON.parse(JSON.stringify(parentNode.data()));
-          newNodeData.id = uuidv4.v4();
+          newNodeData.id = uuidv4();
           if (newNodeData.position !== undefined) {
             newNodeData.position = {
               x: newNodeData.position.x + 100,

functions/onprem/orborus/build.sh

@@ -1,5 +1,5 @@
 NAME=shuffle-orborus
-VERSION=0.9.42
+VERSION=0.9.44
 
 echo "Running docker build with $NAME:$VERSION"
 #docker rmi frikky/shuffle:$NAME --force

functions/onprem/worker/build.sh

@@ -1,5 +1,5 @@
 NAME=shuffle-worker
-VERSION=0.9.43
+VERSION=0.9.44
 
 echo "Running docker build with $NAME:$VERSION"
 #CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin .

functions/onprem/worker/worker.go

@@ -85,7 +85,6 @@ var autoDeploy = map[string]string{
 	"testing:1.0.0":         "frikky/shuffle:testing_1.0.0",
 }
 
-//if !shuffle.ArrayContains(executedIds,
 //fmt.Sprintf("%s_%s", workflowExecution.ExecutionId, action.ID)
 
 // New Worker mappings
@@ -855,6 +854,20 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) {
 			continue
 		}
 
+		newExecId := fmt.Sprintf("%s_%s", workflowExecution.ExecutionId, nextAction)
+		if !shuffle.ArrayContains(executedIds, newExecId) {
+			executedIds = append(executedIds, newExecId)
+			toRemove = append(toRemove, index)
+		} else {
+			log.Printf("\n\n[DEBUG] %s is already executed. Continuing.", newExecId)
+			continue
+		}
+
+		// max 1000 :o
+		if len(executedIds) >= 1000 {
+			executedIds = executedIds[900:999]
+		}
+
 		if action.AppName == "Shuffle Tools" && (action.Name == "skip_me" || action.Name == "router" || action.Name == "route") {
 			err := runSkipAction(topClient, action, workflowExecution.Workflow.ID, workflowExecution.ExecutionId, workflowExecution.Authorization, "SKIPPED")
 			if err != nil {
@@ -2888,7 +2901,8 @@ func main() {
 		}
 	*/
 
-	_, err := shuffle.RunInit(datastore.Client{}, storage.Client{}, "", "", true, "")
+	// Elasticsearch necessary to ensure we'ren ot running with Datastore configurations for minimal/maximal data sizes
+	_, err := shuffle.RunInit(datastore.Client{}, storage.Client{}, "", "", true, "elasticsearch")
 	if err != nil {
 		log.Printf("[ERROR] Failed to run worker init: %s", err)
 	} else {