Optimization: Made apps and executions way faster

backend/go-app/docker.go

@@ -246,6 +246,7 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin
 	if newerr != nil {
 		log.Printf("Failed reading Docker build STDOUT: %s", newerr)
 	} else {
+		log.Printf("STRING: %s", buildBuf.String())
 		if strings.Contains(buildBuf.String(), "errorDetail") {
 			log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), strings.Join(tags, "\n"))
 

backend/go-app/files.go

@@ -43,6 +43,8 @@ type File struct {
 	Md5sum       string   `json:"md5_sum" datastore:"md5_sum"`
 	Sha256sum    string   `json:"sha256_sum" datastore:"sha256_sum"`
 	FileSize     int64    `json:"filesize" datastore:"filesize"`
+	Duplicate    bool     `json:"duplicate" datastore:"duplicate"`
+	Subflows     []string `json:"subflows" datastore:"subflows"`
 }
 
 var basepath = os.Getenv("SHUFFLE_FILE_LOCATION")
@@ -393,7 +395,7 @@ func handleGetFileContent(resp http.ResponseWriter, request *http.Request) {
 		return
 	}
 
-	log.Printf("\n\nUser is trying to download file %s\n\n", fileId)
+	log.Printf("\n\n[INFO] User is trying to download file %s\n\n", fileId)
 
 	// 1. Check user directly
 	// 2. Check workflow execution authorization
@@ -772,6 +774,30 @@ func handleCreateFile(resp http.ResponseWriter, request *http.Request) {
 	fileId := uuid.NewV4().String()
 	downloadPath := fmt.Sprintf("%s/%s", folderPath, fileId)
 
+	duplicateWorkflows := []string{}
+	for _, trigger := range workflow.Triggers {
+		if trigger.AppName == "Shuffle Workflow" && trigger.TriggerType == "SUBFLOW" {
+			for _, parameter := range trigger.Parameters {
+				if parameter.Name == "workflow" && len(parameter.Value) > 0 {
+
+					found := false
+					for _, workflow := range duplicateWorkflows {
+						if workflow == parameter.Value {
+							found = true
+							break
+						}
+					}
+
+					if !found {
+						duplicateWorkflows = append(duplicateWorkflows, parameter.Value)
+					}
+
+					break
+				}
+			}
+		}
+	}
+
 	timeNow := time.Now().Unix()
 	newFile := File{
 		Id:           fileId,
@@ -783,6 +809,7 @@ func handleCreateFile(resp http.ResponseWriter, request *http.Request) {
 		OrgId:        curfile.OrgId,
 		WorkflowId:   curfile.WorkflowId,
 		DownloadPath: downloadPath,
+		Subflows:     duplicateWorkflows,
 	}
 
 	err = setFile(ctx, newFile)
@@ -797,6 +824,7 @@ func handleCreateFile(resp http.ResponseWriter, request *http.Request) {
 
 	resp.WriteHeader(200)
 	resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s"}`, fileId)))
+
 }
 
 func getFile(ctx context.Context, id string) (*File, error) {

backend/go-app/walkoff.go

@@ -893,6 +893,53 @@ func findChildNodes(workflowExecution WorkflowExecution, nodeId string) []string
 	return newNodes
 }
 
+// Checks if data is sent from Worker >0.8.51, which sends a full execution
+// instead of individial results
+func validateNewWorkerExecution(body []byte) error {
+	//type WorkflowExecution struct {
+	//}
+
+	ctx := context.Background()
+	var execution WorkflowExecution
+	err := json.Unmarshal(body, &execution)
+	if err != nil {
+		log.Printf("[WARNING] Failed execution unmarshaling: %s", err)
+		return err
+	}
+
+	baseExecution, err := getWorkflowExecution(ctx, execution.ExecutionId)
+	if err != nil {
+		log.Printf("[ERROR] Failed getting execution (workflowqueue) %s: %s", execution.ExecutionId, err)
+		return err
+	}
+
+	if baseExecution.Authorization != execution.Authorization {
+		return errors.New("Bad authorization when validating execution")
+	}
+
+	if len(baseExecution.Workflow.Actions) != len(execution.Workflow.Actions) {
+		return errors.New(fmt.Sprintf("Bad length of actions: %d", len(execution.Workflow.Actions)))
+	}
+
+	if len(baseExecution.Workflow.Triggers) != len(execution.Workflow.Triggers) {
+		return errors.New(fmt.Sprintf("Bad length of trigger: %d", len(execution.Workflow.Triggers)))
+	}
+
+	// FIXME: Add extra here
+	//executionLength := len(baseExecution.Workflow.Actions)
+	//if executionLength != len(execution.Results) {
+	//	return errors.New(fmt.Sprintf("Bad length of actions vs results: want: %d have: %d", executionLength, len(execution.Results)))
+	//}
+
+	//log.Printf("\n\nSHOULD SET BACKEND DATA FOR EXEC \n\n")
+	err = setWorkflowExecution(ctx, execution, true)
+	if err != nil {
+		log.Printf("Successfully set the execution to wait.")
+	}
+
+	return nil
+}
+
 func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
 	cors := handleCors(resp, request)
 	if cors {
@@ -907,6 +954,17 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
 		return
 	}
 
+	//log.Printf("Actionresult unmarshal: %s", string(body))
+	err = validateNewWorkerExecution(body)
+	if err == nil {
+		log.Printf("[INFO] Set workflowexecution based on new worker")
+		resp.WriteHeader(200)
+		resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Success"}`)))
+		return
+	} else {
+		log.Printf("[WARNING] Failed to handle new execution variant: %s", err)
+	}
+
 	var actionResult ActionResult
 	err = json.Unmarshal(body, &actionResult)
 	if err != nil {
@@ -984,7 +1042,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
 			workflowExecution.Status = "ABORTED"
 			err = setWorkflowExecution(ctx, *workflowExecution, true)
 			if err != nil {
-				log.Printf("Failed ")
+				log.Printf("Failed to set execution during wait")
 			} else {
 				log.Printf("Successfully set the execution to waiting.")
 			}
@@ -2985,7 +3043,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf
 
 	allAuths := []AppAuthenticationStorage{}
 	for _, action := range workflowExecution.Workflow.Actions {
-		action.LargeImage = ""
+		//action.LargeImage = ""
 		if action.ID == workflowExecution.Start {
 			startFound = true
 		}
@@ -3036,6 +3094,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf
 			action.Parameters = newParams
 		}
 
+		action.LargeImage = ""
 		newActions = append(newActions, action)
 
 		// If the node is NOT found, it's supposed to be set to SKIPPED,