Updated compose to 0.8.64

backend/Dockerfile

@@ -9,6 +9,7 @@ ADD ./go-app/walkoff.go /app
 ADD ./go-app/docker.go /app
 ADD ./go-app/codegen.go /app
 ADD ./go-app/files.go /app
+ADD ./go-app/oauth2.go /app
 
 ADD ./go-app/go.mod /app
 

backend/go-app/main.go

@@ -5882,19 +5882,19 @@ func handleCloudJob(job CloudSyncJob) error {
 			log.Printf("[INFO] Should handle outlook webhook for workflow %s with start node %s and data of length %d", job.PrimaryItemId, job.SecondaryItem, len(job.ThirdItem))
 			err = handleCloudExecutionOnprem(job.PrimaryItemId, job.SecondaryItem, "outlook", string(emailBytes))
 			if err != nil {
-				log.Printf("Failed executing workflow from cloud outlook hook: %s", err)
+				log.Printf("[WARNING] Failed executing workflow from cloud outlook hook: %s", err)
 			} else {
-				log.Printf("Successfully executed workflow from cloud outlook hook!")
+				log.Printf("[INFO] Successfully executed workflow from cloud outlook hook!")
 			}
 		}
 	} else if job.Type == "webhook" {
 		if job.Action == "execute" {
-			log.Printf("Should handle normal webhook for workflow %s with start node %s and data %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem)
+			log.Printf("[INFO] Should handle normal webhook for workflow %s with start node %s and data %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem)
 			err := handleCloudExecutionOnprem(job.PrimaryItemId, job.SecondaryItem, "webhook", job.ThirdItem)
 			if err != nil {
-				log.Printf("Failed executing workflow from cloud hook: %s", err)
+				log.Printf("[INFO] Failed executing workflow from cloud hook: %s", err)
 			} else {
-				log.Printf("Successfully executed workflow from cloud hook!")
+				log.Printf("[INFO] Successfully executed workflow from cloud hook!")
 			}
 		}
 
@@ -5903,9 +5903,9 @@ func handleCloudJob(job CloudSyncJob) error {
 			log.Printf("Should handle schedule for workflow %s with start node %s and data %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem)
 			err := handleCloudExecutionOnprem(job.PrimaryItemId, job.SecondaryItem, "schedule", job.ThirdItem)
 			if err != nil {
-				log.Printf("Failed executing workflow from cloud schedule: %s", err)
+				log.Printf("[INFO] Failed executing workflow from cloud schedule: %s", err)
 			} else {
-				log.Printf("Successfully executed workflow from cloud schedule")
+				log.Printf("[INFO] Successfully executed workflow from cloud schedule")
 			}
 		}
 	} else if job.Type == "email_trigger" {

backend/go-app/oauth2.go

@@ -17,6 +17,41 @@ import (
 	"golang.org/x/oauth2"
 )
 
+// This is what the structure should be when it's sent into a workflow
+type ParsedShuffleMail struct {
+	Body struct {
+		URI           []string `json:"uri"`
+		Email         []string `json:"email"`
+		Domain        []string `json:"domain"`
+		ContentHeader struct {
+		} `json:"content_header"`
+		Content     string `json:"content"`
+		ContentType string `json:"content_type"`
+		Hash        string `json:"hash"`
+		RawBody     string `json:"raw_body"`
+	} `json:"body"`
+	Header struct {
+		Subject  string   `json:"subject"`
+		From     string   `json:"from"`
+		To       []string `json:"to"`
+		Date     string   `json:"date"`
+		Received []struct {
+			Src  string   `json:"src"`
+			From []string `json:"from"`
+			By   []string `json:"by"`
+			With string   `json:"with"`
+			Date string   `json:"date"`
+		} `json:"received"`
+		ReceivedDomain []string `json:"received_domain"`
+		ReceivedIP     []string `json:"received_ip"`
+		Header         struct {
+		} `json:"header"`
+	} `json:"header"`
+	MessageID      string   `json:"message_id"`
+	EmailFileid    string   `json:"email_fileid"`
+	AttachmentUids []string `json:"attachment_uids"`
+}
+
 type FullEmail struct {
 	OdataContext               string        `json:"@odata.context"`
 	OdataEtag                  string        `json:"@odata.etag"`
@@ -69,6 +104,19 @@ type FullEmail struct {
 	Flag          struct {
 		Flagstatus string `json:"flagStatus"`
 	} `json:"flag"`
+	Attachments []struct {
+		OdataType             string      `json:"@odata.type"`
+		OdataMediacontenttype string      `json:"@odata.mediaContentType"`
+		ID                    string      `json:"id"`
+		Lastmodifieddatetime  time.Time   `json:"lastModifiedDateTime"`
+		Name                  string      `json:"name"`
+		Contenttype           string      `json:"contentType"`
+		Size                  int         `json:"size"`
+		Isinline              bool        `json:"isInline"`
+		Contentid             interface{} `json:"contentId"`
+		Contentlocation       interface{} `json:"contentLocation"`
+		Contentbytes          string      `json:"contentBytes"`
+	}
 }
 
 type MailData struct {
@@ -118,6 +166,47 @@ type OutlookFolders struct {
 	Value         []OutlookFolder `json:"value"`
 }
 
+func getOutlookAttachment(client *http.Client, emailId, attachmentId string) ([]FullEmail, error) {
+	//requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/users/ec03b4f2-fccf-4c35-b0eb-be85a0f5dd43/mailFolders")
+
+	requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/%s/attachments/%s", emailId, attachmentId)
+	//log.Printf("Outlook email URL: %#v", requestUrl)
+
+	ret, err := client.Get(requestUrl)
+	if err != nil {
+		log.Printf("[INFO] OutlookErr: %s", err)
+		return []FullEmail{}, err
+	}
+
+	body, err := ioutil.ReadAll(ret.Body)
+	if err != nil {
+		log.Printf("[WARNING] Failed body decoding from outlook email")
+		return []FullEmail{}, err
+	}
+
+	//type FullEmail struct {
+	log.Printf("[INFO] Attachment Body: %s", string(body))
+	log.Printf("[INFO] Status email: %d", ret.StatusCode)
+	if ret.StatusCode != 200 {
+		return []FullEmail{}, err
+	}
+
+	//log.Printf("Body: %s", string(body))
+
+	/*
+		parsedmail := FullEmail{}
+		err = json.Unmarshal(body, &parsedmail)
+		if err != nil {
+			log.Printf("[INFO] Email unmarshal error: %s", err)
+			return []FullEmail{}, err
+		}
+
+		emails = append(emails, parsedmail)
+	*/
+
+	return []FullEmail{}, nil
+}
+
 func getOutlookEmail(client *http.Client, maildata MailData) ([]FullEmail, error) {
 	//requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/users/ec03b4f2-fccf-4c35-b0eb-be85a0f5dd43/mailFolders")
 
@@ -991,6 +1080,38 @@ func handleOutlookCallback(resp http.ResponseWriter, request *http.Request) {
 		email = emails[0]
 	}
 
+	// Parse indicators (domains, emails, ips, domains etc)!
+	newEmail := ParsedShuffleMail{}
+	newEmail.Body.ContentType = email.Body.Contenttype
+	newEmail.Body.Content = email.Body.Content
+	newEmail.Body.RawBody = email.Body.Content
+
+	newEmail.Header.Subject = email.Subject
+	newEmail.Header.From = email.From.Emailaddress.Address
+	for _, to := range email.Torecipients {
+		newEmail.Header.To = append(newEmail.Header.To, to.Emailaddress.Address)
+	}
+	newEmail.Header.Date = email.Receiveddatetime.String()
+
+	newEmail.MessageID = email.ID
+
+	if email.Hasattachments {
+		log.Printf("SHOULD HANDLE ATTACHMENTS FOR EMAIL!")
+
+		for _, attachment := range email.Attachments {
+			parsedAttachment, err := getOutlookAttachment(outlookClient, email.ID, attachment.ID)
+			if err != nil {
+				log.Printf("Failed attachment %s: %s", attachment.ID, err)
+				continue
+			}
+
+			log.Printf("ATTACHMENT: %#v", parsedAttachment)
+		}
+		//log.Printf("%#v", attachments)
+		//log.Printf("%s", err)
+		//GET /users/{id | userPrincipalName}/events/{id}/attachments/{id}
+	}
+
 	emailBytes, err := json.Marshal(email)
 	if err != nil {
 		log.Printf("[INFO] Failed email marshaling: %s", err)

backend/go-app/walkoff.go

@@ -2565,6 +2565,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
 
 	// Check every app action and param to see whether they exist
 	//log.Printf("PRE ACTIONS 2")
+	allAuths, autherr := getAllWorkflowAppAuth(ctx, user.ActiveOrg.Id)
 	newActions = []Action{}
 	for _, action := range workflow.Actions {
 		reservedApps := []string{
@@ -2655,34 +2656,60 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
 				// Check to see if the action is valid
 				if curappaction.Name != action.Name {
 					log.Printf("[ERROR] Action %s in app %s doesn't exist.", action.Name, curapp.Name)
-					if workflow.PreviouslySaved {
+					thisError := fmt.Sprintf("%s: Action %s in app %s doesn't exist", action.Label, action.Name, action.AppName)
-						resp.WriteHeader(401)
+					workflow.Errors = append(workflow.Errors, thisError)
-						resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Action %s in app %s doesn't exist"}`, action.Name, curapp.Name)))
+					workflow.IsValid = false
-						return
+					action.Errors = append(action.Errors, thisError)
-					}
+					action.IsValid = false
+					//if workflow.PreviouslySaved {
+					//	resp.WriteHeader(401)
+					//	resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Action %s in app %s doesn't exist"}`, action.Name, curapp.Name)))
+					//	return
+					//}
 				}
 
 				// FIXME - check all parameters to see if they're valid
 				// Includes checking required fields
 
+				selectedAuth := AppAuthenticationStorage{}
+				if len(action.AuthenticationId) > 0 && autherr == nil {
+					for _, auth := range allAuths {
+						if auth.Id == action.AuthenticationId {
+							selectedAuth = auth
+							break
+						}
+					}
+				}
+
 				newParams := []WorkflowAppActionParameter{}
 				for _, param := range curappaction.Parameters {
-					found := false
+					paramFound := false
 
 					// Handles check for parameter exists + value not empty in used fields
 					for _, actionParam := range action.Parameters {
 						if actionParam.Name == param.Name {
-							found = true
+							paramFound = true
 
 							if actionParam.Value == "" && actionParam.Variant == "STATIC_VALUE" && actionParam.Required == true {
-								log.Printf("[WARNING] Appaction %s with required param '%s' is empty. Can't save.", action.Name, param.Name)
+								// Validating if the field is an authentication field
-								//if workflow.PreviouslySaved {
+								if len(selectedAuth.Id) > 0 {
-								//	resp.WriteHeader(401)
+									authFound := false
-								//	resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s in app %s with required param '%s' is empty.", "node_id": "%s"}`, action.Name, action.AppName, param.Name, action.ID)))
+									for _, field := range selectedAuth.Fields {
-								//	return
+										if field.Key == actionParam.Name {
-								//} else {
+											authFound = true
+											//log.Printf("FOUND REQUIRED KEY %s IN AUTH", field.Key)
+											break
+										}
+									}
 
-								thisError := fmt.Sprintf("Missing parameter %s", param.Name)
+									if authFound {
+										newParams = append(newParams, actionParam)
+										continue
+									}
+								}
+
+								log.Printf("[WARNING] Appaction %s with required param '%s' is empty. Can't save.", action.Name, param.Name)
+								thisError := fmt.Sprintf("%s is missing reqired parameter %s", action.Label, param.Name)
 								action.Errors = append(action.Errors, thisError)
 								workflow.Errors = append(workflow.Errors, thisError)
 								action.IsValid = false
@@ -2698,7 +2725,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
 					}
 
 					// Handles check for required params
-					if !found && param.Required {
+					if !paramFound && param.Required {
 						log.Printf("Appaction %s with required param %s doesn't exist.", action.Name, param.Name)
 						thisError := fmt.Sprintf("Parameter %s is required", param.Name)
 						action.Errors = append(action.Errors, thisError)
@@ -2719,13 +2746,10 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
 		}
 	}
 
-	//log.Printf("PRE SAVECHECK")
 	if !workflow.PreviouslySaved {
 		log.Printf("[WORKFLOW INIT] NOT PREVIOUSLY SAVED - SET ACTION AUTH!")
-		//AuthenticationId string `json:"authentication_id,omitempty" datastore:"authentication_id"`
 
-		allAuths, err := getAllWorkflowAppAuth(ctx, user.ActiveOrg.Id)
+		if autherr == nil && len(workflowapps) > 0 && apperr == nil {
-		if err == nil && len(workflowapps) > 0 && apperr == nil {
 			//log.Printf("Setting actions")
 			actionFixing := []Action{}
 			appsAdded := []string{}
@@ -5389,7 +5413,7 @@ func getAppAuthentication(resp http.ResponseWriter, request *http.Request) {
 		return
 	}
 
-	// Cleanup for frontend
+	// Cleanup for frontend usage. User shouldn't be able to get the data.
 	newAuth := []AppAuthenticationStorage{}
 	for _, auth := range allAuths {
 		newAuthField := auth

docker-compose.yml

@@ -2,7 +2,7 @@ version: '3'
 services:
   frontend:
     #build: ./frontend
-    image: ghcr.io/frikky/shuffle-frontend:0.8.63
+    image: ghcr.io/frikky/shuffle-frontend:0.8.64
     container_name: shuffle-frontend
     hostname: shuffle-frontend
     ports:
@@ -17,7 +17,7 @@ services:
       - backend
   backend:
     #build: ./backend
-    image: ghcr.io/frikky/shuffle-backend:0.8.63
+    image: ghcr.io/frikky/shuffle-backend:0.8.64
     container_name: shuffle-backend
     hostname: ${BACKEND_HOSTNAME}
     # Here for debugging:

frontend/src/views/AngularWorkflow.jsx

@@ -204,8 +204,10 @@ const AngularWorkflow = (props) => {
 	})
 
 	const [elements, setElements] = useState([])
+	// No point going as fast, as the nodes aren't realtime anymore, but bulk updated. 
+	// Set it from 2500 to 6000 to reduce overall load
 	const { start, stop } = useInterval({
-	  	duration: 2500,
+	  	duration: 6000,
 	  	startImmediate: false,
 	  	callback: () => {
 				fetchUpdates()