diff --git a/backend/Dockerfile b/backend/Dockerfile index 40bb50ea..ea93625e 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -9,6 +9,7 @@ ADD ./go-app/walkoff.go /app ADD ./go-app/docker.go /app ADD ./go-app/codegen.go /app ADD ./go-app/files.go /app +ADD ./go-app/oauth2.go /app ADD ./go-app/go.mod /app diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 737e8b0e..0d855263 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5882,19 +5882,19 @@ func handleCloudJob(job CloudSyncJob) error { log.Printf("[INFO] Should handle outlook webhook for workflow %s with start node %s and data of length %d", job.PrimaryItemId, job.SecondaryItem, len(job.ThirdItem)) err = handleCloudExecutionOnprem(job.PrimaryItemId, job.SecondaryItem, "outlook", string(emailBytes)) if err != nil { - log.Printf("Failed executing workflow from cloud outlook hook: %s", err) + log.Printf("[WARNING] Failed executing workflow from cloud outlook hook: %s", err) } else { - log.Printf("Successfully executed workflow from cloud outlook hook!") + log.Printf("[INFO] Successfully executed workflow from cloud outlook hook!") } } } else if job.Type == "webhook" { if job.Action == "execute" { - log.Printf("Should handle normal webhook for workflow %s with start node %s and data %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem) + log.Printf("[INFO] Should handle normal webhook for workflow %s with start node %s and data %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem) err := handleCloudExecutionOnprem(job.PrimaryItemId, job.SecondaryItem, "webhook", job.ThirdItem) if err != nil { - log.Printf("Failed executing workflow from cloud hook: %s", err) + log.Printf("[INFO] Failed executing workflow from cloud hook: %s", err) } else { - log.Printf("Successfully executed workflow from cloud hook!") + log.Printf("[INFO] Successfully executed workflow from cloud hook!") } } @@ -5903,9 +5903,9 @@ func handleCloudJob(job CloudSyncJob) error { log.Printf("Should handle schedule for workflow %s with start node %s and data %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem) err := handleCloudExecutionOnprem(job.PrimaryItemId, job.SecondaryItem, "schedule", job.ThirdItem) if err != nil { - log.Printf("Failed executing workflow from cloud schedule: %s", err) + log.Printf("[INFO] Failed executing workflow from cloud schedule: %s", err) } else { - log.Printf("Successfully executed workflow from cloud schedule") + log.Printf("[INFO] Successfully executed workflow from cloud schedule") } } } else if job.Type == "email_trigger" { diff --git a/backend/go-app/oauth2.go b/backend/go-app/oauth2.go index 8c4a355f..b00f6c7a 100644 --- a/backend/go-app/oauth2.go +++ b/backend/go-app/oauth2.go @@ -17,6 +17,41 @@ import ( "golang.org/x/oauth2" ) +// This is what the structure should be when it's sent into a workflow +type ParsedShuffleMail struct { + Body struct { + URI []string `json:"uri"` + Email []string `json:"email"` + Domain []string `json:"domain"` + ContentHeader struct { + } `json:"content_header"` + Content string `json:"content"` + ContentType string `json:"content_type"` + Hash string `json:"hash"` + RawBody string `json:"raw_body"` + } `json:"body"` + Header struct { + Subject string `json:"subject"` + From string `json:"from"` + To []string `json:"to"` + Date string `json:"date"` + Received []struct { + Src string `json:"src"` + From []string `json:"from"` + By []string `json:"by"` + With string `json:"with"` + Date string `json:"date"` + } `json:"received"` + ReceivedDomain []string `json:"received_domain"` + ReceivedIP []string `json:"received_ip"` + Header struct { + } `json:"header"` + } `json:"header"` + MessageID string `json:"message_id"` + EmailFileid string `json:"email_fileid"` + AttachmentUids []string `json:"attachment_uids"` +} + type FullEmail struct { OdataContext string `json:"@odata.context"` OdataEtag string `json:"@odata.etag"` @@ -69,6 +104,19 @@ type FullEmail struct { Flag struct { Flagstatus string `json:"flagStatus"` } `json:"flag"` + Attachments []struct { + OdataType string `json:"@odata.type"` + OdataMediacontenttype string `json:"@odata.mediaContentType"` + ID string `json:"id"` + Lastmodifieddatetime time.Time `json:"lastModifiedDateTime"` + Name string `json:"name"` + Contenttype string `json:"contentType"` + Size int `json:"size"` + Isinline bool `json:"isInline"` + Contentid interface{} `json:"contentId"` + Contentlocation interface{} `json:"contentLocation"` + Contentbytes string `json:"contentBytes"` + } } type MailData struct { @@ -118,6 +166,47 @@ type OutlookFolders struct { Value []OutlookFolder `json:"value"` } +func getOutlookAttachment(client *http.Client, emailId, attachmentId string) ([]FullEmail, error) { + //requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/users/ec03b4f2-fccf-4c35-b0eb-be85a0f5dd43/mailFolders") + + requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/%s/attachments/%s", emailId, attachmentId) + //log.Printf("Outlook email URL: %#v", requestUrl) + + ret, err := client.Get(requestUrl) + if err != nil { + log.Printf("[INFO] OutlookErr: %s", err) + return []FullEmail{}, err + } + + body, err := ioutil.ReadAll(ret.Body) + if err != nil { + log.Printf("[WARNING] Failed body decoding from outlook email") + return []FullEmail{}, err + } + + //type FullEmail struct { + log.Printf("[INFO] Attachment Body: %s", string(body)) + log.Printf("[INFO] Status email: %d", ret.StatusCode) + if ret.StatusCode != 200 { + return []FullEmail{}, err + } + + //log.Printf("Body: %s", string(body)) + + /* + parsedmail := FullEmail{} + err = json.Unmarshal(body, &parsedmail) + if err != nil { + log.Printf("[INFO] Email unmarshal error: %s", err) + return []FullEmail{}, err + } + + emails = append(emails, parsedmail) + */ + + return []FullEmail{}, nil +} + func getOutlookEmail(client *http.Client, maildata MailData) ([]FullEmail, error) { //requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/users/ec03b4f2-fccf-4c35-b0eb-be85a0f5dd43/mailFolders") @@ -991,6 +1080,38 @@ func handleOutlookCallback(resp http.ResponseWriter, request *http.Request) { email = emails[0] } + // Parse indicators (domains, emails, ips, domains etc)! + newEmail := ParsedShuffleMail{} + newEmail.Body.ContentType = email.Body.Contenttype + newEmail.Body.Content = email.Body.Content + newEmail.Body.RawBody = email.Body.Content + + newEmail.Header.Subject = email.Subject + newEmail.Header.From = email.From.Emailaddress.Address + for _, to := range email.Torecipients { + newEmail.Header.To = append(newEmail.Header.To, to.Emailaddress.Address) + } + newEmail.Header.Date = email.Receiveddatetime.String() + + newEmail.MessageID = email.ID + + if email.Hasattachments { + log.Printf("SHOULD HANDLE ATTACHMENTS FOR EMAIL!") + + for _, attachment := range email.Attachments { + parsedAttachment, err := getOutlookAttachment(outlookClient, email.ID, attachment.ID) + if err != nil { + log.Printf("Failed attachment %s: %s", attachment.ID, err) + continue + } + + log.Printf("ATTACHMENT: %#v", parsedAttachment) + } + //log.Printf("%#v", attachments) + //log.Printf("%s", err) + //GET /users/{id | userPrincipalName}/events/{id}/attachments/{id} + } + emailBytes, err := json.Marshal(email) if err != nil { log.Printf("[INFO] Failed email marshaling: %s", err) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 16e1a653..7a20bec6 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -2565,6 +2565,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // Check every app action and param to see whether they exist //log.Printf("PRE ACTIONS 2") + allAuths, autherr := getAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) newActions = []Action{} for _, action := range workflow.Actions { reservedApps := []string{ @@ -2655,34 +2656,60 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // Check to see if the action is valid if curappaction.Name != action.Name { log.Printf("[ERROR] Action %s in app %s doesn't exist.", action.Name, curapp.Name) - if workflow.PreviouslySaved { - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Action %s in app %s doesn't exist"}`, action.Name, curapp.Name))) - return - } + thisError := fmt.Sprintf("%s: Action %s in app %s doesn't exist", action.Label, action.Name, action.AppName) + workflow.Errors = append(workflow.Errors, thisError) + workflow.IsValid = false + action.Errors = append(action.Errors, thisError) + action.IsValid = false + //if workflow.PreviouslySaved { + // resp.WriteHeader(401) + // resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Action %s in app %s doesn't exist"}`, action.Name, curapp.Name))) + // return + //} } // FIXME - check all parameters to see if they're valid // Includes checking required fields + selectedAuth := AppAuthenticationStorage{} + if len(action.AuthenticationId) > 0 && autherr == nil { + for _, auth := range allAuths { + if auth.Id == action.AuthenticationId { + selectedAuth = auth + break + } + } + } + newParams := []WorkflowAppActionParameter{} for _, param := range curappaction.Parameters { - found := false + paramFound := false // Handles check for parameter exists + value not empty in used fields for _, actionParam := range action.Parameters { if actionParam.Name == param.Name { - found = true + paramFound = true if actionParam.Value == "" && actionParam.Variant == "STATIC_VALUE" && actionParam.Required == true { - log.Printf("[WARNING] Appaction %s with required param '%s' is empty. Can't save.", action.Name, param.Name) - //if workflow.PreviouslySaved { - // resp.WriteHeader(401) - // resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s in app %s with required param '%s' is empty.", "node_id": "%s"}`, action.Name, action.AppName, param.Name, action.ID))) - // return - //} else { + // Validating if the field is an authentication field + if len(selectedAuth.Id) > 0 { + authFound := false + for _, field := range selectedAuth.Fields { + if field.Key == actionParam.Name { + authFound = true + //log.Printf("FOUND REQUIRED KEY %s IN AUTH", field.Key) + break + } + } - thisError := fmt.Sprintf("Missing parameter %s", param.Name) + if authFound { + newParams = append(newParams, actionParam) + continue + } + } + + log.Printf("[WARNING] Appaction %s with required param '%s' is empty. Can't save.", action.Name, param.Name) + thisError := fmt.Sprintf("%s is missing reqired parameter %s", action.Label, param.Name) action.Errors = append(action.Errors, thisError) workflow.Errors = append(workflow.Errors, thisError) action.IsValid = false @@ -2698,7 +2725,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } // Handles check for required params - if !found && param.Required { + if !paramFound && param.Required { log.Printf("Appaction %s with required param %s doesn't exist.", action.Name, param.Name) thisError := fmt.Sprintf("Parameter %s is required", param.Name) action.Errors = append(action.Errors, thisError) @@ -2719,13 +2746,10 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } } - //log.Printf("PRE SAVECHECK") if !workflow.PreviouslySaved { log.Printf("[WORKFLOW INIT] NOT PREVIOUSLY SAVED - SET ACTION AUTH!") - //AuthenticationId string `json:"authentication_id,omitempty" datastore:"authentication_id"` - allAuths, err := getAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) - if err == nil && len(workflowapps) > 0 && apperr == nil { + if autherr == nil && len(workflowapps) > 0 && apperr == nil { //log.Printf("Setting actions") actionFixing := []Action{} appsAdded := []string{} @@ -5389,7 +5413,7 @@ func getAppAuthentication(resp http.ResponseWriter, request *http.Request) { return } - // Cleanup for frontend + // Cleanup for frontend usage. User shouldn't be able to get the data. newAuth := []AppAuthenticationStorage{} for _, auth := range allAuths { newAuthField := auth diff --git a/docker-compose.yml b/docker-compose.yml index b26f3c1a..b96b2441 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: frontend: #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.63 + image: ghcr.io/frikky/shuffle-frontend:0.8.64 container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -17,7 +17,7 @@ services: - backend backend: #build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.63 + image: ghcr.io/frikky/shuffle-backend:0.8.64 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 9bb3ae21..40f9b948 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -204,8 +204,10 @@ const AngularWorkflow = (props) => { }) const [elements, setElements] = useState([]) + // No point going as fast, as the nodes aren't realtime anymore, but bulk updated. + // Set it from 2500 to 6000 to reduce overall load const { start, stop } = useInterval({ - duration: 2500, + duration: 6000, startImmediate: false, callback: () => { fetchUpdates()