From 81d3f02fff7f2b00d32d503c2e7cefe340d3a271 Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 22 Nov 2020 12:48:54 +0100 Subject: [PATCH 01/19] #141: Added file upload capability. Next is app testing --- backend/app_sdk/app_base.py | 26 +++++- backend/go-app/main.go | 170 +----------------------------------- 2 files changed, 26 insertions(+), 170 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 18a42a32..a5d99d85 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -833,6 +833,26 @@ class AppBase: print("SHOULD GET FILES BASED ON ORG %s, workflow %s and value(s) %s" % (org_id, full_execution["workflow"]["id"], value)) get_path = "/api/v1/files/%s/content?execution_id=%s" % (value, full_execution["execution_id"]) + print("PATH: %s" % get_path) + headers = { + "Content-Type": "application/json", + "Authorization": "Bearer %s" % self.authorization + } + ret = requests.get("%s%s" % (self.url, get_path), headers=headers) + if ret.status_code == 200: + return ret.text + + print("ERROR GETTING FILE: ") + print("FILE RET CONTENT: %s" % ret.text) + print("FILE RET CODE FILE: %d" % ret.status_code) + + # Sets files in the backend + def set_files(full_execution, files[]): + print("FULL EXEC: %s" % full_execution) + org_id = full_execution["workflow"]["execution_org"]["id"] + print("SHOULD GET FILES BASED ON ORG %s, workflow %s and value(s) %s" % (org_id, full_execution["workflow"]["id"], value)) + get_path = "/api/v1/files/%s/content?execution_id=%s" % (value, full_execution["execution_id"]) + print("PATH: %s" % get_path) headers = { "Content-Type": "application/json", @@ -842,8 +862,6 @@ class AppBase: print("RET CONTENT: %s" % ret.text) print("RET CODE FILE: %d" % ret.status_code) - # r.HandleFunc("/api/v1/files/{fileId}/content", handleGetFileContent).Methods("GET", "OPTIONS") - # Checks whether conditions are met, otherwise set branchcheck, tmpresult = check_branch_conditions(action, fullexecution) if not branchcheck: @@ -911,6 +929,8 @@ class AppBase: multiexecution = False multi_execution_lists = [] for parameter in action["parameters"]: + + # This code handles files. is_file = False try: if parameter["schema"]["type"] == "file": @@ -920,6 +940,8 @@ class AppBase: except KeyError as e: print("SCHEMA ERROR: %s" % e) + + check, value, is_loop = parse_params(action, fullexecution, parameter) if check: raise "Value check error: %s" % Exception(check) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index d7fe77a6..f93ef351 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -7934,172 +7934,6 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { resp.Write(respBody) } -type File struct { - Id string `json:"id" datastore:"id"` - Type string `json:"type" datastore:"type"` - CreatedAt int64 `json:"created_at" datastore:"created_at"` - UpdatedAt int64 `json:"updated_at" datastore:"updated_at"` - Description string `json:"description" datastore:"description"` - ExpiresAt string `json:"expires_at" datastore:"expires_at"` - Status string `json:"status" datastore:"status"` - Filename string `json:"filename" datastore:"filename"` - URL string `json:"url" datastore:"org"` - OrgId string `json:"org_id" datastore:"org_id"` - WorkflowId string `json:"workflow_id" datastore:"workflow_id"` - Workflows []string `json:"workflows" datastore:"workflows"` - DownloadPath string `json:"download_path" datastore:"download_path"` -} - -func handleGetFileContent(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - var fileId string - location := strings.Split(request.URL.String(), "/") - if location[1] == "api" { - if len(location) <= 4 { - log.Printf("Path too short: %d", len(location)) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - log.Printf("\n\nUser is trying to get file %s\n\n", fileId) - - // 1. Check user directly - // 2. Check workflow execution authorization - setOrgId := false - user, err := handleApiAuthentication(resp, request) - if err != nil { - // r.HandleFunc("/api/v1/files/{fileId}/content", handleGetFileContent).Methods("GET", "OPTIONS") - log.Printf("INITIAL Api authentication failed in file download: %s", err) - executionId, ok := request.URL.Query()["execution_id"] - if ok && len(executionId) > 0 { - ctx := context.Background() - workflowExecution, err := getWorkflowExecution(ctx, executionId[0]) - if err != nil { - log.Printf("Couldn't find execution ID %s", executionId[0]) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - apikey := request.Header.Get("Authorization") - if !strings.HasPrefix(apikey, "Bearer ") { - log.Printf("Apikey doesn't start with bearer (2)") - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - apikeyCheck := strings.Split(apikey, " ") - if len(apikeyCheck) != 2 { - log.Printf("Invalid format for apikey (2)") - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - // This is annoying af and is done because of maxlength lol - newApikey := apikeyCheck[1] - if newApikey != workflowExecution.Authorization { - log.Printf("Bad apikey for execution %s. %s vs %s", executionId[0], apikey, workflowExecution.Authorization) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - log.Printf("Authorization is correct for execution %s! %s vs %s", executionId, apikey, workflowExecution.Authorization) - setOrgId = true - } else { - - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - } - - // 1. Verify if the user has access to the file: org_id and workflow - - log.Printf("Should get file %s", fileId) - ctx := context.Background() - file, err := getFile(ctx, fileId) - if err != nil { - log.Printf("File %s not found: %s", fileId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - // This is a workaround for file grabs from an app - if setOrgId == true { - user.ActiveOrg.Id = file.OrgId - } - - found := false - if file.OrgId == user.ActiveOrg.Id { - found = true - } else { - for _, item := range user.Orgs { - if item == file.OrgId { - found = true - break - } - } - } - - if !found { - log.Printf("User %s doesn't have access to %s", user.Username, fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - // Fixme: More auth: org and workflow! - downloadPath := file.DownloadPath - log.Printf("Downloadpath: %s", downloadPath) - Openfile, err := os.Open(downloadPath) - defer Openfile.Close() //Close after function return - if err != nil { - //File not found, send 404 - http.Error(resp, "File not found.", 404) - return - } - - //File is found, create and send the correct headers - //Get the Content-Type of the file - //Create a buffer to store the header of the file in - FileHeader := make([]byte, 512) - //Copy the headers into the FileHeader buffer - Openfile.Read(FileHeader) - //Get content type of file - FileContentType := http.DetectContentType(FileHeader) - - //Get the file size - FileStat, _ := Openfile.Stat() //Get info from file - FileSize := strconv.FormatInt(FileStat.Size(), 10) //Get file size as a string - - //Send the headers - resp.Header().Set("Content-Disposition", "attachment; filename="+fileId) - resp.Header().Set("Content-Type", FileContentType) - resp.Header().Set("Content-Length", FileSize) - - //Send the file - //We read 512 bytes from the file already, so we reset the offset back to 0 - Openfile.Seek(0, 0) - io.Copy(resp, Openfile) //'Copy' the file to the client - return - - //log.Printf("Should download file %s", downloadPath) - //resp.WriteHeader(200) - //resp.Write([]byte("OK")) -} - func initHandlers() { var err error ctx := context.Background() @@ -8233,8 +8067,8 @@ func initHandlers() { // https://developer.box.com/reference/get-files-id-content/ // 1. Creating the "get file" option. Make it possible to run this in the frontend. r.HandleFunc("/api/v1/files/{fileId}/content", handleGetFileContent).Methods("GET", "OPTIONS") - //r.HandleFunc("/api/v1/files/create", handleGetFile).Methods("POST", "OPTIONS") - //r.HandleFunc("/api/v1/files/upload", handleGetFile).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/files/create", handleCreateFile).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/files/{fileId}/upload", handleUploadFile).Methods("POST", "OPTIONS") //r.HandleFunc("/api/v1/files/{fileId}", handleGetFile).Methods("GET", "OPTIONS") //r.HandleFunc("/api/v1/files/{fileId}", handleGetFile).Methods("DELETE", "OPTIONS") From 185c0a57a8670113725ad4906c90700334c8808b Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 22 Nov 2020 18:15:53 +0100 Subject: [PATCH 02/19] #141: Made POC 99% there. Issue is multipart bug in python --- backend/app_sdk/app_base.py | 129 +++++++++++++++++++------ backend/go-app/walkoff.go | 2 +- frontend/src/views/AngularWorkflow.jsx | 46 ++++++++- 3 files changed, 145 insertions(+), 32 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index a5d99d85..4d9579b1 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -828,7 +828,6 @@ class AppBase: # - How can you download / stream a file? # - Can you decide if you want a stream or the files directly? def get_files(full_execution, value): - print("FULL EXEC: %s" % full_execution) org_id = full_execution["workflow"]["execution_org"]["id"] print("SHOULD GET FILES BASED ON ORG %s, workflow %s and value(s) %s" % (org_id, full_execution["workflow"]["id"], value)) get_path = "/api/v1/files/%s/content?execution_id=%s" % (value, full_execution["execution_id"]) @@ -845,22 +844,72 @@ class AppBase: print("ERROR GETTING FILE: ") print("FILE RET CONTENT: %s" % ret.text) print("FILE RET CODE FILE: %d" % ret.status_code) + return "Error getting file(s). Status code %d" % ret.status_code # Sets files in the backend - def set_files(full_execution, files[]): - print("FULL EXEC: %s" % full_execution) + def set_files(full_execution, infiles): + workflow_id = full_execution["workflow"]["id"] org_id = full_execution["workflow"]["execution_org"]["id"] - print("SHOULD GET FILES BASED ON ORG %s, workflow %s and value(s) %s" % (org_id, full_execution["workflow"]["id"], value)) - get_path = "/api/v1/files/%s/content?execution_id=%s" % (value, full_execution["execution_id"]) - - print("PATH: %s" % get_path) headers = { "Content-Type": "application/json", "Authorization": "Bearer %s" % self.authorization } - ret = requests.get("%s%s" % (self.url, get_path), headers=headers) - print("RET CONTENT: %s" % ret.text) - print("RET CODE FILE: %d" % ret.status_code) + + create_path = "/api/v1/files/create?execution_id=%s" % full_execution["execution_id"] + file_ids = [] + for curfile in infiles: + filename = "unspecified" + data = { + "filename": filename, + "workflow_id": workflow_id, + "org_id": org_id, + } + + try: + data["filename"] = curfile["filename"] + filename = curfile["filename"] + except KeyError as e: + print("KeyError in file setup: %s" % e) + pass + + ret = requests.post("%s%s" % (self.url, create_path), headers=headers, json=data) + print("Ret CREATE: %s" % ret.text) + cur_id = "" + if ret.status_code == 200: + print("RET: %s" % ret.text) + ret_json = ret.json() + if not ret_json["success"]: + print("Not success in file upload creation.") + continue + + print("Should handle ID %s" % ret_json["id"]) + file_ids.append(ret_json["id"]) + cur_id = ret_json["id"] + else: + print("Bad status code: %d" % ret.status_code) + continue + + if len(cur_id) == 0: + print("No file ID specified from backend") + continue + + new_headers = { + "Content-Type": "multipart/form-data; charset=utf-8; boundary=\"test boundary Shuffle\"", + "Authorization": "Bearer %s" % self.authorization, + } + + upload_path = "/api/v1/files/%s/upload?execution_id=%s" % (cur_id, full_execution["execution_id"]) + print("Create path: %s" % create_path) + #files={"shuffle_file": open(filename,'rb')} + files={"shuffle_file": (filename, curfile["data"])} + #open(filename,'rb')} + + ret = requests.post("%s%s" % (self.url, upload_path), files=files, headers=new_headers) + print("Ret UPLOAD: %s" % ret.text) + print("Ret2 UPLOAD: %d" % ret.status_code) + + print("IDS TO RETURN: %s" % file_ids) + return file_ids # Checks whether conditions are met, otherwise set branchcheck, tmpresult = check_branch_conditions(action, fullexecution) @@ -929,19 +978,6 @@ class AppBase: multiexecution = False multi_execution_lists = [] for parameter in action["parameters"]: - - # This code handles files. - is_file = False - try: - if parameter["schema"]["type"] == "file": - print("SHOULD HANDLE FILE. Get based on value %s" % parameter["value"]) - get_files(fullexecution, parameter["value"]) - is_file = True - except KeyError as e: - print("SCHEMA ERROR: %s" % e) - - - check, value, is_loop = parse_params(action, fullexecution, parameter) if check: raise "Value check error: %s" % Exception(check) @@ -1057,6 +1093,18 @@ class AppBase: params[parameter["name"]] = value multi_parameters[parameter["name"]] = value + # This code handles files. + try: + if parameter["schema"]["type"] == "file": + print("SHOULD HANDLE FILE. Get based on value %s" % parameter["value"]) + file_value = get_files(fullexecution, value) + print("FILE VALUE: %s" % file_value) + + params[parameter["name"]] = file_value + multi_parameters[parameter["name"]] = file_value + except KeyError as e: + print("SCHEMA ERROR IN FILE HANDLING: %s" % e) + # Fix lists here print("CHECKING multi execution list!") if len(multi_execution_lists) > 0: @@ -1080,14 +1128,37 @@ class AppBase: print("New multi execution length: %d\n" % tmplength) - # FIXME - this is horrible, but works for now - #for i in range(calltimes): if not multiexecution: print("APP_SDK DONE: Starting NORMAL execution of function") - print("Running with params %s" % params) + print("Running with params (0): %s" % params) newres = await func(**params) - print("Return from execution: %s" % newres) - if isinstance(newres, str): + print("Returned from execution.") + if isinstance(newres, tuple): + print("Handling return as tuple") + # Handles files. + filedata = "" + file_ids = [] + print("TUPLE: %s" % newres[1]) + if isinstance(newres[1], list): + print("HANDLING LIST FROM RET") + file_ids = set_files(fullexecution, newres[1]) + elif isinstance(newres[1], object): + print("Handling JSON from ret") + file_ids = set_files(fullexecution, [newres[1]]) + elif isinstance(newres[1], str): + print("Handling STRING from ret") + file_ids = set_files(fullexecution, [newres[1]]) + else: + print("NO FILES TO HANDLE") + + tmp_result = { + "result": newres[0], + "file_ids": file_ids + } + + result = json.dumps(tmp_result) + elif isinstance(newres, str): + print("Handling return as string") result += newres else: try: @@ -1165,7 +1236,7 @@ class AppBase: print("KeyError: %s" % e) baseparams[key] = "KeyError: %s" % e - print("Running with params %s" % baseparams) + print("Running with params %s (1)" % baseparams) ret = await func(**baseparams) print("Return from execution: %s" % ret) if isinstance(ret, dict) or isinstance(ret, list): diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 33c438f7..262981fd 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -2836,7 +2836,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf var allEnvs []Environment if len(workflowExecution.ExecutionOrg) > 0 { - log.Printf("Executing ORG: %s", workflowExecution.ExecutionOrg) + log.Printf("[INFO] Executing ORG: %s", workflowExecution.ExecutionOrg) allEnvironments, err := getEnvironments(ctx, workflowExecution.ExecutionOrg) if err != nil { diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index f7bf156d..485ff4cb 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -2838,13 +2838,55 @@ const AngularWorkflow = (props) => { console.log(selectedActionParameters[count]) if (selectedActionParameters[count].schema !== undefined && selectedActionParameters[count].schema !== null && selectedActionParameters[count].schema.type === "file") { - const fileId = "6daabec1-892b-469c-b603-c902e47223a9" - datafield = `SHOW FILES FROM OTHER NODES? Filename: ${selectedActionParameters[count].value}` + datafield = + + + { + setMenuPosition({ + top: event.pageY, + left: event.pageX, + }) + setShowDropdownNumber(count) + setShowDropdown(true) + setShowAutocomplete(true) + }}/> + + + ) + }} + fullWidth + multiline={multiline} + rows="5" + color="primary" + defaultValue={data.value} + type={"text"} + placeholder={"The file ID to get"} + onChange={(event) => { + changeActionParameter(event, count) + }} + onBlur={(event) => { + }} + /> + //const fileId = "6daabec1-892b-469c-b603-c902e47223a9" + //datafield = `SHOW FILES FROM OTHER NODES? Filename: ${selectedActionParameters[count].value}` + /* if (selectedActionParameters[count].value != fileId) { changeActionParameter(fileId, count) setUpdate(Math.random()) } + */ } else if (selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0) { if (selectedActionParameters[count].value === "" && selectedActionParameters[count].required) { // Rofl, dirty workaround :) From e2e411d40c736c4e56ac4bd195f6bc4faebd19d8 Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 23 Nov 2020 13:36:26 +0100 Subject: [PATCH 03/19] #141: Basic file options added --- .env | 2 ++ backend/Dockerfile | 1 + backend/app_sdk/app_base.py | 1 - backend/app_sdk/build.sh | 2 +- backend/go-app/main.go | 24 +----------------------- backend/go-app/walkoff.go | 2 +- docker-compose.yml | 8 +++++--- 7 files changed, 11 insertions(+), 29 deletions(-) diff --git a/.env b/.env index 93b05920..a43c6669 100644 --- a/.env +++ b/.env @@ -19,7 +19,9 @@ SHUFFLE_DEFAULT_PASSWORD= SHUFFLE_DEFAULT_APIKEY= # Local location of your app directory. Can't use ~/ +# Files will get better at some point. Right now: local saving. SHUFFLE_APP_HOTLOAD_LOCATION=./shuffle-apps +SHUFFLE_FILE_LOCATION=./shuffle-files # Other configs BACKEND_HOSTNAME=shuffle-backend diff --git a/backend/Dockerfile b/backend/Dockerfile index 21422785..1856d84f 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -8,6 +8,7 @@ ADD ./go-app/main.go /app ADD ./go-app/walkoff.go /app ADD ./go-app/docker.go /app ADD ./go-app/codegen.go /app +ADD ./go-app/files.go /app ADD ./go-app/go.mod /app diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 4d9579b1..ca5039cb 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -894,7 +894,6 @@ class AppBase: continue new_headers = { - "Content-Type": "multipart/form-data; charset=utf-8; boundary=\"test boundary Shuffle\"", "Authorization": "Bearer %s" % self.authorization, } diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 6c28cc03..4dbf2574 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,6 +1,6 @@ #!/bin/bash NAME=shuffle-app_sdk -VERSION=0.8.0 +VERSION=0.8.1 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/go-app/main.go b/backend/go-app/main.go index f93ef351..796e9ea7 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -2577,28 +2577,6 @@ func getSession(ctx context.Context, thissession string) (*session, error) { return curUser, nil } -// ListBooks returns a list of books, ordered by title. -func getFile(ctx context.Context, id string) (*File, error) { - key := datastore.NameKey("Files", id, nil) - curFile := &File{} - if err := dbclient.Get(ctx, key, curFile); err != nil { - return &File{}, err - } - - return curFile, nil -} - -func setFile(ctx context.Context, file File) error { - // clear session_token and API_token for user - k := datastore.NameKey("Files", file.Id, nil) - if _, err := dbclient.Put(ctx, k, &file); err != nil { - log.Println(err) - return err - } - - return nil -} - // ListBooks returns a list of books, ordered by title. func getOrg(ctx context.Context, id string) (*Org, error) { key := datastore.NameKey("Organizations", id, nil) @@ -8067,10 +8045,10 @@ func initHandlers() { // https://developer.box.com/reference/get-files-id-content/ // 1. Creating the "get file" option. Make it possible to run this in the frontend. r.HandleFunc("/api/v1/files/{fileId}/content", handleGetFileContent).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/files/{fileId}", handleDeleteFile).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/files/create", handleCreateFile).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/files/{fileId}/upload", handleUploadFile).Methods("POST", "OPTIONS") //r.HandleFunc("/api/v1/files/{fileId}", handleGetFile).Methods("GET", "OPTIONS") - //r.HandleFunc("/api/v1/files/{fileId}", handleGetFile).Methods("DELETE", "OPTIONS") http.Handle("/", r) } diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 262981fd..011db663 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -1830,7 +1830,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { for _, action := range workflow.Actions { allNodes = append(allNodes, action.ID) - if len(action.Errors) > 0 { + if len(action.Errors) > 0 || !action.IsValid { action.IsValid = true action.Errors = []string{} } diff --git a/docker-compose.yml b/docker-compose.yml index 05d3e546..855a06b5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,8 @@ version: '3' services: frontend: - #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.0 + build: ./frontend + image: ghcr.io/frikky/shuffle-frontend:0.8.1 container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -17,7 +17,7 @@ services: - backend backend: #build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.0 + image: ghcr.io/frikky/shuffle-backend:0.8.1 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: @@ -28,9 +28,11 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps + - ${SHUFFLE_FILE_LOCATION}:/shuffle-files environment: - DATASTORE_EMULATOR_HOST=shuffle-database:8000 - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps + - SHUFFLE_FILE_LOCATION=/shuffle-files - ORG_ID=${ORG_ID} - SHUFFLE_APP_DOWNLOAD_LOCATION=${SHUFFLE_APP_DOWNLOAD_LOCATION} - SHUFFLE_DEFAULT_USERNAME=${SHUFFLE_DEFAULT_USERNAME} From 1e583611506d8552655c44f8ab33c6c33cfcbc40 Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 23 Nov 2020 13:36:50 +0100 Subject: [PATCH 04/19] Added files.go to handle file endpoints --- backend/go-app/files.go | 632 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 632 insertions(+) create mode 100644 backend/go-app/files.go diff --git a/backend/go-app/files.go b/backend/go-app/files.go new file mode 100644 index 00000000..16e238a7 --- /dev/null +++ b/backend/go-app/files.go @@ -0,0 +1,632 @@ +package main + +/* + Handles files within Workflows. +*/ + +import ( + //"bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "strconv" + "strings" + "time" + + "cloud.google.com/go/datastore" + "github.com/satori/go.uuid" +) + +type File struct { + Id string `json:"id" datastore:"id"` + Type string `json:"type" datastore:"type"` + CreatedAt int64 `json:"created_at" datastore:"created_at"` + UpdatedAt int64 `json:"updated_at" datastore:"updated_at"` + Description string `json:"description" datastore:"description"` + ExpiresAt string `json:"expires_at" datastore:"expires_at"` + Status string `json:"status" datastore:"status"` + Filename string `json:"filename" datastore:"filename"` + URL string `json:"url" datastore:"org"` + OrgId string `json:"org_id" datastore:"org_id"` + WorkflowId string `json:"workflow_id" datastore:"workflow_id"` + Workflows []string `json:"workflows" datastore:"workflows"` + DownloadPath string `json:"download_path" datastore:"download_path"` +} + +var basepath = os.Getenv("SHUFFLE_FILE_LOCATION") + +func fileAuthentication(request *http.Request) (string, error) { + executionId, ok := request.URL.Query()["execution_id"] + if ok && len(executionId) > 0 { + ctx := context.Background() + workflowExecution, err := getWorkflowExecution(ctx, executionId[0]) + if err != nil { + log.Printf("[ERROR] Couldn't find execution ID %s", executionId[0]) + return "", err + } + + apikey := request.Header.Get("Authorization") + if !strings.HasPrefix(apikey, "Bearer ") { + log.Printf("[ERROR} Apikey doesn't start with bearer (2)") + return "", errors.New("No auth key found") + } + + apikeyCheck := strings.Split(apikey, " ") + if len(apikeyCheck) != 2 { + log.Printf("[ERROR] Invalid format for apikey (2)") + return "", errors.New("No space in authkey") + } + + // This is annoying af and is done because of maxlength lol + newApikey := apikeyCheck[1] + if newApikey != workflowExecution.Authorization { + log.Printf("[ERROR] Bad apikey for execution %s. %s vs %s", executionId[0], apikey, workflowExecution.Authorization) + return "", errors.New("Bad authorization key") + } + + log.Printf("[INFO] Authorization is correct for execution %s! %s vs %s. Setting Org", executionId, apikey, workflowExecution.Authorization) + if len(workflowExecution.ExecutionOrg) > 0 { + return workflowExecution.ExecutionOrg, nil + } else if len(workflowExecution.Workflow.ExecutingOrg.Id) > 0 { + return workflowExecution.ExecutionOrg, nil + } else { + log.Printf("[ERROR] Couldn't find org for workflow execution, but auth was correct.") + } + } + + return "", errors.New("No execution id specified") +} + +// https://golangcode.com/check-if-a-file-exists/ +func fileExists(filename string) bool { + info, err := os.Stat(filename) + if os.IsNotExist(err) { + return false + } + return !info.IsDir() +} + +func handleDeleteFile(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + var fileId string + location := strings.Split(request.URL.String(), "/") + if location[1] == "api" { + if len(location) <= 4 { + log.Printf("[INFO] Path too short: %d", len(location)) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + fileId = location[4] + } + + if strings.Contains(fileId, "?") { + fileId = strings.Split(fileId, "?")[0] + } + + log.Printf("\n\n[INFO] User is trying to delete file %s\n\n", fileId) + + // 1. Check user directly + // 2. Check workflow execution authorization + user, err := handleApiAuthentication(resp, request) + if err != nil { + log.Printf("[INFO] INITIAL Api authentication failed in file deletion: %s", err) + + orgId, err := fileAuthentication(request) + if err != nil { + log.Printf("[ERROR] Bad file authentication in get: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + user.ActiveOrg.Id = orgId + user.Username = "Execution File API" + } + + // 1. Verify if the user has access to the file: org_id and workflow + log.Printf("[INFO] Should DELETE file %s if user has access", fileId) + ctx := context.Background() + file, err := getFile(ctx, fileId) + if err != nil { + log.Printf("[INFO] File %s not found: %s", fileId, err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + found := false + if file.OrgId == user.ActiveOrg.Id { + found = true + } else { + for _, item := range user.Orgs { + if item == file.OrgId { + found = true + break + } + } + } + + if !found { + log.Printf("[INFO] User %s doesn't have access to %s", user.Username, fileId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if file.Status == "deleted" { + log.Printf("[INFO] File with ID %s is already deleted.", fileId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if fileExists(file.DownloadPath) { + err = os.Remove(file.DownloadPath) + if err != nil { + log.Printf("[ERROR] Failed deleting file locally: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed deleting filein path %s"}`, file.DownloadPath))) + return + } + + log.Printf("[INFO] Deleted file %s locally. Next is database.", file.DownloadPath) + } else { + log.Printf("[ERROR] File doesn't exist. Can't delete. Should maybe delete file anyway?") + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "File in location %s doesn't exist"}`, file.DownloadPath))) + return + } + + file.Status = "deleted" + err = setFile(ctx, *file) + if err != nil { + log.Printf("[ERROR] Failed setting file to deleted") + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false, "reason": "Failed setting file to deleted"}`)) + return + } + + /* + //Actually delete it? + err = DeleteKey(ctx, "files", fileId) + if err != nil { + log.Printf("Failed deleting file with ID %s: %s", fileId, err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + */ + + log.Printf("[INFO] Successfully deleted file %s", fileId) + resp.WriteHeader(200) + resp.Write([]byte(`{"success": true}`)) +} + +func handleGetFileContent(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + var fileId string + location := strings.Split(request.URL.String(), "/") + if location[1] == "api" { + if len(location) <= 4 { + log.Printf("Path too short: %d", len(location)) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + fileId = location[4] + } + + log.Printf("\n\nUser is trying to get file %s\n\n", fileId) + + // 1. Check user directly + // 2. Check workflow execution authorization + user, err := handleApiAuthentication(resp, request) + if err != nil { + log.Printf("INITIAL Api authentication failed in file download: %s", err) + + orgId, err := fileAuthentication(request) + if err != nil { + log.Printf("Bad file authentication in get: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + user.ActiveOrg.Id = orgId + user.Username = "Execution File API" + /* + } else { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + */ + } + + // 1. Verify if the user has access to the file: org_id and workflow + log.Printf("Should get file %s", fileId) + ctx := context.Background() + file, err := getFile(ctx, fileId) + if err != nil { + log.Printf("File %s not found: %s", fileId, err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + found := false + if file.OrgId == user.ActiveOrg.Id { + found = true + } else { + for _, item := range user.Orgs { + if item == file.OrgId { + found = true + break + } + } + } + + if !found { + log.Printf("User %s doesn't have access to %s", user.Username, fileId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if file.Status != "active" { + log.Printf("File status isn't active. Can't continue.") + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "The file isn't ready to be downloaded yet. Status required: active"}`)) + return + } + + // Fixme: More auth: org and workflow! + downloadPath := file.DownloadPath + log.Printf("Downloadpath: %s", downloadPath) + Openfile, err := os.Open(downloadPath) + defer Openfile.Close() //Close after function return + if err != nil { + //File not found, send 404 + http.Error(resp, "File not found.", 404) + return + } + + //File is found, create and send the correct headers + //Get the Content-Type of the file + //Create a buffer to store the header of the file in + FileHeader := make([]byte, 512) + //Copy the headers into the FileHeader buffer + Openfile.Read(FileHeader) + //Get content type of file + FileContentType := http.DetectContentType(FileHeader) + + //Get the file size + FileStat, _ := Openfile.Stat() //Get info from file + FileSize := strconv.FormatInt(FileStat.Size(), 10) //Get file size as a string + + //Send the headers + resp.Header().Set("Content-Disposition", "attachment; filename="+fileId) + resp.Header().Set("Content-Type", FileContentType) + resp.Header().Set("Content-Length", FileSize) + + //Send the file + //We read 512 bytes from the file already, so we reset the offset back to 0 + Openfile.Seek(0, 0) + io.Copy(resp, Openfile) //'Copy' the file to the client + return + + //log.Printf("Should download file %s", downloadPath) +} +func handleUploadFile(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + var fileId string + location := strings.Split(request.URL.String(), "/") + if location[1] == "api" { + if len(location) <= 4 { + log.Printf("Path too short: %d", len(location)) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + fileId = location[4] + } + + // 1. Check user directly + // 2. Check workflow execution authorization + user, err := handleApiAuthentication(resp, request) + if err != nil { + log.Printf("INITIAL Api authentication failed in file upload: %s", err) + + orgId, err := fileAuthentication(request) + if err != nil { + log.Printf("Bad file authentication in create file: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + user.ActiveOrg.Id = orgId + user.Username = "Execution File API" + } + + log.Printf("[INFO] Should UPLOAD file %s if user has access", fileId) + ctx := context.Background() + file, err := getFile(ctx, fileId) + if err != nil { + log.Printf("File %s not found: %s", fileId, err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + found := false + if file.OrgId == user.ActiveOrg.Id { + found = true + } else { + for _, item := range user.Orgs { + if item == file.OrgId { + found = true + break + } + } + } + + if !found { + log.Printf("User %s doesn't have access to %s", user.Username, fileId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + log.Printf("[INFO] STATUS: %s", file.Status) + if file.Status != "created" { + log.Printf("File status isn't created. Can't upload.") + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "This file already has data."}`)) + return + } + + request.ParseMultipartForm(32 << 20) + //var buf bytes.Buffer + parsedFile, _, err := request.FormFile("shuffle_file") + if err != nil { + log.Printf("[ERROR] Couldn't upload file: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Failed uploading file"}`)) + return + } + defer parsedFile.Close() + + file.Status = "uploading" + err = setFile(ctx, *file) + if err != nil { + log.Printf("Failed setting file to uploading") + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false, "reason": "Failed setting file to uploading"}`)) + return + } + + //io.Copy(&buf, parsedFile) + //contents := buf.String() + //buf.Reset() + f, err := os.OpenFile(file.DownloadPath, os.O_WRONLY|os.O_CREATE, os.ModePerm) + if err != nil { + // Rolling back file + file.Status = "created" + setFile(ctx, *file) + + log.Printf("Failed creating file: %s", err) + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false}`)) + return + } + + defer f.Close() + io.Copy(f, parsedFile) + + // FIXME: Set this one to 200 anyway? Can't download file then tho.. + file.Status = "active" + err = setFile(ctx, *file) + if err != nil { + log.Printf("[ERROR] Failed setting file back to active") + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false, "reason": "Failed setting file to active"}`)) + return + } + + log.Printf("[INFO] Successfully uploaded file ID %s", file.Id) + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) +} + +func handleCreateFile(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + // 1. Check user directly + // 2. Check workflow execution authorization + user, err := handleApiAuthentication(resp, request) + if err != nil { + log.Printf("[INFO] INITIAL Api authentication failed in file creation: %s", err) + + orgId, err := fileAuthentication(request) + if err != nil { + log.Printf("[ERROR] Bad file authentication in create file: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + user.ActiveOrg.Id = orgId + user.Username = "Execution File API" + } + + body, err := ioutil.ReadAll(request.Body) + if err != nil { + log.Println("Failed reading body") + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to read data"}`))) + return + } + + type FileStructure struct { + Filename string `json:"filename"` + OrgId string `json:"org_id"` + WorkflowId string `json:"workflow_id"` + } + + var curfile FileStructure + err = json.Unmarshal(body, &curfile) + if err != nil { + log.Printf("[ERROR] Failed unmarshaling: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to unmarshal data"}`))) + return + } + + // Loads of validation below + if len(curfile.Filename) == 0 || len(curfile.OrgId) == 0 || len(curfile.WorkflowId) == 0 { + log.Printf("[ERROR] Missing field during upload.") + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Missing field. Required: filename, org_id, workflow_id"}`))) + return + } + + ctx := context.Background() + if user.ActiveOrg.Id != curfile.OrgId { + log.Printf("[ERROR] User can't access org %s", curfile.OrgId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Error with organization"}`)) + return + } + + // Try to get the org and workflow in case they don't exist + workflow, err := getWorkflow(ctx, curfile.WorkflowId) + if err != nil { + log.Printf("[ERROR] Workflow %s doesn't exist.", curfile.WorkflowId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`)) + return + } + + _, err = getOrg(ctx, curfile.OrgId) + if err != nil { + log.Printf("[ERROR] Org %s doesn't exist.", curfile.OrgId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`)) + return + } + + if workflow.ExecutingOrg.Id != curfile.OrgId { + found := false + for _, curorg := range workflow.Org { + if curorg.Id == curfile.OrgId { + found = true + break + } + } + + if !found { + log.Printf("[ERROR] Org %s doesn't have access to %s.", curfile.OrgId, curfile.WorkflowId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`)) + return + } + } + + if strings.Contains(curfile.Filename, "/") || strings.Contains(curfile.Filename, `"`) || strings.Contains(curfile.Filename, "..") { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Invalid characters in filename"}`)) + return + } + + // 1. Create the file object. + if len(basepath) == 0 { + basepath = "shuffle-files" + } + folderPath := fmt.Sprintf("%s/%s/%s", basepath, curfile.OrgId, curfile.WorkflowId) + + // Try to make the full file location + err = os.MkdirAll(folderPath, os.ModePerm) + if err != nil { + log.Printf("[ERROR] Writing issue for file location creation: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Failed creating upload location"}`)) + return + } + + filename := curfile.Filename + fileId := uuid.NewV4().String() + downloadPath := fmt.Sprintf("%s/%s", folderPath, fileId) + + timeNow := time.Now().Unix() + newFile := File{ + Id: fileId, + CreatedAt: timeNow, + UpdatedAt: timeNow, + Description: "", + Status: "created", + Filename: filename, + OrgId: curfile.OrgId, + WorkflowId: curfile.WorkflowId, + DownloadPath: downloadPath, + } + + err = setFile(ctx, newFile) + if err != nil { + log.Printf("[ERROR] Failed setting file: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Failed setting file reference"}`)) + return + } else { + log.Printf("[INFO] Created file %s", newFile.DownloadPath) + } + + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s"}`, fileId))) +} + +func getFile(ctx context.Context, id string) (*File, error) { + key := datastore.NameKey("Files", id, nil) + curFile := &File{} + if err := dbclient.Get(ctx, key, curFile); err != nil { + return &File{}, err + } + + return curFile, nil +} + +func setFile(ctx context.Context, file File) error { + // clear session_token and API_token for user + k := datastore.NameKey("Files", file.Id, nil) + if _, err := dbclient.Put(ctx, k, &file); err != nil { + log.Println(err) + return err + } + + return nil +} From acc6378dc23dc890f829b83f72e963dbf00ae9ed Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 23 Nov 2020 15:50:31 +0100 Subject: [PATCH 05/19] #141: Added file meta to file uploads --- backend/app_sdk/app_base.py | 39 +++++++++--- backend/go-app/files.go | 85 +++++++++++++++++++++++++ backend/go-app/main.go | 2 +- docker-compose.yml | 2 +- frontend/src/views/AngularWorkflow.jsx | 88 +++++++++++++++++++++++++- 5 files changed, 202 insertions(+), 14 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index ca5039cb..7663a299 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -830,21 +830,40 @@ class AppBase: def get_files(full_execution, value): org_id = full_execution["workflow"]["execution_org"]["id"] print("SHOULD GET FILES BASED ON ORG %s, workflow %s and value(s) %s" % (org_id, full_execution["workflow"]["id"], value)) - get_path = "/api/v1/files/%s/content?execution_id=%s" % (value, full_execution["execution_id"]) - print("PATH: %s" % get_path) + get_path = "/api/v1/files/%s?execution_id=%s" % (value, full_execution["execution_id"]) headers = { "Content-Type": "application/json", "Authorization": "Bearer %s" % self.authorization } - ret = requests.get("%s%s" % (self.url, get_path), headers=headers) - if ret.status_code == 200: - return ret.text - print("ERROR GETTING FILE: ") - print("FILE RET CONTENT: %s" % ret.text) - print("FILE RET CODE FILE: %d" % ret.status_code) - return "Error getting file(s). Status code %d" % ret.status_code + ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers) + print("RET1: %s" % ret1.text) + if ret1.status_code != 200: + return { + "filename": "", + "data": "", + "success": False, + } + + content_path = "/api/v1/files/%s/content?execution_id=%s" % (value, full_execution["execution_id"]) + ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers) + print("Ret2: %s" % ret2.text) + if ret2.status_code == 200: + tmpdata = ret1.json() + returndata = { + "success": True, + "filename": tmpdata["filename"], + "data": ret2.text, + } + + return returndata + + return { + "success": False, + "filename": "", + "data": "", + } # Sets files in the backend def set_files(full_execution, infiles): @@ -1094,7 +1113,7 @@ class AppBase: # This code handles files. try: - if parameter["schema"]["type"] == "file": + if parameter["schema"]["type"] == "file" and len(value) > 0: print("SHOULD HANDLE FILE. Get based on value %s" % parameter["value"]) file_value = get_files(fullexecution, value) print("FILE VALUE: %s" % file_value) diff --git a/backend/go-app/files.go b/backend/go-app/files.go index 16e238a7..d7ab890f 100644 --- a/backend/go-app/files.go +++ b/backend/go-app/files.go @@ -92,6 +92,91 @@ func fileExists(filename string) bool { return !info.IsDir() } +func handleGetFileMeta(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + var fileId string + location := strings.Split(request.URL.String(), "/") + if location[1] == "api" { + if len(location) <= 4 { + log.Printf("[INFO] Path too short: %d", len(location)) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + fileId = location[4] + } + + if strings.Contains(fileId, "?") { + fileId = strings.Split(fileId, "?")[0] + } + + log.Printf("\n\n[INFO] User is trying to delete file %s\n\n", fileId) + + // 1. Check user directly + // 2. Check workflow execution authorization + user, err := handleApiAuthentication(resp, request) + if err != nil { + log.Printf("[INFO] INITIAL Api authentication failed in file deletion: %s", err) + + orgId, err := fileAuthentication(request) + if err != nil { + log.Printf("[ERROR] Bad file authentication in get: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + user.ActiveOrg.Id = orgId + user.Username = "Execution File API" + } + + // 1. Verify if the user has access to the file: org_id and workflow + log.Printf("[INFO] Should DELETE file %s if user has access", fileId) + ctx := context.Background() + file, err := getFile(ctx, fileId) + if err != nil { + log.Printf("[INFO] File %s not found: %s", fileId, err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + found := false + if file.OrgId == user.ActiveOrg.Id { + found = true + } else { + for _, item := range user.Orgs { + if item == file.OrgId { + found = true + break + } + } + } + + if !found { + log.Printf("[INFO] User %s doesn't have access to %s", user.Username, fileId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + newBody, err := json.Marshal(file) + if err != nil { + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false, "reason": "Failed to marshal filedata"}`)) + return + } + + log.Printf("[INFO] Successfully deleted file %s", fileId) + resp.WriteHeader(200) + resp.Write([]byte(newBody)) +} + func handleDeleteFile(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 796e9ea7..07d6e047 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -8048,7 +8048,7 @@ func initHandlers() { r.HandleFunc("/api/v1/files/{fileId}", handleDeleteFile).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/files/create", handleCreateFile).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/files/{fileId}/upload", handleUploadFile).Methods("POST", "OPTIONS") - //r.HandleFunc("/api/v1/files/{fileId}", handleGetFile).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/files/{fileId}", handleGetFileMeta).Methods("GET", "OPTIONS") http.Handle("/", r) } diff --git a/docker-compose.yml b/docker-compose.yml index 855a06b5..11a7fe29 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: frontend: build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.1 + image: ghcr.io/frikky/shuffle-frontend:0.8.0 container_name: shuffle-frontend hostname: shuffle-frontend ports: diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 485ff4cb..e2c57f3e 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -899,6 +899,41 @@ const AngularWorkflow = (props) => { const onUnselect = (event) => { console.time("UNSELECT") + // Attempt at rewrite of name in other actions in following nodes. + // Should probably be done in the onBlur for the textfield instead + /* + if (event.target.data().type === "ACTION") { + const nodeaction = event.target.data() + const curaction = workflow.actions.find(a => a.id === nodeaction.id) + console.log("workflowaction: ", curaction) + console.log("nodeaction: ", nodeaction) + if (nodeaction.label !== curaction.label) { + console.log("BEACH!") + + var params = [] + const fixedName = "$"+curaction.label.toLowerCase().replace(" ", "_") + for (var actionkey in workflow.actions) { + if (workflow.actions[actionkey].id === curaction.id) { + continue + } + + for (var paramkey in workflow.actions[actionkey].parameters) { + const param = workflow.actions[actionkey].parameters[paramkey] + if (param.value === null || param.value === undefined || !param.value.includes("$")) { + continue + } + + const innername = param.value.toLowerCase().replace(" ", "_") + if (innername.includes(fixedName)) { + //workflow.actions[actionkey].parameters[paramkey].replace( + //console.log("FOUND!: ", innername) + } + } + } + } + } + */ + // FIXME - check if they have value before overriding like this for no reason. // Would save a lot of time (400~ ms -> 30ms) //console.log("ACTION: ", selectedAction) @@ -1009,8 +1044,29 @@ const AngularWorkflow = (props) => { } setSelectedActionName(curaction.name) - setSelectedAction(curaction) + + /* + var params = [] + const fixedName = "$"+curaction.label.toLowerCase().replace(" ", "_") + for (var actionkey in workflow.actions) { + if (workflow.actions[actionkey].id === curaction.id) { + continue + } + + for (var paramkey in workflow.actions[actionkey].parameters) { + const param = workflow.actions[actionkey].parameters[paramkey] + if (param.value === null || param.value === undefined || !param.value.includes("$")) { + continue + } + + const innername = param.value.toLowerCase().replace(" ", "_") + if (innername.includes(fixedName)) { + console.log("FOUND!: ", innername) + } + } + } + */ } else if (data.type === "TRIGGER") { //console.log("Should handle trigger "+data.triggertype) //console.log(data) @@ -2419,14 +2475,43 @@ const AngularWorkflow = (props) => { // ACTION select // const selectedNameChange = (event) => { + console.log("OLDNAME: ", selectedActionName) event.target.value = event.target.value.replace("(", "") event.target.value = event.target.value.replace(")", "") event.target.value = event.target.value.replace("$", "") event.target.value = event.target.value.replace("#", "") event.target.value = event.target.value.replace(".", "") event.target.value = event.target.value.replace(",", "") + event.target.value = event.target.value.replace(" ", "_") selectedAction.label = event.target.value setSelectedAction(selectedAction) + + /* + if (nodeaction.label !== curaction.label) { + console.log("BEACH!") + + var params = [] + const fixedName = "$"+curaction.label.toLowerCase().replace(" ", "_") + for (var actionkey in workflow.actions) { + if (workflow.actions[actionkey].id === curaction.id) { + continue + } + + for (var paramkey in workflow.actions[actionkey].parameters) { + const param = workflow.actions[actionkey].parameters[paramkey] + if (param.value === null || param.value === undefined || !param.value.includes("$")) { + continue + } + + const innername = param.value.toLowerCase().replace(" ", "_") + if (innername.includes(fixedName)) { + //workflow.actions[actionkey].parameters[paramkey].replace( + //console.log("FOUND!: ", innername) + } + } + } + } + */ } const selectedTriggerChange = (event) => { @@ -2836,7 +2921,6 @@ const AngularWorkflow = (props) => { }} /> - console.log(selectedActionParameters[count]) if (selectedActionParameters[count].schema !== undefined && selectedActionParameters[count].schema !== null && selectedActionParameters[count].schema.type === "file") { datafield = Date: Wed, 25 Nov 2020 17:21:16 +0100 Subject: [PATCH 06/19] #141: Fixed more general use-case issues for files --- backend/app_sdk/app_base.py | 37 +++++++++++--- backend/go-app/files.go | 70 ++++++++++++++++++++------ backend/go-app/main.go | 60 +++++++++++----------- backend/go-app/walkoff.go | 2 +- docker-compose.yml | 6 +-- frontend/src/views/AngularWorkflow.jsx | 59 +++++++++++++++++++++- frontend/src/views/Workflows.jsx | 2 +- 7 files changed, 180 insertions(+), 56 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 7663a299..362ea503 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -148,6 +148,7 @@ class AppBase: print("") + self.full_execution = fullexecution self.logger.info("AFTER FULLEXEC stream result") # Gets the value at the parenthesis level you want @@ -478,7 +479,7 @@ class AppBase: #Actionname: Start_node - print(f"Actionname: {actionname_lower}") + print(f"\nActionname: {actionname_lower}") # 1. Find the action baseresult = "" @@ -549,21 +550,29 @@ class AppBase: except KeyError as error: print(f"KeyError in JSON: {error}") - print(f"After first trycatch") + print(f"After first trycatch. Baseresult: ", baseresult) # 2. Find the JSON data if len(baseresult) == 0: return ""+appendresult, False + print("After second return") if len(parsersplit) == 1: return str(baseresult)+str(appendresult), False baseresult = baseresult.replace("\'", "\"") + baseresult = baseresult.replace(" True,", " true,") + baseresult = baseresult.replace(" False", " false,") + + print("After third parser return - Formatted: ", baseresult) basejson = {} try: basejson = json.loads(baseresult) except json.decoder.JSONDecodeError as e: + print("Parser issue with JSON: %s" % e) return str(baseresult)+str(appendresult), False + + print("After fourth parser return as JSON") data, is_loop = recurse_json(basejson, parsersplit[1:]) parseditem = data @@ -577,6 +586,7 @@ class AppBase: print("SET DATA WRAPPER TO %s!" % parsersplit[-1]) parseditem = "${%s%s}$" % (parsersplit[-1], json.dumps(data)) + print("Before last return") return str(parseditem)+str(appendresult), is_loop # Parses parameters sent to it and returns whether it did it successfully with the values found @@ -854,8 +864,9 @@ class AppBase: returndata = { "success": True, "filename": tmpdata["filename"], - "data": ret2.text, + "data": ret2.content, } + # open('facebook.ico', 'wb').write(r.content) return returndata @@ -1093,7 +1104,21 @@ class AppBase: #replacement = parse_wrapper_start(replacement) tmpitem = tmpitem.replace(key, replacement, -1) - resultarray.append(tmpitem) + + # This code handles files. + isfile = False + try: + if parameter["schema"]["type"] == "file" and len(value) > 0: + print("SHOULD HANDLE FILE IN MULTI. Get based on value %s" % parameter["value"]) + file_value = get_files(fullexecution, tmpitem) + print("FILE VALUE FOR VAL %s: %s" % (tmpitem, file_value)) + resultarray.append(file_value) + + except KeyError as e: + print("SCHEMA ERROR IN FILE HANDLING: %s" % e) + + if not isfile: + resultarray.append(tmpitem) # With this parameter ready, add it to... a greater list of parameters. Rofl print("LENGTH OF ARR: %d" % len(resultarray)) @@ -1114,9 +1139,9 @@ class AppBase: # This code handles files. try: if parameter["schema"]["type"] == "file" and len(value) > 0: - print("SHOULD HANDLE FILE. Get based on value %s" % parameter["value"]) + print("\n SHOULD HANDLE FILE. Get based on value %s. <--- is this a valid ID?" % parameter["value"]) file_value = get_files(fullexecution, value) - print("FILE VALUE: %s" % file_value) + print("FILE VALUE: %s \n" % file_value) params[parameter["name"]] = file_value multi_parameters[parameter["name"]] = file_value diff --git a/backend/go-app/files.go b/backend/go-app/files.go index d7ab890f..df5fdcd7 100644 --- a/backend/go-app/files.go +++ b/backend/go-app/files.go @@ -37,6 +37,7 @@ type File struct { WorkflowId string `json:"workflow_id" datastore:"workflow_id"` Workflows []string `json:"workflows" datastore:"workflows"` DownloadPath string `json:"download_path" datastore:"download_path"` + Md5sum string `json:"md5_sum" datastore:"md5_sum"` } var basepath = os.Getenv("SHUFFLE_FILE_LOCATION") @@ -66,11 +67,14 @@ func fileAuthentication(request *http.Request) (string, error) { // This is annoying af and is done because of maxlength lol newApikey := apikeyCheck[1] if newApikey != workflowExecution.Authorization { - log.Printf("[ERROR] Bad apikey for execution %s. %s vs %s", executionId[0], apikey, workflowExecution.Authorization) + //log.Printf("[ERROR] Bad apikey for execution %s. %s vs %s", executionId[0], apikey, workflowExecution.Authorization) + log.Printf("[ERROR] Bad apikey for execution %s.", executionId[0]) + //%s vs %s", executionId[0], apikey, workflowExecution.Authorization) return "", errors.New("Bad authorization key") } - log.Printf("[INFO] Authorization is correct for execution %s! %s vs %s. Setting Org", executionId, apikey, workflowExecution.Authorization) + log.Printf("[INFO] Authorization is correct for execution %s!", executionId[0]) + //%s vs %s. Setting Org", executionId, apikey, workflowExecution.Authorization) if len(workflowExecution.ExecutionOrg) > 0 { return workflowExecution.ExecutionOrg, nil } else if len(workflowExecution.Workflow.ExecutingOrg.Id) > 0 { @@ -115,7 +119,14 @@ func handleGetFileMeta(resp http.ResponseWriter, request *http.Request) { fileId = strings.Split(fileId, "?")[0] } - log.Printf("\n\n[INFO] User is trying to delete file %s\n\n", fileId) + if len(fileId) != 36 { + log.Printf("Bad format for fileId %s", fileId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`)) + return + } + + log.Printf("\n\n[INFO] User is trying to GET File Meta for %s\n\n", fileId) // 1. Check user directly // 2. Check workflow execution authorization @@ -136,7 +147,7 @@ func handleGetFileMeta(resp http.ResponseWriter, request *http.Request) { } // 1. Verify if the user has access to the file: org_id and workflow - log.Printf("[INFO] Should DELETE file %s if user has access", fileId) + log.Printf("[INFO] Should GET FILE META for %s if user has access", fileId) ctx := context.Background() file, err := getFile(ctx, fileId) if err != nil { @@ -172,7 +183,7 @@ func handleGetFileMeta(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("[INFO] Successfully deleted file %s", fileId) + log.Printf("[INFO] Successfully got file meta for %s", fileId) resp.WriteHeader(200) resp.Write([]byte(newBody)) } @@ -200,6 +211,13 @@ func handleDeleteFile(resp http.ResponseWriter, request *http.Request) { fileId = strings.Split(fileId, "?")[0] } + if len(fileId) != 36 { + log.Printf("Bad format for fileId %s", fileId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`)) + return + } + log.Printf("\n\n[INFO] User is trying to delete file %s\n\n", fileId) // 1. Check user directly @@ -318,7 +336,14 @@ func handleGetFileContent(resp http.ResponseWriter, request *http.Request) { fileId = location[4] } - log.Printf("\n\nUser is trying to get file %s\n\n", fileId) + if len(fileId) != 36 { + log.Printf("Bad format for fileId %s", fileId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`)) + return + } + + log.Printf("\n\nUser is trying to download file %s\n\n", fileId) // 1. Check user directly // 2. Check workflow execution authorization @@ -346,11 +371,11 @@ func handleGetFileContent(resp http.ResponseWriter, request *http.Request) { } // 1. Verify if the user has access to the file: org_id and workflow - log.Printf("Should get file %s", fileId) + log.Printf("[INFO] Should get file %s", fileId) ctx := context.Background() file, err := getFile(ctx, fileId) if err != nil { - log.Printf("File %s not found: %s", fileId, err) + log.Printf("[ERROR] File %s not found: %s", fileId, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -376,7 +401,7 @@ func handleGetFileContent(resp http.ResponseWriter, request *http.Request) { } if file.Status != "active" { - log.Printf("File status isn't active. Can't continue.") + log.Printf("[ERROR] File status isn't active, but %s. Can't continue.", file.Status) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "The file isn't ready to be downloaded yet. Status required: active"}`)) return @@ -384,7 +409,7 @@ func handleGetFileContent(resp http.ResponseWriter, request *http.Request) { // Fixme: More auth: org and workflow! downloadPath := file.DownloadPath - log.Printf("Downloadpath: %s", downloadPath) + log.Printf("[INFO] Downloadpath: %s", downloadPath) Openfile, err := os.Open(downloadPath) defer Openfile.Close() //Close after function return if err != nil { @@ -438,6 +463,13 @@ func handleUploadFile(resp http.ResponseWriter, request *http.Request) { fileId = location[4] } + if len(fileId) != 36 { + log.Printf("Bad format for fileId %s", fileId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`)) + return + } + // 1. Check user directly // 2. Check workflow execution authorization user, err := handleApiAuthentication(resp, request) @@ -494,7 +526,6 @@ func handleUploadFile(resp http.ResponseWriter, request *http.Request) { } request.ParseMultipartForm(32 << 20) - //var buf bytes.Buffer parsedFile, _, err := request.FormFile("shuffle_file") if err != nil { log.Printf("[ERROR] Couldn't upload file: %s", err) @@ -513,16 +544,23 @@ func handleUploadFile(resp http.ResponseWriter, request *http.Request) { return } - //io.Copy(&buf, parsedFile) - //contents := buf.String() - //buf.Reset() + // Can be used for validation files for change + /* + var buf bytes.Buffer + io.Copy(&buf, parsedFile) + contents := buf.Bytes() + md5 := md5sum(contents) + buf.Reset() + */ + md5 := "" + f, err := os.OpenFile(file.DownloadPath, os.O_WRONLY|os.O_CREATE, os.ModePerm) if err != nil { // Rolling back file file.Status = "created" setFile(ctx, *file) - log.Printf("Failed creating file: %s", err) + log.Printf("[ERROR] Failed uploading and creating file: %s", err) resp.WriteHeader(500) resp.Write([]byte(`{"success": false}`)) return @@ -532,7 +570,9 @@ func handleUploadFile(resp http.ResponseWriter, request *http.Request) { io.Copy(f, parsedFile) // FIXME: Set this one to 200 anyway? Can't download file then tho.. + log.Printf("[INFO] MD5 for file %s is %s", file.Filename, md5) file.Status = "active" + file.Md5sum = md5 err = setFile(ctx, *file) if err != nil { log.Printf("[ERROR] Failed setting file back to active") diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 07d6e047..11ee9b80 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -7119,37 +7119,39 @@ func runInit(ctx context.Context) { } } - fileq := datastore.NewQuery("Files").Limit(1) - count, err := dbclient.Count(ctx, fileq) - log.Printf("FILECOUNT: %d", count) - if err == nil && count < 10 { - basepath := "." - filename := "testfile.txt" - fileId := uuid.NewV4().String() - log.Printf("Creating new file reference %s because none exist!", fileId) - workflowId := "2cf1169d-b460-41de-8c36-28b2092866f8" - downloadPath := fmt.Sprintf("%s/%s/%s/%s", basepath, activeOrgs[0].Id, workflowId, fileId) + /* + fileq := datastore.NewQuery("Files").Limit(1) + count, err := dbclient.Count(ctx, fileq) + log.Printf("FILECOUNT: %d", count) + if err == nil && count < 10 { + basepath := "." + filename := "testfile.txt" + fileId := uuid.NewV4().String() + log.Printf("Creating new file reference %s because none exist!", fileId) + workflowId := "2cf1169d-b460-41de-8c36-28b2092866f8" + downloadPath := fmt.Sprintf("%s/%s/%s/%s", basepath, activeOrgs[0].Id, workflowId, fileId) - timeNow := time.Now().Unix() - newFile := File{ - Id: fileId, - CreatedAt: timeNow, - UpdatedAt: timeNow, - Description: "Created by system for testing", - Status: "active", - Filename: filename, - OrgId: activeOrgs[0].Id, - WorkflowId: workflowId, - DownloadPath: downloadPath, - } + timeNow := time.Now().Unix() + newFile := File{ + Id: fileId, + CreatedAt: timeNow, + UpdatedAt: timeNow, + Description: "Created by system for testing", + Status: "active", + Filename: filename, + OrgId: activeOrgs[0].Id, + WorkflowId: workflowId, + DownloadPath: downloadPath, + } - err = setFile(ctx, newFile) - if err != nil { - log.Printf("Failed setting file: %s", err) - } else { - log.Printf("Created file %s in init", newFile.DownloadPath) + err = setFile(ctx, newFile) + if err != nil { + log.Printf("Failed setting file: %s", err) + } else { + log.Printf("Created file %s in init", newFile.DownloadPath) + } } - } + */ var allworkflowapps []AppAuthenticationStorage q = datastore.NewQuery("workflowappauth") @@ -8045,10 +8047,10 @@ func initHandlers() { // https://developer.box.com/reference/get-files-id-content/ // 1. Creating the "get file" option. Make it possible to run this in the frontend. r.HandleFunc("/api/v1/files/{fileId}/content", handleGetFileContent).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/files/{fileId}", handleDeleteFile).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/files/create", handleCreateFile).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/files/{fileId}/upload", handleUploadFile).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/files/{fileId}", handleGetFileMeta).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/files/{fileId}", handleDeleteFile).Methods("DELETE", "OPTIONS") http.Handle("/", r) } diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 011db663..6c4dfb5b 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -2795,7 +2795,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf continue } - log.Printf("Should set %s to SKIPPED as it's NOT a childnode of the startnode.", action.ID) + log.Printf("[WARNING] Set %s to SKIPPED as it's NOT a childnode of the startnode.", action.ID) defaultResults = append(defaultResults, ActionResult{ Action: action, ExecutionId: workflowExecution.ExecutionId, diff --git a/docker-compose.yml b/docker-compose.yml index 11a7fe29..c496eb65 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: frontend: - build: ./frontend + #build: ./frontend image: ghcr.io/frikky/shuffle-frontend:0.8.0 container_name: shuffle-frontend hostname: shuffle-frontend @@ -16,8 +16,8 @@ services: depends_on: - backend backend: - #build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.1 + build: ./backend + image: ghcr.io/frikky/shuffle-backend:0.8.11 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index e2c57f3e..e10b0930 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -5354,7 +5354,7 @@ const AngularWorkflow = (props) => { return null } - const cytoscapeViewWidths = 700 + const cytoscapeViewWidths = 750 const bottomBarStyle = { position: "fixed", right: 20, @@ -5412,6 +5412,62 @@ const AngularWorkflow = (props) => { return null } + const FileMenu = () => { + const [newAnchor, setNewAnchor] = React.useState(null); + const [showShuffleMenu, setShowShuffleMenu] = React.useState(false) + + { /*const [showShuffleMenu, setShowShuffleMenu] = React.useState(true) */} + return ( +
+ { + setShowShuffleMenu(false) + }} + > +
+

This menu is used to control the workflow itself.

+ + Exit on Error
} + control={ + { + workflow.configuration.exit_on_error = !workflow.configuration.exit_on_error + setWorkflow(workflow) + setUpdate("exit_on_error_"+workflow.configuration.exit_on_error ? "true" : "false") + setShowShuffleMenu(false) + }} /> + } + /> + Start from top
} + control={ + { + workflow.configuration.start_from_top = !workflow.configuration.start_from_top + setWorkflow(workflow) + setUpdate("start_from_top_"+workflow.configuration.start_from_top ? "true" : "false") + setShowShuffleMenu(false) + }} /> + } + /> + + + + + + + ) + } + const WorkflowMenu = () => { const [newAnchor, setNewAnchor] = React.useState(null); const [showShuffleMenu, setShowShuffleMenu] = React.useState(false) @@ -5539,6 +5595,7 @@ const AngularWorkflow = (props) => { + {/* */} diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 52f913d5..08499f93 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -360,7 +360,7 @@ const Workflows = (props) => { data.triggers[key].status = "stopped" } } - + data["org"] = [] data.execution_org = {"id": ""} console.log(data) From af1ca402b1834e602ef7d865fbffff454c3fe79c Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 29 Nov 2020 14:05:59 +0100 Subject: [PATCH 07/19] Exposed app sdk functions for files --- backend/app_sdk/app_base.py | 257 ++++++++++++++++++------------------ backend/go-app/main.go | 6 +- 2 files changed, 132 insertions(+), 131 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 362ea503..ae4042b1 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -51,6 +51,130 @@ class AppBase: self.logger.info("Result: %d" % ret.status_code) if ret.status_code != 200: self.logger.info(ret.text) + + # Things to consider for files: + # - How can you download / stream a file? + # - Can you decide if you want a stream or the files directly? + def get_files(self, full_execution, value): + org_id = full_execution["workflow"]["execution_org"]["id"] + print("SHOULD GET FILES BASED ON ORG %s, workflow %s and value(s) %s" % (org_id, full_execution["workflow"]["id"], value)) + + get_path = "/api/v1/files/%s?execution_id=%s" % (value, full_execution["execution_id"]) + headers = { + "Content-Type": "application/json", + "Authorization": "Bearer %s" % self.authorization + } + + ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers) + print("RET1: %s" % ret1.text) + if ret1.status_code != 200: + return { + "filename": "", + "data": "", + "success": False, + } + + content_path = "/api/v1/files/%s/content?execution_id=%s" % (value, full_execution["execution_id"]) + ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers) + print("Ret2: %s" % ret2.text) + if ret2.status_code == 200: + tmpdata = ret1.json() + returndata = { + "success": True, + "filename": tmpdata["filename"], + "data": ret2.content, + } + # open('facebook.ico', 'wb').write(r.content) + + return returndata + + return { + "success": False, + "filename": "", + "data": b"", + } + + # Sets files in the backend + def set_files(self, full_execution, infiles): + workflow_id = full_execution["workflow"]["id"] + org_id = full_execution["workflow"]["execution_org"]["id"] + headers = { + "Content-Type": "application/json", + "Authorization": "Bearer %s" % self.authorization + } + + create_path = "/api/v1/files/create?execution_id=%s" % full_execution["execution_id"] + file_ids = [] + for curfile in infiles: + filename = "unspecified" + data = { + "filename": filename, + "workflow_id": workflow_id, + "org_id": org_id, + } + + try: + data["filename"] = curfile["filename"] + filename = curfile["filename"] + except KeyError as e: + print("KeyError in file setup: %s" % e) + pass + + ret = requests.post("%s%s" % (self.url, create_path), headers=headers, json=data) + print("Ret CREATE: %s" % ret.text) + cur_id = "" + if ret.status_code == 200: + print("RET: %s" % ret.text) + ret_json = ret.json() + if not ret_json["success"]: + print("Not success in file upload creation.") + continue + + print("Should handle ID %s" % ret_json["id"]) + file_ids.append(ret_json["id"]) + cur_id = ret_json["id"] + else: + print("Bad status code: %d" % ret.status_code) + continue + + if len(cur_id) == 0: + print("No file ID specified from backend") + continue + + new_headers = { + "Authorization": "Bearer %s" % self.authorization, + } + + upload_path = "/api/v1/files/%s/upload?execution_id=%s" % (cur_id, full_execution["execution_id"]) + print("Create path: %s" % create_path) + + # FIXME: Typical failure here if data is returned badly formatted + files={"shuffle_file": (filename, curfile["data"])} + #open(filename,'rb')} + + ret = requests.post("%s%s" % (self.url, upload_path), files=files, headers=new_headers) + print("Ret UPLOAD: %s" % ret.text) + print("Ret2 UPLOAD: %d" % ret.status_code) + + print("IDS TO RETURN: %s" % file_ids) + return file_ids + + # Checks whether conditions are met, otherwise set + branchcheck, tmpresult = check_branch_conditions(action, fullexecution) + if not branchcheck: + self.logger.info("Failed one or more branch conditions.") + action_result["result"] = tmpresult + action_result["status"] = "FAILURE" + try: + ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result) + self.logger.info("Result: %d" % ret.status_code) + if ret.status_code != 200: + self.logger.info(ret.text) + except requests.exceptions.ConnectionError as e: + self.logger.exception(e) + + print("\n\nRETURNING BECAUSE A BRANCH FAILED\n\n") + return async def execute_action(self, action): # FIXME - add request for the function STARTING here. Use "results stream" or something @@ -834,129 +958,6 @@ class AppBase: return True, "" - # Things to consider for files: - # - How can you download / stream a file? - # - Can you decide if you want a stream or the files directly? - def get_files(full_execution, value): - org_id = full_execution["workflow"]["execution_org"]["id"] - print("SHOULD GET FILES BASED ON ORG %s, workflow %s and value(s) %s" % (org_id, full_execution["workflow"]["id"], value)) - - get_path = "/api/v1/files/%s?execution_id=%s" % (value, full_execution["execution_id"]) - headers = { - "Content-Type": "application/json", - "Authorization": "Bearer %s" % self.authorization - } - - ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers) - print("RET1: %s" % ret1.text) - if ret1.status_code != 200: - return { - "filename": "", - "data": "", - "success": False, - } - - content_path = "/api/v1/files/%s/content?execution_id=%s" % (value, full_execution["execution_id"]) - ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers) - print("Ret2: %s" % ret2.text) - if ret2.status_code == 200: - tmpdata = ret1.json() - returndata = { - "success": True, - "filename": tmpdata["filename"], - "data": ret2.content, - } - # open('facebook.ico', 'wb').write(r.content) - - return returndata - - return { - "success": False, - "filename": "", - "data": "", - } - - # Sets files in the backend - def set_files(full_execution, infiles): - workflow_id = full_execution["workflow"]["id"] - org_id = full_execution["workflow"]["execution_org"]["id"] - headers = { - "Content-Type": "application/json", - "Authorization": "Bearer %s" % self.authorization - } - - create_path = "/api/v1/files/create?execution_id=%s" % full_execution["execution_id"] - file_ids = [] - for curfile in infiles: - filename = "unspecified" - data = { - "filename": filename, - "workflow_id": workflow_id, - "org_id": org_id, - } - - try: - data["filename"] = curfile["filename"] - filename = curfile["filename"] - except KeyError as e: - print("KeyError in file setup: %s" % e) - pass - - ret = requests.post("%s%s" % (self.url, create_path), headers=headers, json=data) - print("Ret CREATE: %s" % ret.text) - cur_id = "" - if ret.status_code == 200: - print("RET: %s" % ret.text) - ret_json = ret.json() - if not ret_json["success"]: - print("Not success in file upload creation.") - continue - - print("Should handle ID %s" % ret_json["id"]) - file_ids.append(ret_json["id"]) - cur_id = ret_json["id"] - else: - print("Bad status code: %d" % ret.status_code) - continue - - if len(cur_id) == 0: - print("No file ID specified from backend") - continue - - new_headers = { - "Authorization": "Bearer %s" % self.authorization, - } - - upload_path = "/api/v1/files/%s/upload?execution_id=%s" % (cur_id, full_execution["execution_id"]) - print("Create path: %s" % create_path) - #files={"shuffle_file": open(filename,'rb')} - files={"shuffle_file": (filename, curfile["data"])} - #open(filename,'rb')} - - ret = requests.post("%s%s" % (self.url, upload_path), files=files, headers=new_headers) - print("Ret UPLOAD: %s" % ret.text) - print("Ret2 UPLOAD: %d" % ret.status_code) - - print("IDS TO RETURN: %s" % file_ids) - return file_ids - - # Checks whether conditions are met, otherwise set - branchcheck, tmpresult = check_branch_conditions(action, fullexecution) - if not branchcheck: - self.logger.info("Failed one or more branch conditions.") - action_result["result"] = tmpresult - action_result["status"] = "FAILURE" - try: - ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result) - self.logger.info("Result: %d" % ret.status_code) - if ret.status_code != 200: - self.logger.info(ret.text) - except requests.exceptions.ConnectionError as e: - self.logger.exception(e) - - print("\n\nRETURNING BECAUSE A BRANCH FAILED\n\n") - return - # Replace name cus there might be issues # Not doing lower() as there might be user-made functions actionname = action["name"] @@ -1110,7 +1111,7 @@ class AppBase: try: if parameter["schema"]["type"] == "file" and len(value) > 0: print("SHOULD HANDLE FILE IN MULTI. Get based on value %s" % parameter["value"]) - file_value = get_files(fullexecution, tmpitem) + file_value = self.get_files(fullexecution, tmpitem) print("FILE VALUE FOR VAL %s: %s" % (tmpitem, file_value)) resultarray.append(file_value) @@ -1140,7 +1141,7 @@ class AppBase: try: if parameter["schema"]["type"] == "file" and len(value) > 0: print("\n SHOULD HANDLE FILE. Get based on value %s. <--- is this a valid ID?" % parameter["value"]) - file_value = get_files(fullexecution, value) + file_value = self.get_files(fullexecution, value) print("FILE VALUE: %s \n" % file_value) params[parameter["name"]] = file_value @@ -1184,13 +1185,13 @@ class AppBase: print("TUPLE: %s" % newres[1]) if isinstance(newres[1], list): print("HANDLING LIST FROM RET") - file_ids = set_files(fullexecution, newres[1]) + file_ids = self.set_files(fullexecution, newres[1]) elif isinstance(newres[1], object): print("Handling JSON from ret") - file_ids = set_files(fullexecution, [newres[1]]) + file_ids = self.set_files(fullexecution, [newres[1]]) elif isinstance(newres[1], str): print("Handling STRING from ret") - file_ids = set_files(fullexecution, [newres[1]]) + file_ids = self.set_files(fullexecution, [newres[1]]) else: print("NO FILES TO HANDLE") diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 11ee9b80..403a2920 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -2422,7 +2422,7 @@ func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { } if count == 0 { - log.Printf("No users - redirecting for management user") + log.Printf("[WARNING] No users - redirecting for management user") resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "stay"}`))) return @@ -2555,12 +2555,12 @@ func getApikey(ctx context.Context, apikey string) (User, error) { var users []User _, err := dbclient.GetAll(ctx, q, &users) if err != nil { - log.Printf("Error getting users apikey (getapikey): %s", err) + log.Printf("[ERROR] Error getting users apikey (getapikey): %s", err) return User{}, err } if len(users) == 0 { - log.Printf("No users found for apikey %s", apikey) + log.Printf("[WARNING] No users found for apikey %s", apikey) return User{}, err } From 68296d5af7c4ddb418b41a87e29646481c51b18b Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 30 Nov 2020 10:30:56 +0100 Subject: [PATCH 08/19] #204: Cleaned up issues with help from @dadokkia --- backend/app_sdk/app_base.py | 50 +++++++++++++++++++------------------ backend/app_sdk/build.sh | 2 +- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index ae4042b1..07104afa 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -55,7 +55,8 @@ class AppBase: # Things to consider for files: # - How can you download / stream a file? # - Can you decide if you want a stream or the files directly? - def get_files(self, full_execution, value): + def get_file(self, value): + full_execution = self.full_execution org_id = full_execution["workflow"]["execution_org"]["id"] print("SHOULD GET FILES BASED ON ORG %s, workflow %s and value(s) %s" % (org_id, full_execution["workflow"]["id"], value)) @@ -95,7 +96,8 @@ class AppBase: } # Sets files in the backend - def set_files(self, full_execution, infiles): + def set_files(self, infiles): + full_execution = self.full_execution workflow_id = full_execution["workflow"]["id"] org_id = full_execution["workflow"]["execution_org"]["id"] headers = { @@ -158,23 +160,6 @@ class AppBase: print("IDS TO RETURN: %s" % file_ids) return file_ids - - # Checks whether conditions are met, otherwise set - branchcheck, tmpresult = check_branch_conditions(action, fullexecution) - if not branchcheck: - self.logger.info("Failed one or more branch conditions.") - action_result["result"] = tmpresult - action_result["status"] = "FAILURE" - try: - ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result) - self.logger.info("Result: %d" % ret.status_code) - if ret.status_code != 200: - self.logger.info(ret.text) - except requests.exceptions.ConnectionError as e: - self.logger.exception(e) - - print("\n\nRETURNING BECAUSE A BRANCH FAILED\n\n") - return async def execute_action(self, action): # FIXME - add request for the function STARTING here. Use "results stream" or something @@ -958,6 +943,23 @@ class AppBase: return True, "" + # Checks whether conditions are met, otherwise set + branchcheck, tmpresult = check_branch_conditions(action, fullexecution) + if not branchcheck: + self.logger.info("Failed one or more branch conditions.") + action_result["result"] = tmpresult + action_result["status"] = "FAILURE" + try: + ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result) + self.logger.info("Result: %d" % ret.status_code) + if ret.status_code != 200: + self.logger.info(ret.text) + except requests.exceptions.ConnectionError as e: + self.logger.exception(e) + + print("\n\nRETURNING BECAUSE A BRANCH FAILED\n\n") + return + # Replace name cus there might be issues # Not doing lower() as there might be user-made functions actionname = action["name"] @@ -1111,7 +1113,7 @@ class AppBase: try: if parameter["schema"]["type"] == "file" and len(value) > 0: print("SHOULD HANDLE FILE IN MULTI. Get based on value %s" % parameter["value"]) - file_value = self.get_files(fullexecution, tmpitem) + file_value = self.get_file(tmpitem) print("FILE VALUE FOR VAL %s: %s" % (tmpitem, file_value)) resultarray.append(file_value) @@ -1141,7 +1143,7 @@ class AppBase: try: if parameter["schema"]["type"] == "file" and len(value) > 0: print("\n SHOULD HANDLE FILE. Get based on value %s. <--- is this a valid ID?" % parameter["value"]) - file_value = self.get_files(fullexecution, value) + file_value = self.get_file(value) print("FILE VALUE: %s \n" % file_value) params[parameter["name"]] = file_value @@ -1185,13 +1187,13 @@ class AppBase: print("TUPLE: %s" % newres[1]) if isinstance(newres[1], list): print("HANDLING LIST FROM RET") - file_ids = self.set_files(fullexecution, newres[1]) + file_ids = self.set_files(newres[1]) elif isinstance(newres[1], object): print("Handling JSON from ret") - file_ids = self.set_files(fullexecution, [newres[1]]) + file_ids = self.set_files([newres[1]]) elif isinstance(newres[1], str): print("Handling STRING from ret") - file_ids = self.set_files(fullexecution, [newres[1]]) + file_ids = self.set_files([newres[1]]) else: print("NO FILES TO HANDLE") diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 4dbf2574..73148aa2 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,6 +1,6 @@ #!/bin/bash NAME=shuffle-app_sdk -VERSION=0.8.1 +VERSION=0.8.11 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION From 099e73916e9fcafbf92b49464e61f0074dcf94b4 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 1 Dec 2020 15:15:19 +0100 Subject: [PATCH 09/19] BUG: Fixed transactional issue with workflowexecutions --- backend/go-app/files.go | 2 +- backend/go-app/walkoff.go | 51 ++++++++++++++++++++++++++++++++++----- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/backend/go-app/files.go b/backend/go-app/files.go index df5fdcd7..aa614f06 100644 --- a/backend/go-app/files.go +++ b/backend/go-app/files.go @@ -683,7 +683,7 @@ func handleCreateFile(resp http.ResponseWriter, request *http.Request) { } } - if strings.Contains(curfile.Filename, "/") || strings.Contains(curfile.Filename, `"`) || strings.Contains(curfile.Filename, "..") { + if strings.Contains(curfile.Filename, "/") || strings.Contains(curfile.Filename, `"`) || strings.Contains(curfile.Filename, "..") || strings.Contains(curfile.Filename, "~") { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Invalid characters in filename"}`)) return diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 6c4dfb5b..e12fe8ce 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -954,6 +954,31 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { return } + runWorkflowExecutionTransaction(0, workflowExecution.ExecutionId, actionResult, resp) +} + +// Will make sure transactions are always ran for an execution. This is recursive if it fails. Allowed to fail up to 5 times +func runWorkflowExecutionTransaction(attempts int64, workflowExecutionId string, actionResult ActionResult, resp http.ResponseWriter) { + ctx := context.Background() + // Should start a tx for the execution here + tx, err := dbclient.NewTransaction(ctx) + if err != nil { + log.Printf("client.NewTransaction: %v", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed creating transaction"}`))) + return + } + + key := datastore.NameKey("workflowexecution", workflowExecutionId, nil) + workflowExecution := &WorkflowExecution{} + if err := tx.Get(key, workflowExecution); err != nil { + log.Printf("tx.Get bug: %v", err) + tx.Rollback() + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting the workflow key"}`))) + return + } + if actionResult.Status == "ABORTED" || actionResult.Status == "FAILURE" { log.Printf("Actionresult is %s. Should set workflowExecution and exit all running functions", actionResult.Status) @@ -1228,18 +1253,32 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { } } - err = setWorkflowExecution(ctx, *workflowExecution) - if err != nil { - //workflowExecution.Result = "Error setting workflow: result too large" - //workflowExecution.Status = "FINISHED" - //workflowExecution.CompletedAt = int64(time.Now().Unix()) + // Transactions: https://cloud.google.com/datastore/docs/concepts/transactions#datastore-datastore-transactional-update-go + // Prevents timing issues + //ExecutionId + if _, err := tx.Put(key, workflowExecution); err != nil { + tx.Rollback() + log.Printf("[ERROR] tx.Put bug: %v", err) - log.Printf("Error saving workflow execution actionresult setting: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting workflowexecution actionresult: %s"}`, err))) return } + if _, err = tx.Commit(); err != nil { + log.Printf("[ERROR] tx.Commit: %v", err) + + if attempts >= 0 { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + attempts += 1 + runWorkflowExecutionTransaction(attempts, workflowExecutionId, actionResult, resp) + return + } + resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } From 93491f5ffb9f9e4f8d0691720b456edd0f924793 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 1 Dec 2020 16:49:54 +0100 Subject: [PATCH 10/19] #204: Worked on multi-files. Found coroutine async issues --- backend/app_sdk/app_base.py | 67 ++++++++++++++++++++------ backend/app_sdk/build.sh | 2 +- backend/go-app/walkoff.go | 16 +++--- frontend/src/views/AngularWorkflow.jsx | 2 +- 4 files changed, 64 insertions(+), 23 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 07104afa..acf2e071 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -56,8 +56,11 @@ class AppBase: # - How can you download / stream a file? # - Can you decide if you want a stream or the files directly? def get_file(self, value): + print("INSIDE GET_FILE") + full_execution = self.full_execution org_id = full_execution["workflow"]["execution_org"]["id"] + print("SHOULD GET FILES BASED ON ORG %s, workflow %s and value(s) %s" % (org_id, full_execution["workflow"]["id"], value)) get_path = "/api/v1/files/%s?execution_id=%s" % (value, full_execution["execution_id"]) @@ -317,7 +320,7 @@ class AppBase: try: return int(data) except ValueError: - print("ValueError while casting %s" % data) + print("ValueError while casting %s to int" % data) return data if "lower" in thistype: return data.lower() @@ -329,15 +332,19 @@ class AppBase: return data.strip() if "split" in thistype: return data.split() - if "len" in thistype or "length" in thistype: + if "len" in thistype or "length" in thistype or "lenght" in thistype: tmp = "" try: - tmp = json.loads(data) + tmpdata = data.replace("\'", "\"") + tmp = json.loads(tmpdata) except: + print("Passing bug") pass if isinstance(tmp, list): - return str(len(tmp)) + return len(tmp) + elif isinstance(tmp, object): + return len(tmp) return str(len(data)) if "parse" in thistype: @@ -383,7 +390,7 @@ class AppBase: #print("Running %s" % data) # Look for the INNER wrapper first, then move out - wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length"] + wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length", "lenght"] found = False for wrapper in wrappers: if wrapper not in data.lower(): @@ -413,6 +420,7 @@ class AppBase: continue parsed_value = parse_type(innervalue[0], thistype.lower()) + print("Parsed value from %s: %s" % (thistype, parsed_value)) return parsed_value print("DATA: %s\n" % data) @@ -1057,13 +1065,40 @@ class AppBase: minlength = len(json_replacement) tmpitem = tmpitem.replace(actualitem[0][0], replacement, 1) - params[parameter["name"]] = tmpitem - multi_execution_lists.append(json_replacement) - multi_parameters[parameter["name"]] = json_replacement - #print("LENGTH OF ARR: %d" % len(resultarray)) - #print("RESULTARRAY: %s" % resultarray) - print("MULTI finished: %s" % replacement) + # This code handles files. + print("(1) ------------ PARAM: %s" % parameter["schema"]["type"]) + resultarray = [] + isfile = False + try: + if parameter["schema"]["type"] == "file" and len(value) > 0: + print("(1) SHOULD HANDLE FILE IN MULTI. Get based on value %s" % tmpitem) + # This is silly :) + for tmp_file_split in json.loads(tmpitem): + print("PRE GET FILE %s" % tmp_file_split) + file_value = self.get_file(tmp_file_split) + print("PRE AWAIT %s" % file_value) + await file_value + print("POST AWAIT %s" % file_value) + resultarray.append(file_value) + print("(1) FILE VALUE FOR VAL %s: %s" % (tmp_file_split, file_value)) + + isfile = True + except KeyError as e: + print("(1) SCHEMA ERROR IN FILE HANDLING: %s" % e) + except json.decoder.JSONDecodeError as e: + print("(1) JSON ERROR IN FILE HANDLING: %s" % e) + + if not isfile: + params[parameter["name"]] = tmpitem + multi_parameters[parameter["name"]] = json_replacement + else: + print("Resultarray: %s" % resultarray) + params[parameter["name"]] = resultarray + multi_parameters[parameter["name"]] = resultarray + + multi_execution_lists.append(json_replacement) + print("MULTI finished: %s" % json_replacement) else: # This is here to handle for loops within variables.. kindof # 1. Find the length of the longest array @@ -1109,14 +1144,15 @@ class AppBase: # This code handles files. + print("------------ PARAM: %s" % parameter["schema"]["type"]) isfile = False try: if parameter["schema"]["type"] == "file" and len(value) > 0: print("SHOULD HANDLE FILE IN MULTI. Get based on value %s" % parameter["value"]) - file_value = self.get_file(tmpitem) + file_value = await self.get_file(tmpitem) print("FILE VALUE FOR VAL %s: %s" % (tmpitem, file_value)) resultarray.append(file_value) - + isfile = True except KeyError as e: print("SCHEMA ERROR IN FILE HANDLING: %s" % e) @@ -1143,7 +1179,7 @@ class AppBase: try: if parameter["schema"]["type"] == "file" and len(value) > 0: print("\n SHOULD HANDLE FILE. Get based on value %s. <--- is this a valid ID?" % parameter["value"]) - file_value = self.get_file(value) + file_value = await self.get_file(value) print("FILE VALUE: %s \n" % file_value) params[parameter["name"]] = file_value @@ -1282,7 +1318,8 @@ class AppBase: print("KeyError: %s" % e) baseparams[key] = "KeyError: %s" % e - print("Running with params %s (1)" % baseparams) + + print("Running with params (1): %s" % baseparams) ret = await func(**baseparams) print("Return from execution: %s" % ret) if isinstance(ret, dict) or isinstance(ret, list): diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 73148aa2..4532a20e 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,6 +1,6 @@ #!/bin/bash NAME=shuffle-app_sdk -VERSION=0.8.11 +VERSION=0.8.2 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index e12fe8ce..bf28ebe1 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -954,12 +954,11 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { return } - runWorkflowExecutionTransaction(0, workflowExecution.ExecutionId, actionResult, resp) + runWorkflowExecutionTransaction(ctx, 0, workflowExecution.ExecutionId, actionResult, resp) } // Will make sure transactions are always ran for an execution. This is recursive if it fails. Allowed to fail up to 5 times -func runWorkflowExecutionTransaction(attempts int64, workflowExecutionId string, actionResult ActionResult, resp http.ResponseWriter) { - ctx := context.Background() +func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workflowExecutionId string, actionResult ActionResult, resp http.ResponseWriter) { // Should start a tx for the execution here tx, err := dbclient.NewTransaction(ctx) if err != nil { @@ -1266,16 +1265,21 @@ func runWorkflowExecutionTransaction(attempts int64, workflowExecutionId string, } if _, err = tx.Commit(); err != nil { - log.Printf("[ERROR] tx.Commit: %v", err) + if attempts >= 5 { + log.Printf("[ERROR] QUITTING: tx.Commit %d: %v", attempts, err) + tx.Rollback() + workflowExecution.Status = "ABORTED" + setWorkflowExecution(ctx, *workflowExecution) - if attempts >= 0 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } + log.Printf("[WARNING] tx.Commit %d: %v", attempts, err) + attempts += 1 - runWorkflowExecutionTransaction(attempts, workflowExecutionId, actionResult, resp) + runWorkflowExecutionTransaction(ctx, attempts, workflowExecutionId, actionResult, resp) return } diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index e10b0930..b07a0c17 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -5902,7 +5902,7 @@ const AngularWorkflow = (props) => { try { const tmp = String(JSON.parse(showResult)) if (!showResult.includes("{") && !showResult.includes("[")) { - console.log("IN HERE: ", tmp) + //console.log("IN HERE: ", tmp) jsonvalid = false } } catch (e) { From da8230aae76c3f007e40a46edd8f0d34f32a7128 Mon Sep 17 00:00:00 2001 From: frikky Date: Wed, 2 Dec 2020 11:32:18 +0100 Subject: [PATCH 11/19] Fixed basic multifile grabbing --- backend/app_sdk/app_base.py | 46 ++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index acf2e071..837a26c6 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -1,4 +1,5 @@ import os +import copy import sys import re import time @@ -1074,12 +1075,12 @@ class AppBase: if parameter["schema"]["type"] == "file" and len(value) > 0: print("(1) SHOULD HANDLE FILE IN MULTI. Get based on value %s" % tmpitem) # This is silly :) + # Q: Is there something wrong with the download system? + # It seems to return "FILE CONTENT: %s" with the ID as %s for tmp_file_split in json.loads(tmpitem): - print("PRE GET FILE %s" % tmp_file_split) + print("(1) PRE GET FILE %s" % tmp_file_split) file_value = self.get_file(tmp_file_split) - print("PRE AWAIT %s" % file_value) - await file_value - print("POST AWAIT %s" % file_value) + print("(1) POST AWAIT %s" % file_value) resultarray.append(file_value) print("(1) FILE VALUE FOR VAL %s: %s" % (tmp_file_split, file_value)) @@ -1144,17 +1145,25 @@ class AppBase: # This code handles files. - print("------------ PARAM: %s" % parameter["schema"]["type"]) + print("(2) ------------ PARAM: %s" % parameter["schema"]["type"]) isfile = False try: if parameter["schema"]["type"] == "file" and len(value) > 0: - print("SHOULD HANDLE FILE IN MULTI. Get based on value %s" % parameter["value"]) - file_value = await self.get_file(tmpitem) - print("FILE VALUE FOR VAL %s: %s" % (tmpitem, file_value)) - resultarray.append(file_value) + print("(2) SHOULD HANDLE FILE IN MULTI. Get based on value %s" % parameter["value"]) + + for tmp_file_split in json.loads(parameter["value"]): + print("(2) PRE GET FILE %s" % tmp_file_split) + file_value = self.get_file(tmp_file_split) + print("(2) POST AWAIT %s" % file_value) + resultarray.append(file_value) + print("(2) FILE VALUE FOR VAL %s: %s" % (tmp_file_split, file_value)) + + isfile = True except KeyError as e: - print("SCHEMA ERROR IN FILE HANDLING: %s" % e) + print("(2) SCHEMA ERROR IN FILE HANDLING: %s" % e) + except json.decoder.JSONDecodeError as e: + print("(2) JSON ERROR IN FILE HANDLING: %s" % e) if not isfile: resultarray.append(tmpitem) @@ -1179,7 +1188,7 @@ class AppBase: try: if parameter["schema"]["type"] == "file" and len(value) > 0: print("\n SHOULD HANDLE FILE. Get based on value %s. <--- is this a valid ID?" % parameter["value"]) - file_value = await self.get_file(value) + file_value = self.get_file(value) print("FILE VALUE: %s \n" % file_value) params[parameter["name"]] = file_value @@ -1250,14 +1259,19 @@ class AppBase: print("Can't handle type %s value from function" % (type(newres))) print("POST NEWRES RESULT: ", result) else: - print("APP_SDK DONE: Starting MULTI execution with values %s of length %d" % (multi_parameters, minlength)) + print("APP_SDK DONE: Starting MULTI execution (length: %d) with values %s" % (minlength, multi_parameters)) # 1. Use number of executions based on the arrays being similar # 2. Find the right value from the parsed multi_params results = [] json_object = False for i in range(0, minlength): # To be able to use the results as a list: - baseparams = json.loads(json.dumps(multi_parameters)) + print("1: %s" % multi_parameters) + #baseparams = json.loads(json.dumps(multi_parameters)) + baseparams = copy.deepcopy(multi_parameters) + + print("2: %s: %s" % (type(baseparams), baseparams)) + # {'call': ['GoogleSafebrowsing_2_0', 'VirusTotal_GetReport_3_0']} # 1. Check if list length is same as minlength # 2. If NOT same length, duplicate based on length of array @@ -1268,10 +1282,11 @@ class AppBase: try: firstlist = True for key, value in baseparams.items(): - + print("Itemtype: %s" % type(value)) if isinstance(value, list): try: newvalue = value[i] + print("NEWVALUE: %s" % newvalue) except IndexError: pass @@ -1311,6 +1326,8 @@ class AppBase: firstlist = False baseparams[key] = newvalue + + print("3") except IndexError as e: print("IndexError: %s" % e) baseparams[key] = "IndexError: %s" % e @@ -1319,6 +1336,7 @@ class AppBase: baseparams[key] = "KeyError: %s" % e + print("4") print("Running with params (1): %s" % baseparams) ret = await func(**baseparams) print("Return from execution: %s" % ret) From f1e135f8e37014a2a2874db3867244dbe20dca0e Mon Sep 17 00:00:00 2001 From: frikky Date: Thu, 3 Dec 2020 02:33:21 +0100 Subject: [PATCH 12/19] Fixed md5 and sha256 sums --- backend/go-app/files.go | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/backend/go-app/files.go b/backend/go-app/files.go index aa614f06..026c8e26 100644 --- a/backend/go-app/files.go +++ b/backend/go-app/files.go @@ -5,8 +5,9 @@ package main */ import ( - //"bytes" + "bytes" "context" + "crypto/sha256" "encoding/json" "errors" "fmt" @@ -28,6 +29,8 @@ type File struct { Type string `json:"type" datastore:"type"` CreatedAt int64 `json:"created_at" datastore:"created_at"` UpdatedAt int64 `json:"updated_at" datastore:"updated_at"` + MetaAccessAt int64 `json:"meta_access_at" datastore:"meta_access_at"` + DownloadAt int64 `json:"last_downloaded" datastore:"last_downloaded"` Description string `json:"description" datastore:"description"` ExpiresAt string `json:"expires_at" datastore:"expires_at"` Status string `json:"status" datastore:"status"` @@ -38,6 +41,7 @@ type File struct { Workflows []string `json:"workflows" datastore:"workflows"` DownloadPath string `json:"download_path" datastore:"download_path"` Md5sum string `json:"md5_sum" datastore:"md5_sum"` + Sha256sum string `json:"sha256_sum" datastore:"sha256_sum"` } var basepath = os.Getenv("SHUFFLE_FILE_LOCATION") @@ -545,14 +549,14 @@ func handleUploadFile(resp http.ResponseWriter, request *http.Request) { } // Can be used for validation files for change - /* - var buf bytes.Buffer - io.Copy(&buf, parsedFile) - contents := buf.Bytes() - md5 := md5sum(contents) - buf.Reset() - */ - md5 := "" + var buf bytes.Buffer + io.Copy(&buf, parsedFile) + contents := buf.Bytes() + md5 := md5sum(contents) + buf.Reset() + + sha256Sum := sha256.Sum256(contents) + //parsedFile.Reset() f, err := os.OpenFile(file.DownloadPath, os.O_WRONLY|os.O_CREATE, os.ModePerm) if err != nil { @@ -567,12 +571,15 @@ func handleUploadFile(resp http.ResponseWriter, request *http.Request) { } defer f.Close() + parsedFile.Seek(0, io.SeekStart) io.Copy(f, parsedFile) // FIXME: Set this one to 200 anyway? Can't download file then tho.. - log.Printf("[INFO] MD5 for file %s is %s", file.Filename, md5) file.Status = "active" file.Md5sum = md5 + file.Sha256sum = fmt.Sprintf("%x", sha256Sum) + log.Printf("[INFO] MD5 for file %s (%s) is %s and SHA256 is %s", file.Filename, file.Id, file.Md5sum, file.Sha256sum) + err = setFile(ctx, *file) if err != nil { log.Printf("[ERROR] Failed setting file back to active") From 0ef55b2f03dd56d15bd931e206413e44bfea9b6c Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 6 Dec 2020 05:28:26 +0100 Subject: [PATCH 13/19] #210: Fixed hostmode docker builds --- backend/go-app/docker.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index a68e1643..b0398045 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -181,11 +181,11 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin // Dockerfile is inside the TAR itself. Not local context // docker build --build-arg http_proxy=http://my.proxy.url buildOptions := types.ImageBuildOptions{ - Remove: true, - Tags: tags, - BuildArgs: map[string]*string{}, - NetworkMode: "host", + Remove: true, + Tags: tags, + BuildArgs: map[string]*string{}, } + // NetworkMode: "host", httpProxy := os.Getenv("HTTP_PROXY") if len(httpProxy) > 0 { @@ -244,11 +244,11 @@ func buildImage(tags []string, dockerfileFolder string) error { dockerFileTarReader := bytes.NewReader(buf.Bytes()) buildOptions := types.ImageBuildOptions{ - Remove: true, - Tags: tags, - BuildArgs: map[string]*string{}, - NetworkMode: "host", + Remove: true, + Tags: tags, + BuildArgs: map[string]*string{}, } + //NetworkMode: "host", httpProxy := os.Getenv("HTTP_PROXY") if len(httpProxy) > 0 { From a20b848aba9d780898d7fe52159ced84583a02ca Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 6 Dec 2020 05:30:55 +0100 Subject: [PATCH 14/19] Major changes to multi-file in Shuffle. Not finished yet. --- backend/app_sdk/app_base.py | 101 ++++++++----- backend/go-app/files.go | 2 +- backend/go-app/main.go | 4 +- backend/go-app/walkoff.go | 2 +- docker-compose.yml | 4 +- frontend/src/views/Apps.jsx | 19 ++- frontend/src/views/Workflows.jsx | 236 +++++++++++++++++-------------- 7 files changed, 214 insertions(+), 154 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 837a26c6..fffef6c3 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -57,47 +57,68 @@ class AppBase: # - How can you download / stream a file? # - Can you decide if you want a stream or the files directly? def get_file(self, value): - print("INSIDE GET_FILE") - full_execution = self.full_execution org_id = full_execution["workflow"]["execution_org"]["id"] print("SHOULD GET FILES BASED ON ORG %s, workflow %s and value(s) %s" % (org_id, full_execution["workflow"]["id"], value)) - get_path = "/api/v1/files/%s?execution_id=%s" % (value, full_execution["execution_id"]) - headers = { - "Content-Type": "application/json", - "Authorization": "Bearer %s" % self.authorization - } + if isinstance(value, list): + print("IS LIST!") + #if len(value) == 1: + # value = value[0] + else: + value = [value] - ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers) - print("RET1: %s" % ret1.text) - if ret1.status_code != 200: + returns = [] + for item in value: + print("VALUE: %s" % item) + if len(item) != 36: + print("Bad length for value") + continue + #return { + # "filename": "", + # "data": "", + # "success": False, + #} + + get_path = "/api/v1/files/%s?execution_id=%s" % (item, full_execution["execution_id"]) + headers = { + "Content-Type": "application/json", + "Authorization": "Bearer %s" % self.authorization + } + + ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers) + print("RET1: %s" % ret1.text) + if ret1.status_code != 200: + returns.append({ + "filename": "", + "data": "", + "success": False, + }) + continue + + content_path = "/api/v1/files/%s/content?execution_id=%s" % (item, full_execution["execution_id"]) + ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers) + print("Ret2: %s" % ret2.text) + if ret2.status_code == 200: + tmpdata = ret1.json() + returndata = { + "success": True, + "filename": tmpdata["filename"], + "data": ret2.content, + } + returns.append(returndata) + + if len(returns) == 0: return { - "filename": "", - "data": "", "success": False, + "filename": "", + "data": b"", } - - content_path = "/api/v1/files/%s/content?execution_id=%s" % (value, full_execution["execution_id"]) - ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers) - print("Ret2: %s" % ret2.text) - if ret2.status_code == 200: - tmpdata = ret1.json() - returndata = { - "success": True, - "filename": tmpdata["filename"], - "data": ret2.content, - } - # open('facebook.ico', 'wb').write(r.content) - - return returndata - - return { - "success": False, - "filename": "", - "data": b"", - } + elif len(returns) == 1: + return returns[0] + else: + return returns # Sets files in the backend def set_files(self, infiles): @@ -522,6 +543,7 @@ class AppBase: # Magical way of returning which makes app sdk identify # it as multi execution return newvalue, True + elif len(actualitem) > 0: # FIXME: This is absolutely not perfect. print("In recursion v2: ", actualitem) @@ -704,7 +726,7 @@ class AppBase: print("SET DATA WRAPPER TO %s!" % parsersplit[-1]) parseditem = "${%s%s}$" % (parsersplit[-1], json.dumps(data)) - print("Before last return") + print("Before last return with %s" % appendresult) return str(parseditem)+str(appendresult), is_loop # Parses parameters sent to it and returns whether it did it successfully with the values found @@ -1205,15 +1227,22 @@ class AppBase: if listitem in filteredlist: continue - filteredlist.append(listitem) + # FIXME: Subsub required?. Recursion! + # Basically multiply what we have with the outer loop? + # + if isinstance(listitem, list): + for subitem in listitem: + filteredlist.append(subitem) + else: + filteredlist.append(listitem) #print("New list length: %d" % len(filteredlist)) if len(filteredlist) > 1: print("Calculating new multi-loop length with %d lists" % len(filteredlist)) tmplength = 1 for innerlist in filteredlist: - print("List length: %d. %d*%d" % (len(innerlist), len(innerlist), tmplength)) tmplength = len(innerlist)*tmplength + print("List length: %d. %d*%d" % (tmplength, len(innerlist), tmplength)) minlength = tmplength @@ -1257,6 +1286,7 @@ class AppBase: except ValueError: result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres) print("Can't handle type %s value from function" % (type(newres))) + print("POST NEWRES RESULT: ", result) else: print("APP_SDK DONE: Starting MULTI execution (length: %d) with values %s" % (minlength, multi_parameters)) @@ -1286,7 +1316,6 @@ class AppBase: if isinstance(value, list): try: newvalue = value[i] - print("NEWVALUE: %s" % newvalue) except IndexError: pass diff --git a/backend/go-app/files.go b/backend/go-app/files.go index 026c8e26..a2856e2b 100644 --- a/backend/go-app/files.go +++ b/backend/go-app/files.go @@ -1,7 +1,7 @@ package main /* - Handles files within Workflows. + Handles files within Workflows.of Shuffle */ import ( diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 403a2920..742d3b65 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -6813,7 +6813,7 @@ func remoteOrgJobHandler(org Org, interval int) error { respBody, err := ioutil.ReadAll(newresp.Body) if err != nil { - log.Printf("Failed body read in job sync: %s", err) + log.Printf("[ERROR] Failed body read in job sync: %s", err) return err } @@ -6821,7 +6821,7 @@ func remoteOrgJobHandler(org Org, interval int) error { err = remoteOrgJobController(org, respBody) if err != nil { - log.Printf("Failed job controller run: %s", err) + log.Printf("[ERROR] Failed job controller run for %s: %s", respBody, err) return err } return nil diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index bf28ebe1..70307a82 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -5765,7 +5765,7 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) { } // Query for the specifci workflowId - q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(20) + q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(30) var workflowExecutions []WorkflowExecution _, err = dbclient.GetAll(ctx, q, &workflowExecutions) if err != nil { diff --git a/docker-compose.yml b/docker-compose.yml index c496eb65..014798f3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: frontend: #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.0 + image: ghcr.io/frikky/shuffle-frontend:0.8.3 container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -17,7 +17,7 @@ services: - backend backend: build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.11 + image: ghcr.io/frikky/shuffle-backend:0.8.3 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 9bb8dda4..69ed1dc1 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -116,7 +116,7 @@ const Apps = (props) => { const [apps, setApps] = React.useState([]) const [filteredApps, setFilteredApps] = React.useState([]) const [validation, setValidation] = React.useState(false) - const [isLoading, setIsLoading] = React.useState(false) + const [isLoading, setIsLoading] = React.useState(true) const [appSearchLoading, setAppSearchLoading] = React.useState(false) const [selectedAction, setSelectedAction] = React.useState({}) const [searchBackend, setSearchBackend] = React.useState(false) @@ -207,6 +207,7 @@ const Apps = (props) => { credentials: "include", }) .then((response) => { + setIsLoading(false) if (response.status !== 200) { console.log("Status not 200 for apps :O!") } @@ -231,6 +232,7 @@ const Apps = (props) => { }) .catch(error => { alert.error(error.toString()) + setIsLoading(false) }); } @@ -742,7 +744,7 @@ const Apps = (props) => {
diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 08499f93..98583320 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -74,6 +74,7 @@ const Workflows = (props) => { const [update, setUpdate] = React.useState("test"); const [deleteModalOpen, setDeleteModalOpen] = React.useState(false); const [editingWorkflow, setEditingWorkflow] = React.useState({}) + const [executionLoading, setExecutionLoading] = React.useState(false) const { start, stop } = useInterval({ duration: 5000, startImmediate: false, @@ -245,6 +246,7 @@ const Workflows = (props) => { } const getWorkflowExecution = (id) => { + setExecutionLoading(true) fetch(globalUrl+"/api/v1/workflows/"+id+"/executions", { method: 'GET', headers: { @@ -254,6 +256,7 @@ const Workflows = (props) => { credentials: "include", }) .then((response) => { + setExecutionLoading(false) if (response.status !== 200) { console.log("Status not 200 for WORKFLOW EXECUTION :O!") } @@ -275,6 +278,7 @@ const Workflows = (props) => { } }) .catch(error => { + setExecutionLoading(false) alert.error(error.toString()) }); } @@ -858,10 +862,17 @@ const Workflows = (props) => { ) } + return ( -

- There are no executiondetails yet. Click "execute" to run your first one. -

+ executionLoading ? +
+ +
+ : +

+ There are no executiondetails yet. Click "execute" to run your first one. +

+ ) } @@ -880,9 +891,14 @@ const Workflows = (props) => { ) } return ( -

- There are no executions for this workflow yet -

+ executionLoading ? +
+ +
+ : +

+ There are no executions for this workflow yet +

) } @@ -1111,107 +1127,113 @@ const Workflows = (props) => { workflowViewStyle.display = "none" } - const workflowView = workflows.length > 0 ? -
-
-
-
-

Workflows

+ const WorkflowView = () => { + if (workflows.length === 0) { + return ( +
+ +
+

Welcome to Shuffle

+
+
+

+ Shuffle is a flexible, easy to use, automation platform allowing users to integrate their services and devices freely. It's made to significantly reduce the amount of manual labor, and is focused on security applications. Click here to learn more. +

+
+
+ If you want to jump straight into it, click here to create your first workflow: +
+
+ +
+
-
- - - - {/* - - - - */} - - - - - - - upload = ref} onChange={importFiles} /> -
-
- + ) + } -
- {workflows.map((data, index) => { - return ( - - ) - })} -
-
-
-
-
-

Executions: {selectedWorkflow.name}

+ return ( +
+
+
+
+

Workflows

+
+
+ + + + {/* + + + + */} + + + + + + + upload = ref} onChange={importFiles} /> +
-
- -
-
- -
- -
-
-
-
-
-

Execution Timeline

-
-
- Collapse results
} - control={ {setCollapseJson(!collapseJson)}} />} - /> -
-
- -
- -
-
-
- : -
- -
-

Welcome to Shuffle

-
-
-

- Shuffle is a flexible, easy to use, automation platform allowing users to integrate their services and devices freely. It's made to significantly reduce the amount of manual labor, and is focused on security applications. Click here to learn more. -

-
-
- If you want to jump straight into it, click here to create your first workflow: -
-
- -
- -
+ +
+ {workflows.map((data, index) => { + return ( + + ) + })} +
+
+
+
+
+

Executions: {selectedWorkflow.name}

+
+
+ +
+
+ +
+ +
+
+
+
+
+

Execution Timeline

+
+
+ Collapse results
} + control={ {setCollapseJson(!collapseJson)}} />} + /> +
+
+ +
+ +
+
+
+ ) + } const importWorkflowsFromUrl = (url) => { console.log("IMPORT WORKFLOWS FROM ", downloadUrl) @@ -1384,13 +1406,17 @@ const Workflows = (props) => { const loadedCheck = isLoaded && isLoggedIn && workflowDone ?
- {workflowView} + {modalView} {deleteModal} {workflowDownloadModalOpen}
: -
+
+ + + Loading Workflows +
From 8355adb7b0cd1f4964c418ddd753b53c5e495b11 Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 6 Dec 2020 15:42:21 +0100 Subject: [PATCH 15/19] Fixed compose for push --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 014798f3..5ed77582 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,7 @@ services: depends_on: - backend backend: - build: ./backend + #build: ./backend image: ghcr.io/frikky/shuffle-backend:0.8.3 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} From feae9de6269fc479bf681f1d773b7c8b48d008c9 Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 6 Dec 2020 17:17:58 +0100 Subject: [PATCH 16/19] Majorly improved app loading from Github --- backend/go-app/main.go | 2 +- backend/go-app/walkoff.go | 106 +++++++++++++++++++++---- docker-compose.yml | 4 +- frontend/src/views/AngularWorkflow.jsx | 103 ++++++++++++------------ frontend/src/views/Workflows.jsx | 80 ++++++++++++------- 5 files changed, 196 insertions(+), 99 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 742d3b65..babee811 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -6530,7 +6530,7 @@ func handleAppHotload(location string, forceUpdate bool) error { } //log.Printf("Reading app folder: %#v", dir) - err = iterateAppGithubFolders(fs, dir, "", "", forceUpdate) + _, _, err = iterateAppGithubFolders(fs, dir, "", "", forceUpdate) if err != nil { log.Printf("Err: %s", err) return err diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 70307a82..6093edf6 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -1403,6 +1403,7 @@ func getWorkflows(resp http.ResponseWriter, request *http.Request) { q := datastore.NewQuery("workflow").Filter("owner =", user.Id) if user.Role == "admin" { q = datastore.NewQuery("workflow").Filter("org_id =", user.ActiveOrg.Id) + log.Printf("[INFO] Getting workflows (ADMIN) for organization %s", user.ActiveOrg.Id) } var workflows []Workflow @@ -2235,9 +2236,11 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // Handles check for required params if !found && param.Required { log.Printf("Appaction %s with required param %s doesn't exist.", action.Name, param.Name) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name))) - return + action.Errors = append(action.Errors, "Parameter %s is required", param.Name) + //newActions = append(newActions, action) + //resp.WriteHeader(401) + //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name))) + //return } } @@ -2252,6 +2255,12 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { workflow.IsValid = true log.Printf("Tags: %#v", workflow.Tags) + // FIXME: Is this too drastic? May lead to issues in the future. + // Should maybe make a copy for the old org. + if workflow.OrgId != user.ActiveOrg.Id { + workflow.OrgId = user.ActiveOrg.Id + } + err = setWorkflow(ctx, workflow, fileId) if err != nil { log.Printf("Failed saving workflow to database: %s", err) @@ -2277,7 +2286,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { Errors: workflow.Errors, } - log.Printf("Saved new version of workflow %s (%s)", workflow.Name, fileId) + log.Printf("Saved new version of workflow %s (%s) for org %s", workflow.Name, fileId, workflow.OrgId) resp.WriteHeader(200) newBody, err := json.Marshal(returndata) if err != nil { @@ -5379,11 +5388,23 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra return err } +type buildLaterStruct struct { + Tags []string + Extra string +} + // Onlyname is used to -func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate bool) error { +func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate bool) ([]buildLaterStruct, []buildLaterStruct, error) { var err error allapps := []WorkflowApp{} + reservedNames := []string{ + "OWA", + "NLP", + } + + buildLaterFirst := []buildLaterStruct{} + buildLaterList := []buildLaterStruct{} // It's here to prevent getting them in every iteration ctx := context.Background() @@ -5403,11 +5424,20 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin } // Go routine? Hmm, this can be super quick I guess - err = iterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate) + buildFirst, buildLast, err := iterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate) if err != nil { log.Printf("Error reading folder: %s", err) continue } + + for _, item := range buildFirst { + buildLaterFirst = append(buildLaterFirst, item) + } + + for _, item := range buildLast { + buildLaterList = append(buildLaterList, item) + } + case mode.IsRegular(): // Check the file filename := file.Name() @@ -5605,22 +5635,66 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin //log.Printf("Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion) - /// Only upload if successful and no errors - err = buildImageMemory(fs, tags, extra) - if err != nil { - log.Printf("Failed image build memory: %s", err) - } else { - if len(tags) > 0 { - log.Printf("Successfully built image %s", tags[0]) - } else { - log.Printf("Successfully built Docker image") + reservedFound := false + buildLater := buildLaterStruct{ + Tags: tags, + Extra: extra, + } + for _, appname := range reservedNames { + if strings.ToUpper(workflowapp.Name) == strings.ToUpper(appname) { + buildLaterList = append(buildLaterList, buildLater) + + reservedFound = true + break } } + + /// Only upload if successful and no errors + if !reservedFound { + buildLaterFirst = append(buildLaterFirst, buildLater) + } else { + log.Printf("\n\n[WARNING] Skipping build of %s to later\n\n", workflowapp.Name) + } } } } - return err + if len(buildLaterFirst) == 0 && len(buildLaterList) == 0 { + return buildLaterFirst, buildLaterList, err + } + + //log.Printf("BUILDLATERFIRST: %d, BUILDLATERLIST: %d", len(buildLaterFirst), len(buildLaterList)) + if len(extra) == 0 { + log.Printf("[INFO] Starting build of %d containers (FIRST)", len(buildLaterFirst)) + for _, item := range buildLaterFirst { + err = buildImageMemory(fs, item.Tags, item.Extra) + if err != nil { + log.Printf("Failed image build memory: %s", err) + } else { + if len(item.Tags) > 0 { + log.Printf("Successfully built image %s", item.Tags[0]) + } else { + log.Printf("Successfully built Docker image") + } + } + } + + log.Printf("Starting build of %d skipped docker images", len(buildLaterList)) + for _, item := range buildLaterList { + err = buildImageMemory(fs, item.Tags, item.Extra) + if err != nil { + log.Printf("Failed image build memory: %s", err) + } else { + if len(item.Tags) > 0 { + log.Printf("Successfully built image %s", item.Tags[0]) + } else { + log.Printf("Successfully built Docker image") + } + } + } + } + + return buildLaterFirst, buildLaterList, err } func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { diff --git a/docker-compose.yml b/docker-compose.yml index 5ed77582..62894b11 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: frontend: - #build: ./frontend + build: ./frontend image: ghcr.io/frikky/shuffle-frontend:0.8.3 container_name: shuffle-frontend hostname: shuffle-frontend @@ -16,7 +16,7 @@ services: depends_on: - backend backend: - #build: ./backend + build: ./backend image: ghcr.io/frikky/shuffle-backend:0.8.3 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index b07a0c17..e47c7c6c 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -191,7 +191,7 @@ const AngularWorkflow = (props) => { const cloudSyncEnabled = props.userdata !== undefined && props.userdata.active_org !== null && props.userdata.active_org !== undefined ? props.userdata.active_org.cloud_sync === true : false //const triggerEnvironments = cloudSyncEnabled ? ["cloud", "onprem"] : environments - const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" + const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" const triggerEnvironments = isCloud ? ["cloud"] : ["cloud", "onprem"] const unloadText = 'Are you sure you want to leave without saving (CTRL+S)?' @@ -1334,6 +1334,7 @@ const AngularWorkflow = (props) => { } else { setEnvironments({"name": "Onprem", "type": "onprem"}) } + return } @@ -4677,7 +4678,6 @@ const AngularWorkflow = (props) => { placeholder={selectedTrigger.label} onChange={selectedTriggerChange} /> - {showEnvironment ?
Environment @@ -4725,7 +4725,6 @@ const AngularWorkflow = (props) => { })}
- : null}
@@ -5221,59 +5220,57 @@ const AngularWorkflow = (props) => { placeholder={selectedTrigger.label} onChange={selectedTriggerChange} /> - {showEnvironment ? -
- - Environment - - { - selectedTrigger.environment = e.target.value - setSelectedTrigger(selectedTrigger) - if (e.target.value === "cloud") { - console.log("Set cloud config") - workflow.triggers[selectedTriggerIndex].parameters[0].value = "*/2 * * * *" + } + }} + fullWidth + onChange={(e) => { + selectedTrigger.environment = e.target.value + setSelectedTrigger(selectedTrigger) + if (e.target.value === "cloud") { + console.log("Set cloud config") + workflow.triggers[selectedTriggerIndex].parameters[0].value = "*/2 * * * *" - //var tmpvalue = workflow.triggers[selectedTriggerIndex].parameters[0].value.split("/") - //const urlpath = tmpvalue.slice(3, tmpvalue.length) - //const newurl = "https://shuffler.io/"+urlpath.join("/") - //workflow.triggers[selectedTriggerIndex].parameters[0].value = newurl - } else { - console.log("Set cloud config") - //var tmpvalue = workflow.triggers[selectedTriggerIndex].parameters[0].value.split("/") - //const urlpath = tmpvalue.slice(3, tmpvalue.length) - //const newurl = window.location.origin+"/"+urlpath.join("/") - workflow.triggers[selectedTriggerIndex].parameters[0].value = "120" - } + //var tmpvalue = workflow.triggers[selectedTriggerIndex].parameters[0].value.split("/") + //const urlpath = tmpvalue.slice(3, tmpvalue.length) + //const newurl = "https://shuffler.io/"+urlpath.join("/") + //workflow.triggers[selectedTriggerIndex].parameters[0].value = newurl + } else { + console.log("Set cloud config") + //var tmpvalue = workflow.triggers[selectedTriggerIndex].parameters[0].value.split("/") + //const urlpath = tmpvalue.slice(3, tmpvalue.length) + //const newurl = window.location.origin+"/"+urlpath.join("/") + workflow.triggers[selectedTriggerIndex].parameters[0].value = "120" + } - setWorkflow(workflow) - setUpdate(Math.random()) - }} - style={{backgroundColor: inputColor, color: "white", height: "50px"}} - > - {triggerEnvironments.map(data => { - if (data.archived) { - return null - } - - return ( - - {data} - - ) - })} - -
- : null} + setWorkflow(workflow) + setUpdate(Math.random()) + }} + style={{backgroundColor: inputColor, color: "white", height: "50px"}} + > + {triggerEnvironments.map(data => { + if (data.archived) { + return null + } + + return ( + + {data} + + ) + })} + +
diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 98583320..56ef2f86 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -15,6 +15,7 @@ import FormControlLabel from '@material-ui/core/FormControlLabel'; import Chip from '@material-ui/core/Chip'; import Switch from '@material-ui/core/Switch'; import Typography from '@material-ui/core/Typography'; +import Zoom from '@material-ui/core/Zoom'; import CircularProgress from '@material-ui/core/CircularProgress'; import CachedIcon from '@material-ui/icons/Cached'; @@ -365,6 +366,7 @@ const Workflows = (props) => { } } data["org"] = [] + data["org_id"] = "" data.execution_org = {"id": ""} console.log(data) @@ -1028,8 +1030,19 @@ const Workflows = (props) => { }, }} > + +
+ {editingWorkflow.id !== undefined ? "Editing" : "New"} workflow +
+ + + +
+
+ -
{editingWorkflow.id !== undefined ? "Editing" : "New"} workflow
 setNewWorkflowName(event.target.value)} @@ -1127,6 +1140,35 @@ const Workflows = (props) => { workflowViewStyle.display = "none" } + const workflowButtons = + + {workflows.length > 0 ? + + + + : null} + + + + upload = ref} onChange={importFiles} /> + {workflows.length > 0 ? + + + + : null} + + + + + const WorkflowView = () => { if (workflows.length === 0) { return ( @@ -1143,15 +1185,21 @@ const Workflows = (props) => {
If you want to jump straight into it, click here to create your first workflow:
-
+
+ + + ..OR + + {workflowButtons} +
) } - return ( + return (
@@ -1159,29 +1207,7 @@ const Workflows = (props) => {

Workflows

- - - - {/* - - - - */} - - - - - - - upload = ref} onChange={importFiles} /> + {workflowButtons}
@@ -1189,7 +1215,7 @@ const Workflows = (props) => {
{workflows.map((data, index) => { return ( - + ) })}
From cb01fd2cc722c3eaae4bb89b0fedcc8cc8a43d86 Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 6 Dec 2020 17:44:52 +0100 Subject: [PATCH 17/19] Last cleanup for master --- backend/go-app/walkoff.go | 6 +++++- docker-compose.yml | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 6093edf6..1d0deea1 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -5391,6 +5391,7 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra type buildLaterStruct struct { Tags []string Extra string + Id string } // Onlyname is used to @@ -5635,11 +5636,14 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin //log.Printf("Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion) - reservedFound := false + // ID can be used to e.g. set a build status. buildLater := buildLaterStruct{ Tags: tags, Extra: extra, + Id: workflowapp.ID, } + + reservedFound := false for _, appname := range reservedNames { if strings.ToUpper(workflowapp.Name) == strings.ToUpper(appname) { buildLaterList = append(buildLaterList, buildLater) diff --git a/docker-compose.yml b/docker-compose.yml index 62894b11..5ed77582 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: frontend: - build: ./frontend + #build: ./frontend image: ghcr.io/frikky/shuffle-frontend:0.8.3 container_name: shuffle-frontend hostname: shuffle-frontend @@ -16,7 +16,7 @@ services: depends_on: - backend backend: - build: ./backend + #build: ./backend image: ghcr.io/frikky/shuffle-backend:0.8.3 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} From 7474ad4ced75b613aeb9486ca4109452994f153e Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 6 Dec 2020 17:55:50 +0100 Subject: [PATCH 18/19] BUG: Fixed issue with wrong owner of workflow for file uploads --- backend/go-app/walkoff.go | 3 +++ docker-compose.yml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 1d0deea1..b1da0f70 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -2258,7 +2258,10 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // FIXME: Is this too drastic? May lead to issues in the future. // Should maybe make a copy for the old org. if workflow.OrgId != user.ActiveOrg.Id { + log.Printf("[WARNING] Editing workflow to be owned by %s", user.ActiveOrg.Id) workflow.OrgId = user.ActiveOrg.Id + workflow.ExecutingOrg = user.ActiveOrg + workflow.Org = append(workflow.Org, user.ActiveOrg) } err = setWorkflow(ctx, workflow, fileId) diff --git a/docker-compose.yml b/docker-compose.yml index 5ed77582..014798f3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,7 @@ services: depends_on: - backend backend: - #build: ./backend + build: ./backend image: ghcr.io/frikky/shuffle-backend:0.8.3 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} From b13c3f3765f070f9e501dee0752ed853adf911b4 Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 6 Dec 2020 18:05:30 +0100 Subject: [PATCH 19/19] Last compose cleanup --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 014798f3..5ed77582 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,7 @@ services: depends_on: - backend backend: - build: ./backend + #build: ./backend image: ghcr.io/frikky/shuffle-backend:0.8.3 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME}