diff --git a/.env b/.env index 93b05920..a43c6669 100644 --- a/.env +++ b/.env @@ -19,7 +19,9 @@ SHUFFLE_DEFAULT_PASSWORD= SHUFFLE_DEFAULT_APIKEY= # Local location of your app directory. Can't use ~/ +# Files will get better at some point. Right now: local saving. SHUFFLE_APP_HOTLOAD_LOCATION=./shuffle-apps +SHUFFLE_FILE_LOCATION=./shuffle-files # Other configs BACKEND_HOSTNAME=shuffle-backend diff --git a/backend/Dockerfile b/backend/Dockerfile index 21422785..1856d84f 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -8,6 +8,7 @@ ADD ./go-app/main.go /app ADD ./go-app/walkoff.go /app ADD ./go-app/docker.go /app ADD ./go-app/codegen.go /app +ADD ./go-app/files.go /app ADD ./go-app/go.mod /app diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 18a42a32..fffef6c3 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -1,4 +1,5 @@ import os +import copy import sys import re import time @@ -51,6 +52,139 @@ class AppBase: self.logger.info("Result: %d" % ret.status_code) if ret.status_code != 200: self.logger.info(ret.text) + + # Things to consider for files: + # - How can you download / stream a file? + # - Can you decide if you want a stream or the files directly? + def get_file(self, value): + full_execution = self.full_execution + org_id = full_execution["workflow"]["execution_org"]["id"] + + print("SHOULD GET FILES BASED ON ORG %s, workflow %s and value(s) %s" % (org_id, full_execution["workflow"]["id"], value)) + + if isinstance(value, list): + print("IS LIST!") + #if len(value) == 1: + # value = value[0] + else: + value = [value] + + returns = [] + for item in value: + print("VALUE: %s" % item) + if len(item) != 36: + print("Bad length for value") + continue + #return { + # "filename": "", + # "data": "", + # "success": False, + #} + + get_path = "/api/v1/files/%s?execution_id=%s" % (item, full_execution["execution_id"]) + headers = { + "Content-Type": "application/json", + "Authorization": "Bearer %s" % self.authorization + } + + ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers) + print("RET1: %s" % ret1.text) + if ret1.status_code != 200: + returns.append({ + "filename": "", + "data": "", + "success": False, + }) + continue + + content_path = "/api/v1/files/%s/content?execution_id=%s" % (item, full_execution["execution_id"]) + ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers) + print("Ret2: %s" % ret2.text) + if ret2.status_code == 200: + tmpdata = ret1.json() + returndata = { + "success": True, + "filename": tmpdata["filename"], + "data": ret2.content, + } + returns.append(returndata) + + if len(returns) == 0: + return { + "success": False, + "filename": "", + "data": b"", + } + elif len(returns) == 1: + return returns[0] + else: + return returns + + # Sets files in the backend + def set_files(self, infiles): + full_execution = self.full_execution + workflow_id = full_execution["workflow"]["id"] + org_id = full_execution["workflow"]["execution_org"]["id"] + headers = { + "Content-Type": "application/json", + "Authorization": "Bearer %s" % self.authorization + } + + create_path = "/api/v1/files/create?execution_id=%s" % full_execution["execution_id"] + file_ids = [] + for curfile in infiles: + filename = "unspecified" + data = { + "filename": filename, + "workflow_id": workflow_id, + "org_id": org_id, + } + + try: + data["filename"] = curfile["filename"] + filename = curfile["filename"] + except KeyError as e: + print("KeyError in file setup: %s" % e) + pass + + ret = requests.post("%s%s" % (self.url, create_path), headers=headers, json=data) + print("Ret CREATE: %s" % ret.text) + cur_id = "" + if ret.status_code == 200: + print("RET: %s" % ret.text) + ret_json = ret.json() + if not ret_json["success"]: + print("Not success in file upload creation.") + continue + + print("Should handle ID %s" % ret_json["id"]) + file_ids.append(ret_json["id"]) + cur_id = ret_json["id"] + else: + print("Bad status code: %d" % ret.status_code) + continue + + if len(cur_id) == 0: + print("No file ID specified from backend") + continue + + new_headers = { + "Authorization": "Bearer %s" % self.authorization, + } + + upload_path = "/api/v1/files/%s/upload?execution_id=%s" % (cur_id, full_execution["execution_id"]) + print("Create path: %s" % create_path) + + # FIXME: Typical failure here if data is returned badly formatted + files={"shuffle_file": (filename, curfile["data"])} + #open(filename,'rb')} + + ret = requests.post("%s%s" % (self.url, upload_path), files=files, headers=new_headers) + print("Ret UPLOAD: %s" % ret.text) + print("Ret2 UPLOAD: %d" % ret.status_code) + + print("IDS TO RETURN: %s" % file_ids) + return file_ids async def execute_action(self, action): # FIXME - add request for the function STARTING here. Use "results stream" or something @@ -148,6 +282,7 @@ class AppBase: print("") + self.full_execution = fullexecution self.logger.info("AFTER FULLEXEC stream result") # Gets the value at the parenthesis level you want @@ -207,7 +342,7 @@ class AppBase: try: return int(data) except ValueError: - print("ValueError while casting %s" % data) + print("ValueError while casting %s to int" % data) return data if "lower" in thistype: return data.lower() @@ -219,15 +354,19 @@ class AppBase: return data.strip() if "split" in thistype: return data.split() - if "len" in thistype or "length" in thistype: + if "len" in thistype or "length" in thistype or "lenght" in thistype: tmp = "" try: - tmp = json.loads(data) + tmpdata = data.replace("\'", "\"") + tmp = json.loads(tmpdata) except: + print("Passing bug") pass if isinstance(tmp, list): - return str(len(tmp)) + return len(tmp) + elif isinstance(tmp, object): + return len(tmp) return str(len(data)) if "parse" in thistype: @@ -273,7 +412,7 @@ class AppBase: #print("Running %s" % data) # Look for the INNER wrapper first, then move out - wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length"] + wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length", "lenght"] found = False for wrapper in wrappers: if wrapper not in data.lower(): @@ -303,6 +442,7 @@ class AppBase: continue parsed_value = parse_type(innervalue[0], thistype.lower()) + print("Parsed value from %s: %s" % (thistype, parsed_value)) return parsed_value print("DATA: %s\n" % data) @@ -403,6 +543,7 @@ class AppBase: # Magical way of returning which makes app sdk identify # it as multi execution return newvalue, True + elif len(actualitem) > 0: # FIXME: This is absolutely not perfect. print("In recursion v2: ", actualitem) @@ -478,7 +619,7 @@ class AppBase: #Actionname: Start_node - print(f"Actionname: {actionname_lower}") + print(f"\nActionname: {actionname_lower}") # 1. Find the action baseresult = "" @@ -549,21 +690,29 @@ class AppBase: except KeyError as error: print(f"KeyError in JSON: {error}") - print(f"After first trycatch") + print(f"After first trycatch. Baseresult: ", baseresult) # 2. Find the JSON data if len(baseresult) == 0: return ""+appendresult, False + print("After second return") if len(parsersplit) == 1: return str(baseresult)+str(appendresult), False baseresult = baseresult.replace("\'", "\"") + baseresult = baseresult.replace(" True,", " true,") + baseresult = baseresult.replace(" False", " false,") + + print("After third parser return - Formatted: ", baseresult) basejson = {} try: basejson = json.loads(baseresult) except json.decoder.JSONDecodeError as e: + print("Parser issue with JSON: %s" % e) return str(baseresult)+str(appendresult), False + + print("After fourth parser return as JSON") data, is_loop = recurse_json(basejson, parsersplit[1:]) parseditem = data @@ -577,6 +726,7 @@ class AppBase: print("SET DATA WRAPPER TO %s!" % parsersplit[-1]) parseditem = "${%s%s}$" % (parsersplit[-1], json.dumps(data)) + print("Before last return with %s" % appendresult) return str(parseditem)+str(appendresult), is_loop # Parses parameters sent to it and returns whether it did it successfully with the values found @@ -824,26 +974,6 @@ class AppBase: return True, "" - # Things to consider for files: - # - How can you download / stream a file? - # - Can you decide if you want a stream or the files directly? - def get_files(full_execution, value): - print("FULL EXEC: %s" % full_execution) - org_id = full_execution["workflow"]["execution_org"]["id"] - print("SHOULD GET FILES BASED ON ORG %s, workflow %s and value(s) %s" % (org_id, full_execution["workflow"]["id"], value)) - get_path = "/api/v1/files/%s/content?execution_id=%s" % (value, full_execution["execution_id"]) - - print("PATH: %s" % get_path) - headers = { - "Content-Type": "application/json", - "Authorization": "Bearer %s" % self.authorization - } - ret = requests.get("%s%s" % (self.url, get_path), headers=headers) - print("RET CONTENT: %s" % ret.text) - print("RET CODE FILE: %d" % ret.status_code) - - # r.HandleFunc("/api/v1/files/{fileId}/content", handleGetFileContent).Methods("GET", "OPTIONS") - # Checks whether conditions are met, otherwise set branchcheck, tmpresult = check_branch_conditions(action, fullexecution) if not branchcheck: @@ -911,15 +1041,6 @@ class AppBase: multiexecution = False multi_execution_lists = [] for parameter in action["parameters"]: - is_file = False - try: - if parameter["schema"]["type"] == "file": - print("SHOULD HANDLE FILE. Get based on value %s" % parameter["value"]) - get_files(fullexecution, parameter["value"]) - is_file = True - except KeyError as e: - print("SCHEMA ERROR: %s" % e) - check, value, is_loop = parse_params(action, fullexecution, parameter) if check: raise "Value check error: %s" % Exception(check) @@ -967,13 +1088,40 @@ class AppBase: minlength = len(json_replacement) tmpitem = tmpitem.replace(actualitem[0][0], replacement, 1) - params[parameter["name"]] = tmpitem - multi_execution_lists.append(json_replacement) - multi_parameters[parameter["name"]] = json_replacement - #print("LENGTH OF ARR: %d" % len(resultarray)) - #print("RESULTARRAY: %s" % resultarray) - print("MULTI finished: %s" % replacement) + # This code handles files. + print("(1) ------------ PARAM: %s" % parameter["schema"]["type"]) + resultarray = [] + isfile = False + try: + if parameter["schema"]["type"] == "file" and len(value) > 0: + print("(1) SHOULD HANDLE FILE IN MULTI. Get based on value %s" % tmpitem) + # This is silly :) + # Q: Is there something wrong with the download system? + # It seems to return "FILE CONTENT: %s" with the ID as %s + for tmp_file_split in json.loads(tmpitem): + print("(1) PRE GET FILE %s" % tmp_file_split) + file_value = self.get_file(tmp_file_split) + print("(1) POST AWAIT %s" % file_value) + resultarray.append(file_value) + print("(1) FILE VALUE FOR VAL %s: %s" % (tmp_file_split, file_value)) + + isfile = True + except KeyError as e: + print("(1) SCHEMA ERROR IN FILE HANDLING: %s" % e) + except json.decoder.JSONDecodeError as e: + print("(1) JSON ERROR IN FILE HANDLING: %s" % e) + + if not isfile: + params[parameter["name"]] = tmpitem + multi_parameters[parameter["name"]] = json_replacement + else: + print("Resultarray: %s" % resultarray) + params[parameter["name"]] = resultarray + multi_parameters[parameter["name"]] = resultarray + + multi_execution_lists.append(json_replacement) + print("MULTI finished: %s" % json_replacement) else: # This is here to handle for loops within variables.. kindof # 1. Find the length of the longest array @@ -1017,7 +1165,30 @@ class AppBase: #replacement = parse_wrapper_start(replacement) tmpitem = tmpitem.replace(key, replacement, -1) - resultarray.append(tmpitem) + + # This code handles files. + print("(2) ------------ PARAM: %s" % parameter["schema"]["type"]) + isfile = False + try: + if parameter["schema"]["type"] == "file" and len(value) > 0: + print("(2) SHOULD HANDLE FILE IN MULTI. Get based on value %s" % parameter["value"]) + + for tmp_file_split in json.loads(parameter["value"]): + print("(2) PRE GET FILE %s" % tmp_file_split) + file_value = self.get_file(tmp_file_split) + print("(2) POST AWAIT %s" % file_value) + resultarray.append(file_value) + print("(2) FILE VALUE FOR VAL %s: %s" % (tmp_file_split, file_value)) + + + isfile = True + except KeyError as e: + print("(2) SCHEMA ERROR IN FILE HANDLING: %s" % e) + except json.decoder.JSONDecodeError as e: + print("(2) JSON ERROR IN FILE HANDLING: %s" % e) + + if not isfile: + resultarray.append(tmpitem) # With this parameter ready, add it to... a greater list of parameters. Rofl print("LENGTH OF ARR: %d" % len(resultarray)) @@ -1035,6 +1206,18 @@ class AppBase: params[parameter["name"]] = value multi_parameters[parameter["name"]] = value + # This code handles files. + try: + if parameter["schema"]["type"] == "file" and len(value) > 0: + print("\n SHOULD HANDLE FILE. Get based on value %s. <--- is this a valid ID?" % parameter["value"]) + file_value = self.get_file(value) + print("FILE VALUE: %s \n" % file_value) + + params[parameter["name"]] = file_value + multi_parameters[parameter["name"]] = file_value + except KeyError as e: + print("SCHEMA ERROR IN FILE HANDLING: %s" % e) + # Fix lists here print("CHECKING multi execution list!") if len(multi_execution_lists) > 0: @@ -1044,28 +1227,58 @@ class AppBase: if listitem in filteredlist: continue - filteredlist.append(listitem) + # FIXME: Subsub required?. Recursion! + # Basically multiply what we have with the outer loop? + # + if isinstance(listitem, list): + for subitem in listitem: + filteredlist.append(subitem) + else: + filteredlist.append(listitem) #print("New list length: %d" % len(filteredlist)) if len(filteredlist) > 1: print("Calculating new multi-loop length with %d lists" % len(filteredlist)) tmplength = 1 for innerlist in filteredlist: - print("List length: %d. %d*%d" % (len(innerlist), len(innerlist), tmplength)) tmplength = len(innerlist)*tmplength + print("List length: %d. %d*%d" % (tmplength, len(innerlist), tmplength)) minlength = tmplength print("New multi execution length: %d\n" % tmplength) - # FIXME - this is horrible, but works for now - #for i in range(calltimes): if not multiexecution: print("APP_SDK DONE: Starting NORMAL execution of function") - print("Running with params %s" % params) + print("Running with params (0): %s" % params) newres = await func(**params) - print("Return from execution: %s" % newres) - if isinstance(newres, str): + print("Returned from execution.") + if isinstance(newres, tuple): + print("Handling return as tuple") + # Handles files. + filedata = "" + file_ids = [] + print("TUPLE: %s" % newres[1]) + if isinstance(newres[1], list): + print("HANDLING LIST FROM RET") + file_ids = self.set_files(newres[1]) + elif isinstance(newres[1], object): + print("Handling JSON from ret") + file_ids = self.set_files([newres[1]]) + elif isinstance(newres[1], str): + print("Handling STRING from ret") + file_ids = self.set_files([newres[1]]) + else: + print("NO FILES TO HANDLE") + + tmp_result = { + "result": newres[0], + "file_ids": file_ids + } + + result = json.dumps(tmp_result) + elif isinstance(newres, str): + print("Handling return as string") result += newres else: try: @@ -1073,16 +1286,22 @@ class AppBase: except ValueError: result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres) print("Can't handle type %s value from function" % (type(newres))) + print("POST NEWRES RESULT: ", result) else: - print("APP_SDK DONE: Starting MULTI execution with values %s of length %d" % (multi_parameters, minlength)) + print("APP_SDK DONE: Starting MULTI execution (length: %d) with values %s" % (minlength, multi_parameters)) # 1. Use number of executions based on the arrays being similar # 2. Find the right value from the parsed multi_params results = [] json_object = False for i in range(0, minlength): # To be able to use the results as a list: - baseparams = json.loads(json.dumps(multi_parameters)) + print("1: %s" % multi_parameters) + #baseparams = json.loads(json.dumps(multi_parameters)) + baseparams = copy.deepcopy(multi_parameters) + + print("2: %s: %s" % (type(baseparams), baseparams)) + # {'call': ['GoogleSafebrowsing_2_0', 'VirusTotal_GetReport_3_0']} # 1. Check if list length is same as minlength # 2. If NOT same length, duplicate based on length of array @@ -1093,7 +1312,7 @@ class AppBase: try: firstlist = True for key, value in baseparams.items(): - + print("Itemtype: %s" % type(value)) if isinstance(value, list): try: newvalue = value[i] @@ -1136,6 +1355,8 @@ class AppBase: firstlist = False baseparams[key] = newvalue + + print("3") except IndexError as e: print("IndexError: %s" % e) baseparams[key] = "IndexError: %s" % e @@ -1143,7 +1364,9 @@ class AppBase: print("KeyError: %s" % e) baseparams[key] = "KeyError: %s" % e - print("Running with params %s" % baseparams) + + print("4") + print("Running with params (1): %s" % baseparams) ret = await func(**baseparams) print("Return from execution: %s" % ret) if isinstance(ret, dict) or isinstance(ret, list): diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 6c28cc03..4532a20e 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,6 +1,6 @@ #!/bin/bash NAME=shuffle-app_sdk -VERSION=0.8.0 +VERSION=0.8.2 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index a68e1643..b0398045 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -181,11 +181,11 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin // Dockerfile is inside the TAR itself. Not local context // docker build --build-arg http_proxy=http://my.proxy.url buildOptions := types.ImageBuildOptions{ - Remove: true, - Tags: tags, - BuildArgs: map[string]*string{}, - NetworkMode: "host", + Remove: true, + Tags: tags, + BuildArgs: map[string]*string{}, } + // NetworkMode: "host", httpProxy := os.Getenv("HTTP_PROXY") if len(httpProxy) > 0 { @@ -244,11 +244,11 @@ func buildImage(tags []string, dockerfileFolder string) error { dockerFileTarReader := bytes.NewReader(buf.Bytes()) buildOptions := types.ImageBuildOptions{ - Remove: true, - Tags: tags, - BuildArgs: map[string]*string{}, - NetworkMode: "host", + Remove: true, + Tags: tags, + BuildArgs: map[string]*string{}, } + //NetworkMode: "host", httpProxy := os.Getenv("HTTP_PROXY") if len(httpProxy) > 0 { diff --git a/backend/go-app/files.go b/backend/go-app/files.go new file mode 100644 index 00000000..a2856e2b --- /dev/null +++ b/backend/go-app/files.go @@ -0,0 +1,764 @@ +package main + +/* + Handles files within Workflows.of Shuffle +*/ + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "strconv" + "strings" + "time" + + "cloud.google.com/go/datastore" + "github.com/satori/go.uuid" +) + +type File struct { + Id string `json:"id" datastore:"id"` + Type string `json:"type" datastore:"type"` + CreatedAt int64 `json:"created_at" datastore:"created_at"` + UpdatedAt int64 `json:"updated_at" datastore:"updated_at"` + MetaAccessAt int64 `json:"meta_access_at" datastore:"meta_access_at"` + DownloadAt int64 `json:"last_downloaded" datastore:"last_downloaded"` + Description string `json:"description" datastore:"description"` + ExpiresAt string `json:"expires_at" datastore:"expires_at"` + Status string `json:"status" datastore:"status"` + Filename string `json:"filename" datastore:"filename"` + URL string `json:"url" datastore:"org"` + OrgId string `json:"org_id" datastore:"org_id"` + WorkflowId string `json:"workflow_id" datastore:"workflow_id"` + Workflows []string `json:"workflows" datastore:"workflows"` + DownloadPath string `json:"download_path" datastore:"download_path"` + Md5sum string `json:"md5_sum" datastore:"md5_sum"` + Sha256sum string `json:"sha256_sum" datastore:"sha256_sum"` +} + +var basepath = os.Getenv("SHUFFLE_FILE_LOCATION") + +func fileAuthentication(request *http.Request) (string, error) { + executionId, ok := request.URL.Query()["execution_id"] + if ok && len(executionId) > 0 { + ctx := context.Background() + workflowExecution, err := getWorkflowExecution(ctx, executionId[0]) + if err != nil { + log.Printf("[ERROR] Couldn't find execution ID %s", executionId[0]) + return "", err + } + + apikey := request.Header.Get("Authorization") + if !strings.HasPrefix(apikey, "Bearer ") { + log.Printf("[ERROR} Apikey doesn't start with bearer (2)") + return "", errors.New("No auth key found") + } + + apikeyCheck := strings.Split(apikey, " ") + if len(apikeyCheck) != 2 { + log.Printf("[ERROR] Invalid format for apikey (2)") + return "", errors.New("No space in authkey") + } + + // This is annoying af and is done because of maxlength lol + newApikey := apikeyCheck[1] + if newApikey != workflowExecution.Authorization { + //log.Printf("[ERROR] Bad apikey for execution %s. %s vs %s", executionId[0], apikey, workflowExecution.Authorization) + log.Printf("[ERROR] Bad apikey for execution %s.", executionId[0]) + //%s vs %s", executionId[0], apikey, workflowExecution.Authorization) + return "", errors.New("Bad authorization key") + } + + log.Printf("[INFO] Authorization is correct for execution %s!", executionId[0]) + //%s vs %s. Setting Org", executionId, apikey, workflowExecution.Authorization) + if len(workflowExecution.ExecutionOrg) > 0 { + return workflowExecution.ExecutionOrg, nil + } else if len(workflowExecution.Workflow.ExecutingOrg.Id) > 0 { + return workflowExecution.ExecutionOrg, nil + } else { + log.Printf("[ERROR] Couldn't find org for workflow execution, but auth was correct.") + } + } + + return "", errors.New("No execution id specified") +} + +// https://golangcode.com/check-if-a-file-exists/ +func fileExists(filename string) bool { + info, err := os.Stat(filename) + if os.IsNotExist(err) { + return false + } + return !info.IsDir() +} + +func handleGetFileMeta(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + var fileId string + location := strings.Split(request.URL.String(), "/") + if location[1] == "api" { + if len(location) <= 4 { + log.Printf("[INFO] Path too short: %d", len(location)) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + fileId = location[4] + } + + if strings.Contains(fileId, "?") { + fileId = strings.Split(fileId, "?")[0] + } + + if len(fileId) != 36 { + log.Printf("Bad format for fileId %s", fileId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`)) + return + } + + log.Printf("\n\n[INFO] User is trying to GET File Meta for %s\n\n", fileId) + + // 1. Check user directly + // 2. Check workflow execution authorization + user, err := handleApiAuthentication(resp, request) + if err != nil { + log.Printf("[INFO] INITIAL Api authentication failed in file deletion: %s", err) + + orgId, err := fileAuthentication(request) + if err != nil { + log.Printf("[ERROR] Bad file authentication in get: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + user.ActiveOrg.Id = orgId + user.Username = "Execution File API" + } + + // 1. Verify if the user has access to the file: org_id and workflow + log.Printf("[INFO] Should GET FILE META for %s if user has access", fileId) + ctx := context.Background() + file, err := getFile(ctx, fileId) + if err != nil { + log.Printf("[INFO] File %s not found: %s", fileId, err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + found := false + if file.OrgId == user.ActiveOrg.Id { + found = true + } else { + for _, item := range user.Orgs { + if item == file.OrgId { + found = true + break + } + } + } + + if !found { + log.Printf("[INFO] User %s doesn't have access to %s", user.Username, fileId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + newBody, err := json.Marshal(file) + if err != nil { + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false, "reason": "Failed to marshal filedata"}`)) + return + } + + log.Printf("[INFO] Successfully got file meta for %s", fileId) + resp.WriteHeader(200) + resp.Write([]byte(newBody)) +} + +func handleDeleteFile(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + var fileId string + location := strings.Split(request.URL.String(), "/") + if location[1] == "api" { + if len(location) <= 4 { + log.Printf("[INFO] Path too short: %d", len(location)) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + fileId = location[4] + } + + if strings.Contains(fileId, "?") { + fileId = strings.Split(fileId, "?")[0] + } + + if len(fileId) != 36 { + log.Printf("Bad format for fileId %s", fileId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`)) + return + } + + log.Printf("\n\n[INFO] User is trying to delete file %s\n\n", fileId) + + // 1. Check user directly + // 2. Check workflow execution authorization + user, err := handleApiAuthentication(resp, request) + if err != nil { + log.Printf("[INFO] INITIAL Api authentication failed in file deletion: %s", err) + + orgId, err := fileAuthentication(request) + if err != nil { + log.Printf("[ERROR] Bad file authentication in get: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + user.ActiveOrg.Id = orgId + user.Username = "Execution File API" + } + + // 1. Verify if the user has access to the file: org_id and workflow + log.Printf("[INFO] Should DELETE file %s if user has access", fileId) + ctx := context.Background() + file, err := getFile(ctx, fileId) + if err != nil { + log.Printf("[INFO] File %s not found: %s", fileId, err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + found := false + if file.OrgId == user.ActiveOrg.Id { + found = true + } else { + for _, item := range user.Orgs { + if item == file.OrgId { + found = true + break + } + } + } + + if !found { + log.Printf("[INFO] User %s doesn't have access to %s", user.Username, fileId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if file.Status == "deleted" { + log.Printf("[INFO] File with ID %s is already deleted.", fileId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if fileExists(file.DownloadPath) { + err = os.Remove(file.DownloadPath) + if err != nil { + log.Printf("[ERROR] Failed deleting file locally: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed deleting filein path %s"}`, file.DownloadPath))) + return + } + + log.Printf("[INFO] Deleted file %s locally. Next is database.", file.DownloadPath) + } else { + log.Printf("[ERROR] File doesn't exist. Can't delete. Should maybe delete file anyway?") + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "File in location %s doesn't exist"}`, file.DownloadPath))) + return + } + + file.Status = "deleted" + err = setFile(ctx, *file) + if err != nil { + log.Printf("[ERROR] Failed setting file to deleted") + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false, "reason": "Failed setting file to deleted"}`)) + return + } + + /* + //Actually delete it? + err = DeleteKey(ctx, "files", fileId) + if err != nil { + log.Printf("Failed deleting file with ID %s: %s", fileId, err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + */ + + log.Printf("[INFO] Successfully deleted file %s", fileId) + resp.WriteHeader(200) + resp.Write([]byte(`{"success": true}`)) +} + +func handleGetFileContent(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + var fileId string + location := strings.Split(request.URL.String(), "/") + if location[1] == "api" { + if len(location) <= 4 { + log.Printf("Path too short: %d", len(location)) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + fileId = location[4] + } + + if len(fileId) != 36 { + log.Printf("Bad format for fileId %s", fileId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`)) + return + } + + log.Printf("\n\nUser is trying to download file %s\n\n", fileId) + + // 1. Check user directly + // 2. Check workflow execution authorization + user, err := handleApiAuthentication(resp, request) + if err != nil { + log.Printf("INITIAL Api authentication failed in file download: %s", err) + + orgId, err := fileAuthentication(request) + if err != nil { + log.Printf("Bad file authentication in get: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + user.ActiveOrg.Id = orgId + user.Username = "Execution File API" + /* + } else { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + */ + } + + // 1. Verify if the user has access to the file: org_id and workflow + log.Printf("[INFO] Should get file %s", fileId) + ctx := context.Background() + file, err := getFile(ctx, fileId) + if err != nil { + log.Printf("[ERROR] File %s not found: %s", fileId, err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + found := false + if file.OrgId == user.ActiveOrg.Id { + found = true + } else { + for _, item := range user.Orgs { + if item == file.OrgId { + found = true + break + } + } + } + + if !found { + log.Printf("User %s doesn't have access to %s", user.Username, fileId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if file.Status != "active" { + log.Printf("[ERROR] File status isn't active, but %s. Can't continue.", file.Status) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "The file isn't ready to be downloaded yet. Status required: active"}`)) + return + } + + // Fixme: More auth: org and workflow! + downloadPath := file.DownloadPath + log.Printf("[INFO] Downloadpath: %s", downloadPath) + Openfile, err := os.Open(downloadPath) + defer Openfile.Close() //Close after function return + if err != nil { + //File not found, send 404 + http.Error(resp, "File not found.", 404) + return + } + + //File is found, create and send the correct headers + //Get the Content-Type of the file + //Create a buffer to store the header of the file in + FileHeader := make([]byte, 512) + //Copy the headers into the FileHeader buffer + Openfile.Read(FileHeader) + //Get content type of file + FileContentType := http.DetectContentType(FileHeader) + + //Get the file size + FileStat, _ := Openfile.Stat() //Get info from file + FileSize := strconv.FormatInt(FileStat.Size(), 10) //Get file size as a string + + //Send the headers + resp.Header().Set("Content-Disposition", "attachment; filename="+fileId) + resp.Header().Set("Content-Type", FileContentType) + resp.Header().Set("Content-Length", FileSize) + + //Send the file + //We read 512 bytes from the file already, so we reset the offset back to 0 + Openfile.Seek(0, 0) + io.Copy(resp, Openfile) //'Copy' the file to the client + return + + //log.Printf("Should download file %s", downloadPath) +} +func handleUploadFile(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + var fileId string + location := strings.Split(request.URL.String(), "/") + if location[1] == "api" { + if len(location) <= 4 { + log.Printf("Path too short: %d", len(location)) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + fileId = location[4] + } + + if len(fileId) != 36 { + log.Printf("Bad format for fileId %s", fileId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`)) + return + } + + // 1. Check user directly + // 2. Check workflow execution authorization + user, err := handleApiAuthentication(resp, request) + if err != nil { + log.Printf("INITIAL Api authentication failed in file upload: %s", err) + + orgId, err := fileAuthentication(request) + if err != nil { + log.Printf("Bad file authentication in create file: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + user.ActiveOrg.Id = orgId + user.Username = "Execution File API" + } + + log.Printf("[INFO] Should UPLOAD file %s if user has access", fileId) + ctx := context.Background() + file, err := getFile(ctx, fileId) + if err != nil { + log.Printf("File %s not found: %s", fileId, err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + found := false + if file.OrgId == user.ActiveOrg.Id { + found = true + } else { + for _, item := range user.Orgs { + if item == file.OrgId { + found = true + break + } + } + } + + if !found { + log.Printf("User %s doesn't have access to %s", user.Username, fileId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + log.Printf("[INFO] STATUS: %s", file.Status) + if file.Status != "created" { + log.Printf("File status isn't created. Can't upload.") + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "This file already has data."}`)) + return + } + + request.ParseMultipartForm(32 << 20) + parsedFile, _, err := request.FormFile("shuffle_file") + if err != nil { + log.Printf("[ERROR] Couldn't upload file: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Failed uploading file"}`)) + return + } + defer parsedFile.Close() + + file.Status = "uploading" + err = setFile(ctx, *file) + if err != nil { + log.Printf("Failed setting file to uploading") + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false, "reason": "Failed setting file to uploading"}`)) + return + } + + // Can be used for validation files for change + var buf bytes.Buffer + io.Copy(&buf, parsedFile) + contents := buf.Bytes() + md5 := md5sum(contents) + buf.Reset() + + sha256Sum := sha256.Sum256(contents) + //parsedFile.Reset() + + f, err := os.OpenFile(file.DownloadPath, os.O_WRONLY|os.O_CREATE, os.ModePerm) + if err != nil { + // Rolling back file + file.Status = "created" + setFile(ctx, *file) + + log.Printf("[ERROR] Failed uploading and creating file: %s", err) + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false}`)) + return + } + + defer f.Close() + parsedFile.Seek(0, io.SeekStart) + io.Copy(f, parsedFile) + + // FIXME: Set this one to 200 anyway? Can't download file then tho.. + file.Status = "active" + file.Md5sum = md5 + file.Sha256sum = fmt.Sprintf("%x", sha256Sum) + log.Printf("[INFO] MD5 for file %s (%s) is %s and SHA256 is %s", file.Filename, file.Id, file.Md5sum, file.Sha256sum) + + err = setFile(ctx, *file) + if err != nil { + log.Printf("[ERROR] Failed setting file back to active") + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false, "reason": "Failed setting file to active"}`)) + return + } + + log.Printf("[INFO] Successfully uploaded file ID %s", file.Id) + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) +} + +func handleCreateFile(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + // 1. Check user directly + // 2. Check workflow execution authorization + user, err := handleApiAuthentication(resp, request) + if err != nil { + log.Printf("[INFO] INITIAL Api authentication failed in file creation: %s", err) + + orgId, err := fileAuthentication(request) + if err != nil { + log.Printf("[ERROR] Bad file authentication in create file: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + user.ActiveOrg.Id = orgId + user.Username = "Execution File API" + } + + body, err := ioutil.ReadAll(request.Body) + if err != nil { + log.Println("Failed reading body") + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to read data"}`))) + return + } + + type FileStructure struct { + Filename string `json:"filename"` + OrgId string `json:"org_id"` + WorkflowId string `json:"workflow_id"` + } + + var curfile FileStructure + err = json.Unmarshal(body, &curfile) + if err != nil { + log.Printf("[ERROR] Failed unmarshaling: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to unmarshal data"}`))) + return + } + + // Loads of validation below + if len(curfile.Filename) == 0 || len(curfile.OrgId) == 0 || len(curfile.WorkflowId) == 0 { + log.Printf("[ERROR] Missing field during upload.") + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Missing field. Required: filename, org_id, workflow_id"}`))) + return + } + + ctx := context.Background() + if user.ActiveOrg.Id != curfile.OrgId { + log.Printf("[ERROR] User can't access org %s", curfile.OrgId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Error with organization"}`)) + return + } + + // Try to get the org and workflow in case they don't exist + workflow, err := getWorkflow(ctx, curfile.WorkflowId) + if err != nil { + log.Printf("[ERROR] Workflow %s doesn't exist.", curfile.WorkflowId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`)) + return + } + + _, err = getOrg(ctx, curfile.OrgId) + if err != nil { + log.Printf("[ERROR] Org %s doesn't exist.", curfile.OrgId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`)) + return + } + + if workflow.ExecutingOrg.Id != curfile.OrgId { + found := false + for _, curorg := range workflow.Org { + if curorg.Id == curfile.OrgId { + found = true + break + } + } + + if !found { + log.Printf("[ERROR] Org %s doesn't have access to %s.", curfile.OrgId, curfile.WorkflowId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`)) + return + } + } + + if strings.Contains(curfile.Filename, "/") || strings.Contains(curfile.Filename, `"`) || strings.Contains(curfile.Filename, "..") || strings.Contains(curfile.Filename, "~") { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Invalid characters in filename"}`)) + return + } + + // 1. Create the file object. + if len(basepath) == 0 { + basepath = "shuffle-files" + } + folderPath := fmt.Sprintf("%s/%s/%s", basepath, curfile.OrgId, curfile.WorkflowId) + + // Try to make the full file location + err = os.MkdirAll(folderPath, os.ModePerm) + if err != nil { + log.Printf("[ERROR] Writing issue for file location creation: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Failed creating upload location"}`)) + return + } + + filename := curfile.Filename + fileId := uuid.NewV4().String() + downloadPath := fmt.Sprintf("%s/%s", folderPath, fileId) + + timeNow := time.Now().Unix() + newFile := File{ + Id: fileId, + CreatedAt: timeNow, + UpdatedAt: timeNow, + Description: "", + Status: "created", + Filename: filename, + OrgId: curfile.OrgId, + WorkflowId: curfile.WorkflowId, + DownloadPath: downloadPath, + } + + err = setFile(ctx, newFile) + if err != nil { + log.Printf("[ERROR] Failed setting file: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Failed setting file reference"}`)) + return + } else { + log.Printf("[INFO] Created file %s", newFile.DownloadPath) + } + + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s"}`, fileId))) +} + +func getFile(ctx context.Context, id string) (*File, error) { + key := datastore.NameKey("Files", id, nil) + curFile := &File{} + if err := dbclient.Get(ctx, key, curFile); err != nil { + return &File{}, err + } + + return curFile, nil +} + +func setFile(ctx context.Context, file File) error { + // clear session_token and API_token for user + k := datastore.NameKey("Files", file.Id, nil) + if _, err := dbclient.Put(ctx, k, &file); err != nil { + log.Println(err) + return err + } + + return nil +} diff --git a/backend/go-app/main.go b/backend/go-app/main.go index e574423f..64167166 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -2422,7 +2422,7 @@ func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { } if count == 0 { - log.Printf("No users - redirecting for management user") + log.Printf("[WARNING] No users - redirecting for management user") resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "stay"}`))) return @@ -2555,12 +2555,12 @@ func getApikey(ctx context.Context, apikey string) (User, error) { var users []User _, err := dbclient.GetAll(ctx, q, &users) if err != nil { - log.Printf("Error getting users apikey (getapikey): %s", err) + log.Printf("[ERROR] Error getting users apikey (getapikey): %s", err) return User{}, err } if len(users) == 0 { - log.Printf("No users found for apikey %s", apikey) + log.Printf("[WARNING] No users found for apikey %s", apikey) return User{}, err } @@ -2577,28 +2577,6 @@ func getSession(ctx context.Context, thissession string) (*session, error) { return curUser, nil } -// ListBooks returns a list of books, ordered by title. -func getFile(ctx context.Context, id string) (*File, error) { - key := datastore.NameKey("Files", id, nil) - curFile := &File{} - if err := dbclient.Get(ctx, key, curFile); err != nil { - return &File{}, err - } - - return curFile, nil -} - -func setFile(ctx context.Context, file File) error { - // clear session_token and API_token for user - k := datastore.NameKey("Files", file.Id, nil) - if _, err := dbclient.Put(ctx, k, &file); err != nil { - log.Println(err) - return err - } - - return nil -} - // ListBooks returns a list of books, ordered by title. func getOrg(ctx context.Context, id string) (*Org, error) { key := datastore.NameKey("Organizations", id, nil) @@ -6552,7 +6530,7 @@ func handleAppHotload(location string, forceUpdate bool) error { } //log.Printf("Reading app folder: %#v", dir) - err = iterateAppGithubFolders(fs, dir, "", "", forceUpdate) + _, _, err = iterateAppGithubFolders(fs, dir, "", "", forceUpdate) if err != nil { log.Printf("Err: %s", err) return err @@ -6835,7 +6813,7 @@ func remoteOrgJobHandler(org Org, interval int) error { respBody, err := ioutil.ReadAll(newresp.Body) if err != nil { - log.Printf("Failed body read in job sync: %s", err) + log.Printf("[ERROR] Failed body read in job sync: %s", err) return err } @@ -6843,7 +6821,7 @@ func remoteOrgJobHandler(org Org, interval int) error { err = remoteOrgJobController(org, respBody) if err != nil { - log.Printf("Failed job controller run: %s", err) + log.Printf("[ERROR] Failed job controller run for %s: %s", respBody, err) return err } return nil @@ -7141,37 +7119,39 @@ func runInit(ctx context.Context) { } } - fileq := datastore.NewQuery("Files").Limit(1) - count, err := dbclient.Count(ctx, fileq) - log.Printf("FILECOUNT: %d", count) - if err == nil && count < 10 { - basepath := "." - filename := "testfile.txt" - fileId := uuid.NewV4().String() - log.Printf("Creating new file reference %s because none exist!", fileId) - workflowId := "2cf1169d-b460-41de-8c36-28b2092866f8" - downloadPath := fmt.Sprintf("%s/%s/%s/%s", basepath, activeOrgs[0].Id, workflowId, fileId) + /* + fileq := datastore.NewQuery("Files").Limit(1) + count, err := dbclient.Count(ctx, fileq) + log.Printf("FILECOUNT: %d", count) + if err == nil && count < 10 { + basepath := "." + filename := "testfile.txt" + fileId := uuid.NewV4().String() + log.Printf("Creating new file reference %s because none exist!", fileId) + workflowId := "2cf1169d-b460-41de-8c36-28b2092866f8" + downloadPath := fmt.Sprintf("%s/%s/%s/%s", basepath, activeOrgs[0].Id, workflowId, fileId) - timeNow := time.Now().Unix() - newFile := File{ - Id: fileId, - CreatedAt: timeNow, - UpdatedAt: timeNow, - Description: "Created by system for testing", - Status: "active", - Filename: filename, - OrgId: activeOrgs[0].Id, - WorkflowId: workflowId, - DownloadPath: downloadPath, - } + timeNow := time.Now().Unix() + newFile := File{ + Id: fileId, + CreatedAt: timeNow, + UpdatedAt: timeNow, + Description: "Created by system for testing", + Status: "active", + Filename: filename, + OrgId: activeOrgs[0].Id, + WorkflowId: workflowId, + DownloadPath: downloadPath, + } - err = setFile(ctx, newFile) - if err != nil { - log.Printf("Failed setting file: %s", err) - } else { - log.Printf("Created file %s in init", newFile.DownloadPath) + err = setFile(ctx, newFile) + if err != nil { + log.Printf("Failed setting file: %s", err) + } else { + log.Printf("Created file %s in init", newFile.DownloadPath) + } } - } + */ var allworkflowapps []AppAuthenticationStorage q = datastore.NewQuery("workflowappauth") @@ -7934,172 +7914,6 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { resp.Write(respBody) } -type File struct { - Id string `json:"id" datastore:"id"` - Type string `json:"type" datastore:"type"` - CreatedAt int64 `json:"created_at" datastore:"created_at"` - UpdatedAt int64 `json:"updated_at" datastore:"updated_at"` - Description string `json:"description" datastore:"description"` - ExpiresAt string `json:"expires_at" datastore:"expires_at"` - Status string `json:"status" datastore:"status"` - Filename string `json:"filename" datastore:"filename"` - URL string `json:"url" datastore:"org"` - OrgId string `json:"org_id" datastore:"org_id"` - WorkflowId string `json:"workflow_id" datastore:"workflow_id"` - Workflows []string `json:"workflows" datastore:"workflows"` - DownloadPath string `json:"download_path" datastore:"download_path"` -} - -func handleGetFileContent(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - var fileId string - location := strings.Split(request.URL.String(), "/") - if location[1] == "api" { - if len(location) <= 4 { - log.Printf("Path too short: %d", len(location)) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - log.Printf("\n\nUser is trying to get file %s\n\n", fileId) - - // 1. Check user directly - // 2. Check workflow execution authorization - setOrgId := false - user, err := handleApiAuthentication(resp, request) - if err != nil { - // r.HandleFunc("/api/v1/files/{fileId}/content", handleGetFileContent).Methods("GET", "OPTIONS") - log.Printf("INITIAL Api authentication failed in file download: %s", err) - executionId, ok := request.URL.Query()["execution_id"] - if ok && len(executionId) > 0 { - ctx := context.Background() - workflowExecution, err := getWorkflowExecution(ctx, executionId[0]) - if err != nil { - log.Printf("Couldn't find execution ID %s", executionId[0]) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - apikey := request.Header.Get("Authorization") - if !strings.HasPrefix(apikey, "Bearer ") { - log.Printf("Apikey doesn't start with bearer (2)") - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - apikeyCheck := strings.Split(apikey, " ") - if len(apikeyCheck) != 2 { - log.Printf("Invalid format for apikey (2)") - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - // This is annoying af and is done because of maxlength lol - newApikey := apikeyCheck[1] - if newApikey != workflowExecution.Authorization { - log.Printf("Bad apikey for execution %s. %s vs %s", executionId[0], apikey, workflowExecution.Authorization) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - log.Printf("Authorization is correct for execution %s! %s vs %s", executionId, apikey, workflowExecution.Authorization) - setOrgId = true - } else { - - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - } - - // 1. Verify if the user has access to the file: org_id and workflow - - log.Printf("Should get file %s", fileId) - ctx := context.Background() - file, err := getFile(ctx, fileId) - if err != nil { - log.Printf("File %s not found: %s", fileId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - // This is a workaround for file grabs from an app - if setOrgId == true { - user.ActiveOrg.Id = file.OrgId - } - - found := false - if file.OrgId == user.ActiveOrg.Id { - found = true - } else { - for _, item := range user.Orgs { - if item == file.OrgId { - found = true - break - } - } - } - - if !found { - log.Printf("User %s doesn't have access to %s", user.Username, fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - // Fixme: More auth: org and workflow! - downloadPath := file.DownloadPath - log.Printf("Downloadpath: %s", downloadPath) - Openfile, err := os.Open(downloadPath) - defer Openfile.Close() //Close after function return - if err != nil { - //File not found, send 404 - http.Error(resp, "File not found.", 404) - return - } - - //File is found, create and send the correct headers - //Get the Content-Type of the file - //Create a buffer to store the header of the file in - FileHeader := make([]byte, 512) - //Copy the headers into the FileHeader buffer - Openfile.Read(FileHeader) - //Get content type of file - FileContentType := http.DetectContentType(FileHeader) - - //Get the file size - FileStat, _ := Openfile.Stat() //Get info from file - FileSize := strconv.FormatInt(FileStat.Size(), 10) //Get file size as a string - - //Send the headers - resp.Header().Set("Content-Disposition", "attachment; filename="+fileId) - resp.Header().Set("Content-Type", FileContentType) - resp.Header().Set("Content-Length", FileSize) - - //Send the file - //We read 512 bytes from the file already, so we reset the offset back to 0 - Openfile.Seek(0, 0) - io.Copy(resp, Openfile) //'Copy' the file to the client - return - - //log.Printf("Should download file %s", downloadPath) - //resp.WriteHeader(200) - //resp.Write([]byte("OK")) -} - func initHandlers() { var err error ctx := context.Background() @@ -8233,10 +8047,10 @@ func initHandlers() { // https://developer.box.com/reference/get-files-id-content/ // 1. Creating the "get file" option. Make it possible to run this in the frontend. r.HandleFunc("/api/v1/files/{fileId}/content", handleGetFileContent).Methods("GET", "OPTIONS") - //r.HandleFunc("/api/v1/files/create", handleGetFile).Methods("POST", "OPTIONS") - //r.HandleFunc("/api/v1/files/upload", handleGetFile).Methods("POST", "OPTIONS") - //r.HandleFunc("/api/v1/files/{fileId}", handleGetFile).Methods("GET", "OPTIONS") - //r.HandleFunc("/api/v1/files/{fileId}", handleGetFile).Methods("DELETE", "OPTIONS") + r.HandleFunc("/api/v1/files/create", handleCreateFile).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/files/{fileId}/upload", handleUploadFile).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/files/{fileId}", handleGetFileMeta).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/files/{fileId}", handleDeleteFile).Methods("DELETE", "OPTIONS") http.Handle("/", r) } diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index f06c5a06..7372121b 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -954,6 +954,30 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { return } + runWorkflowExecutionTransaction(ctx, 0, workflowExecution.ExecutionId, actionResult, resp) +} + +// Will make sure transactions are always ran for an execution. This is recursive if it fails. Allowed to fail up to 5 times +func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workflowExecutionId string, actionResult ActionResult, resp http.ResponseWriter) { + // Should start a tx for the execution here + tx, err := dbclient.NewTransaction(ctx) + if err != nil { + log.Printf("client.NewTransaction: %v", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed creating transaction"}`))) + return + } + + key := datastore.NameKey("workflowexecution", workflowExecutionId, nil) + workflowExecution := &WorkflowExecution{} + if err := tx.Get(key, workflowExecution); err != nil { + log.Printf("tx.Get bug: %v", err) + tx.Rollback() + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting the workflow key"}`))) + return + } + if actionResult.Status == "ABORTED" || actionResult.Status == "FAILURE" { log.Printf("Actionresult is %s. Should set workflowExecution and exit all running functions", actionResult.Status) @@ -1228,18 +1252,37 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { } } - err = setWorkflowExecution(ctx, *workflowExecution) - if err != nil { - //workflowExecution.Result = "Error setting workflow: result too large" - //workflowExecution.Status = "FINISHED" - //workflowExecution.CompletedAt = int64(time.Now().Unix()) + // Transactions: https://cloud.google.com/datastore/docs/concepts/transactions#datastore-datastore-transactional-update-go + // Prevents timing issues + //ExecutionId + if _, err := tx.Put(key, workflowExecution); err != nil { + tx.Rollback() + log.Printf("[ERROR] tx.Put bug: %v", err) - log.Printf("Error saving workflow execution actionresult setting: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting workflowexecution actionresult: %s"}`, err))) return } + if _, err = tx.Commit(); err != nil { + if attempts >= 5 { + log.Printf("[ERROR] QUITTING: tx.Commit %d: %v", attempts, err) + tx.Rollback() + workflowExecution.Status = "ABORTED" + setWorkflowExecution(ctx, *workflowExecution) + + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + log.Printf("[WARNING] tx.Commit %d: %v", attempts, err) + + attempts += 1 + runWorkflowExecutionTransaction(ctx, attempts, workflowExecutionId, actionResult, resp) + return + } + resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } @@ -1360,6 +1403,7 @@ func getWorkflows(resp http.ResponseWriter, request *http.Request) { q := datastore.NewQuery("workflow").Filter("owner =", user.Id) if user.Role == "admin" { q = datastore.NewQuery("workflow").Filter("org_id =", user.ActiveOrg.Id) + log.Printf("[INFO] Getting workflows (ADMIN) for organization %s", user.ActiveOrg.Id) } var workflows []Workflow @@ -1830,7 +1874,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { for _, action := range workflow.Actions { allNodes = append(allNodes, action.ID) - if len(action.Errors) > 0 { + if len(action.Errors) > 0 || !action.IsValid { action.IsValid = true action.Errors = []string{} } @@ -2192,9 +2236,11 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // Handles check for required params if !found && param.Required { log.Printf("Appaction %s with required param %s doesn't exist.", action.Name, param.Name) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name))) - return + action.Errors = append(action.Errors, "Parameter %s is required", param.Name) + //newActions = append(newActions, action) + //resp.WriteHeader(401) + //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name))) + //return } } @@ -2209,6 +2255,15 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { workflow.IsValid = true log.Printf("Tags: %#v", workflow.Tags) + // FIXME: Is this too drastic? May lead to issues in the future. + // Should maybe make a copy for the old org. + if workflow.OrgId != user.ActiveOrg.Id { + log.Printf("[WARNING] Editing workflow to be owned by %s", user.ActiveOrg.Id) + workflow.OrgId = user.ActiveOrg.Id + workflow.ExecutingOrg = user.ActiveOrg + workflow.Org = append(workflow.Org, user.ActiveOrg) + } + err = setWorkflow(ctx, workflow, fileId) if err != nil { log.Printf("Failed saving workflow to database: %s", err) @@ -2234,7 +2289,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { Errors: workflow.Errors, } - log.Printf("Saved new version of workflow %s (%s)", workflow.Name, fileId) + log.Printf("Saved new version of workflow %s (%s) for org %s", workflow.Name, fileId, workflow.OrgId) resp.WriteHeader(200) newBody, err := json.Marshal(returndata) if err != nil { @@ -2795,7 +2850,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf continue } - log.Printf("Should set %s to SKIPPED as it's NOT a childnode of the startnode.", action.ID) + log.Printf("[WARNING] Set %s to SKIPPED as it's NOT a childnode of the startnode.", action.ID) defaultResults = append(defaultResults, ActionResult{ Action: action, ExecutionId: workflowExecution.ExecutionId, @@ -2836,7 +2891,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf var allEnvs []Environment if len(workflowExecution.ExecutionOrg) > 0 { - log.Printf("Executing ORG: %s", workflowExecution.ExecutionOrg) + log.Printf("[INFO] Executing ORG: %s", workflowExecution.ExecutionOrg) allEnvironments, err := getEnvironments(ctx, workflowExecution.ExecutionOrg) if err != nil { @@ -5336,11 +5391,24 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra return err } +type buildLaterStruct struct { + Tags []string + Extra string + Id string +} + // Onlyname is used to -func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate bool) error { +func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate bool) ([]buildLaterStruct, []buildLaterStruct, error) { var err error allapps := []WorkflowApp{} + reservedNames := []string{ + "OWA", + "NLP", + } + + buildLaterFirst := []buildLaterStruct{} + buildLaterList := []buildLaterStruct{} // It's here to prevent getting them in every iteration ctx := context.Background() @@ -5360,11 +5428,20 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin } // Go routine? Hmm, this can be super quick I guess - err = iterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate) + buildFirst, buildLast, err := iterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate) if err != nil { log.Printf("Error reading folder: %s", err) continue } + + for _, item := range buildFirst { + buildLaterFirst = append(buildLaterFirst, item) + } + + for _, item := range buildLast { + buildLaterList = append(buildLaterList, item) + } + case mode.IsRegular(): // Check the file filename := file.Name() @@ -5562,22 +5639,69 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin //log.Printf("Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion) - /// Only upload if successful and no errors - err = buildImageMemory(fs, tags, extra) - if err != nil { - log.Printf("Failed image build memory: %s", err) - } else { - if len(tags) > 0 { - log.Printf("Successfully built image %s", tags[0]) - } else { - log.Printf("Successfully built Docker image") + // ID can be used to e.g. set a build status. + buildLater := buildLaterStruct{ + Tags: tags, + Extra: extra, + Id: workflowapp.ID, + } + + reservedFound := false + for _, appname := range reservedNames { + if strings.ToUpper(workflowapp.Name) == strings.ToUpper(appname) { + buildLaterList = append(buildLaterList, buildLater) + + reservedFound = true + break } } + + /// Only upload if successful and no errors + if !reservedFound { + buildLaterFirst = append(buildLaterFirst, buildLater) + } else { + log.Printf("\n\n[WARNING] Skipping build of %s to later\n\n", workflowapp.Name) + } } } } - return err + if len(buildLaterFirst) == 0 && len(buildLaterList) == 0 { + return buildLaterFirst, buildLaterList, err + } + + //log.Printf("BUILDLATERFIRST: %d, BUILDLATERLIST: %d", len(buildLaterFirst), len(buildLaterList)) + if len(extra) == 0 { + log.Printf("[INFO] Starting build of %d containers (FIRST)", len(buildLaterFirst)) + for _, item := range buildLaterFirst { + err = buildImageMemory(fs, item.Tags, item.Extra) + if err != nil { + log.Printf("Failed image build memory: %s", err) + } else { + if len(item.Tags) > 0 { + log.Printf("Successfully built image %s", item.Tags[0]) + } else { + log.Printf("Successfully built Docker image") + } + } + } + + log.Printf("Starting build of %d skipped docker images", len(buildLaterList)) + for _, item := range buildLaterList { + err = buildImageMemory(fs, item.Tags, item.Extra) + if err != nil { + log.Printf("Failed image build memory: %s", err) + } else { + if len(item.Tags) > 0 { + log.Printf("Successfully built image %s", item.Tags[0]) + } else { + log.Printf("Successfully built Docker image") + } + } + } + } + + return buildLaterFirst, buildLaterList, err } func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { @@ -5722,7 +5846,7 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) { } // Query for the specifci workflowId - q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(20) + q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(30) var workflowExecutions []WorkflowExecution _, err = dbclient.GetAll(ctx, q, &workflowExecutions) if err != nil { diff --git a/docker-compose.yml b/docker-compose.yml index 05d3e546..5ed77582 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: frontend: #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.0 + image: ghcr.io/frikky/shuffle-frontend:0.8.3 container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -17,7 +17,7 @@ services: - backend backend: #build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.0 + image: ghcr.io/frikky/shuffle-backend:0.8.3 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: @@ -28,9 +28,11 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps + - ${SHUFFLE_FILE_LOCATION}:/shuffle-files environment: - DATASTORE_EMULATOR_HOST=shuffle-database:8000 - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps + - SHUFFLE_FILE_LOCATION=/shuffle-files - ORG_ID=${ORG_ID} - SHUFFLE_APP_DOWNLOAD_LOCATION=${SHUFFLE_APP_DOWNLOAD_LOCATION} - SHUFFLE_DEFAULT_USERNAME=${SHUFFLE_DEFAULT_USERNAME} diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index f7bf156d..e47c7c6c 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -191,7 +191,7 @@ const AngularWorkflow = (props) => { const cloudSyncEnabled = props.userdata !== undefined && props.userdata.active_org !== null && props.userdata.active_org !== undefined ? props.userdata.active_org.cloud_sync === true : false //const triggerEnvironments = cloudSyncEnabled ? ["cloud", "onprem"] : environments - const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" + const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" const triggerEnvironments = isCloud ? ["cloud"] : ["cloud", "onprem"] const unloadText = 'Are you sure you want to leave without saving (CTRL+S)?' @@ -899,6 +899,41 @@ const AngularWorkflow = (props) => { const onUnselect = (event) => { console.time("UNSELECT") + // Attempt at rewrite of name in other actions in following nodes. + // Should probably be done in the onBlur for the textfield instead + /* + if (event.target.data().type === "ACTION") { + const nodeaction = event.target.data() + const curaction = workflow.actions.find(a => a.id === nodeaction.id) + console.log("workflowaction: ", curaction) + console.log("nodeaction: ", nodeaction) + if (nodeaction.label !== curaction.label) { + console.log("BEACH!") + + var params = [] + const fixedName = "$"+curaction.label.toLowerCase().replace(" ", "_") + for (var actionkey in workflow.actions) { + if (workflow.actions[actionkey].id === curaction.id) { + continue + } + + for (var paramkey in workflow.actions[actionkey].parameters) { + const param = workflow.actions[actionkey].parameters[paramkey] + if (param.value === null || param.value === undefined || !param.value.includes("$")) { + continue + } + + const innername = param.value.toLowerCase().replace(" ", "_") + if (innername.includes(fixedName)) { + //workflow.actions[actionkey].parameters[paramkey].replace( + //console.log("FOUND!: ", innername) + } + } + } + } + } + */ + // FIXME - check if they have value before overriding like this for no reason. // Would save a lot of time (400~ ms -> 30ms) //console.log("ACTION: ", selectedAction) @@ -1009,8 +1044,29 @@ const AngularWorkflow = (props) => { } setSelectedActionName(curaction.name) - setSelectedAction(curaction) + + /* + var params = [] + const fixedName = "$"+curaction.label.toLowerCase().replace(" ", "_") + for (var actionkey in workflow.actions) { + if (workflow.actions[actionkey].id === curaction.id) { + continue + } + + for (var paramkey in workflow.actions[actionkey].parameters) { + const param = workflow.actions[actionkey].parameters[paramkey] + if (param.value === null || param.value === undefined || !param.value.includes("$")) { + continue + } + + const innername = param.value.toLowerCase().replace(" ", "_") + if (innername.includes(fixedName)) { + console.log("FOUND!: ", innername) + } + } + } + */ } else if (data.type === "TRIGGER") { //console.log("Should handle trigger "+data.triggertype) //console.log(data) @@ -1278,6 +1334,7 @@ const AngularWorkflow = (props) => { } else { setEnvironments({"name": "Onprem", "type": "onprem"}) } + return } @@ -2419,14 +2476,43 @@ const AngularWorkflow = (props) => { // ACTION select // const selectedNameChange = (event) => { + console.log("OLDNAME: ", selectedActionName) event.target.value = event.target.value.replace("(", "") event.target.value = event.target.value.replace(")", "") event.target.value = event.target.value.replace("$", "") event.target.value = event.target.value.replace("#", "") event.target.value = event.target.value.replace(".", "") event.target.value = event.target.value.replace(",", "") + event.target.value = event.target.value.replace(" ", "_") selectedAction.label = event.target.value setSelectedAction(selectedAction) + + /* + if (nodeaction.label !== curaction.label) { + console.log("BEACH!") + + var params = [] + const fixedName = "$"+curaction.label.toLowerCase().replace(" ", "_") + for (var actionkey in workflow.actions) { + if (workflow.actions[actionkey].id === curaction.id) { + continue + } + + for (var paramkey in workflow.actions[actionkey].parameters) { + const param = workflow.actions[actionkey].parameters[paramkey] + if (param.value === null || param.value === undefined || !param.value.includes("$")) { + continue + } + + const innername = param.value.toLowerCase().replace(" ", "_") + if (innername.includes(fixedName)) { + //workflow.actions[actionkey].parameters[paramkey].replace( + //console.log("FOUND!: ", innername) + } + } + } + } + */ } const selectedTriggerChange = (event) => { @@ -2836,15 +2922,56 @@ const AngularWorkflow = (props) => { }} /> - console.log(selectedActionParameters[count]) if (selectedActionParameters[count].schema !== undefined && selectedActionParameters[count].schema !== null && selectedActionParameters[count].schema.type === "file") { - const fileId = "6daabec1-892b-469c-b603-c902e47223a9" - datafield = `SHOW FILES FROM OTHER NODES? Filename: ${selectedActionParameters[count].value}` + datafield = + + + { + setMenuPosition({ + top: event.pageY, + left: event.pageX, + }) + setShowDropdownNumber(count) + setShowDropdown(true) + setShowAutocomplete(true) + }}/> + + + ) + }} + fullWidth + multiline={multiline} + rows="5" + color="primary" + defaultValue={data.value} + type={"text"} + placeholder={"The file ID to get"} + onChange={(event) => { + changeActionParameter(event, count) + }} + onBlur={(event) => { + }} + /> + //const fileId = "6daabec1-892b-469c-b603-c902e47223a9" + //datafield = `SHOW FILES FROM OTHER NODES? Filename: ${selectedActionParameters[count].value}` + /* if (selectedActionParameters[count].value != fileId) { changeActionParameter(fileId, count) setUpdate(Math.random()) } + */ } else if (selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0) { if (selectedActionParameters[count].value === "" && selectedActionParameters[count].required) { // Rofl, dirty workaround :) @@ -4551,7 +4678,6 @@ const AngularWorkflow = (props) => { placeholder={selectedTrigger.label} onChange={selectedTriggerChange} /> - {showEnvironment ?
Environment @@ -4599,7 +4725,6 @@ const AngularWorkflow = (props) => { })}
- : null}
@@ -5095,59 +5220,57 @@ const AngularWorkflow = (props) => { placeholder={selectedTrigger.label} onChange={selectedTriggerChange} /> - {showEnvironment ? -
- - Environment - - { - selectedTrigger.environment = e.target.value - setSelectedTrigger(selectedTrigger) - if (e.target.value === "cloud") { - console.log("Set cloud config") - workflow.triggers[selectedTriggerIndex].parameters[0].value = "*/2 * * * *" + } + }} + fullWidth + onChange={(e) => { + selectedTrigger.environment = e.target.value + setSelectedTrigger(selectedTrigger) + if (e.target.value === "cloud") { + console.log("Set cloud config") + workflow.triggers[selectedTriggerIndex].parameters[0].value = "*/2 * * * *" - //var tmpvalue = workflow.triggers[selectedTriggerIndex].parameters[0].value.split("/") - //const urlpath = tmpvalue.slice(3, tmpvalue.length) - //const newurl = "https://shuffler.io/"+urlpath.join("/") - //workflow.triggers[selectedTriggerIndex].parameters[0].value = newurl - } else { - console.log("Set cloud config") - //var tmpvalue = workflow.triggers[selectedTriggerIndex].parameters[0].value.split("/") - //const urlpath = tmpvalue.slice(3, tmpvalue.length) - //const newurl = window.location.origin+"/"+urlpath.join("/") - workflow.triggers[selectedTriggerIndex].parameters[0].value = "120" - } + //var tmpvalue = workflow.triggers[selectedTriggerIndex].parameters[0].value.split("/") + //const urlpath = tmpvalue.slice(3, tmpvalue.length) + //const newurl = "https://shuffler.io/"+urlpath.join("/") + //workflow.triggers[selectedTriggerIndex].parameters[0].value = newurl + } else { + console.log("Set cloud config") + //var tmpvalue = workflow.triggers[selectedTriggerIndex].parameters[0].value.split("/") + //const urlpath = tmpvalue.slice(3, tmpvalue.length) + //const newurl = window.location.origin+"/"+urlpath.join("/") + workflow.triggers[selectedTriggerIndex].parameters[0].value = "120" + } - setWorkflow(workflow) - setUpdate(Math.random()) - }} - style={{backgroundColor: inputColor, color: "white", height: "50px"}} - > - {triggerEnvironments.map(data => { - if (data.archived) { - return null - } - - return ( - - {data} - - ) - })} - -
- : null} + setWorkflow(workflow) + setUpdate(Math.random()) + }} + style={{backgroundColor: inputColor, color: "white", height: "50px"}} + > + {triggerEnvironments.map(data => { + if (data.archived) { + return null + } + + return ( + + {data} + + ) + })} + +
@@ -5228,7 +5351,7 @@ const AngularWorkflow = (props) => { return null } - const cytoscapeViewWidths = 700 + const cytoscapeViewWidths = 750 const bottomBarStyle = { position: "fixed", right: 20, @@ -5286,6 +5409,62 @@ const AngularWorkflow = (props) => { return null } + const FileMenu = () => { + const [newAnchor, setNewAnchor] = React.useState(null); + const [showShuffleMenu, setShowShuffleMenu] = React.useState(false) + + { /*const [showShuffleMenu, setShowShuffleMenu] = React.useState(true) */} + return ( +
+ { + setShowShuffleMenu(false) + }} + > +
+

This menu is used to control the workflow itself.

+ + Exit on Error
} + control={ + { + workflow.configuration.exit_on_error = !workflow.configuration.exit_on_error + setWorkflow(workflow) + setUpdate("exit_on_error_"+workflow.configuration.exit_on_error ? "true" : "false") + setShowShuffleMenu(false) + }} /> + } + /> + Start from top
} + control={ + { + workflow.configuration.start_from_top = !workflow.configuration.start_from_top + setWorkflow(workflow) + setUpdate("start_from_top_"+workflow.configuration.start_from_top ? "true" : "false") + setShowShuffleMenu(false) + }} /> + } + /> +
+ + + + +
+ ) + } + const WorkflowMenu = () => { const [newAnchor, setNewAnchor] = React.useState(null); const [showShuffleMenu, setShowShuffleMenu] = React.useState(false) @@ -5413,6 +5592,7 @@ const AngularWorkflow = (props) => { + {/* */}
@@ -5719,7 +5899,7 @@ const AngularWorkflow = (props) => { try { const tmp = String(JSON.parse(showResult)) if (!showResult.includes("{") && !showResult.includes("[")) { - console.log("IN HERE: ", tmp) + //console.log("IN HERE: ", tmp) jsonvalid = false } } catch (e) { diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 9bb8dda4..69ed1dc1 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -116,7 +116,7 @@ const Apps = (props) => { const [apps, setApps] = React.useState([]) const [filteredApps, setFilteredApps] = React.useState([]) const [validation, setValidation] = React.useState(false) - const [isLoading, setIsLoading] = React.useState(false) + const [isLoading, setIsLoading] = React.useState(true) const [appSearchLoading, setAppSearchLoading] = React.useState(false) const [selectedAction, setSelectedAction] = React.useState({}) const [searchBackend, setSearchBackend] = React.useState(false) @@ -207,6 +207,7 @@ const Apps = (props) => { credentials: "include", }) .then((response) => { + setIsLoading(false) if (response.status !== 200) { console.log("Status not 200 for apps :O!") } @@ -231,6 +232,7 @@ const Apps = (props) => { }) .catch(error => { alert.error(error.toString()) + setIsLoading(false) }); } @@ -742,7 +744,7 @@ const Apps = (props) => {
diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 52f913d5..56ef2f86 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -15,6 +15,7 @@ import FormControlLabel from '@material-ui/core/FormControlLabel'; import Chip from '@material-ui/core/Chip'; import Switch from '@material-ui/core/Switch'; import Typography from '@material-ui/core/Typography'; +import Zoom from '@material-ui/core/Zoom'; import CircularProgress from '@material-ui/core/CircularProgress'; import CachedIcon from '@material-ui/icons/Cached'; @@ -74,6 +75,7 @@ const Workflows = (props) => { const [update, setUpdate] = React.useState("test"); const [deleteModalOpen, setDeleteModalOpen] = React.useState(false); const [editingWorkflow, setEditingWorkflow] = React.useState({}) + const [executionLoading, setExecutionLoading] = React.useState(false) const { start, stop } = useInterval({ duration: 5000, startImmediate: false, @@ -245,6 +247,7 @@ const Workflows = (props) => { } const getWorkflowExecution = (id) => { + setExecutionLoading(true) fetch(globalUrl+"/api/v1/workflows/"+id+"/executions", { method: 'GET', headers: { @@ -254,6 +257,7 @@ const Workflows = (props) => { credentials: "include", }) .then((response) => { + setExecutionLoading(false) if (response.status !== 200) { console.log("Status not 200 for WORKFLOW EXECUTION :O!") } @@ -275,6 +279,7 @@ const Workflows = (props) => { } }) .catch(error => { + setExecutionLoading(false) alert.error(error.toString()) }); } @@ -360,7 +365,8 @@ const Workflows = (props) => { data.triggers[key].status = "stopped" } } - + data["org"] = [] + data["org_id"] = "" data.execution_org = {"id": ""} console.log(data) @@ -858,10 +864,17 @@ const Workflows = (props) => { ) } + return ( -

- There are no executiondetails yet. Click "execute" to run your first one. -

+ executionLoading ? +
+ +
+ : +

+ There are no executiondetails yet. Click "execute" to run your first one. +

+ ) } @@ -880,9 +893,14 @@ const Workflows = (props) => { ) } return ( -

- There are no executions for this workflow yet -

+ executionLoading ? +
+ +
+ : +

+ There are no executions for this workflow yet +

) } @@ -1012,8 +1030,19 @@ const Workflows = (props) => { }, }} > + +
+ {editingWorkflow.id !== undefined ? "Editing" : "New"} workflow +
+ + + +
+
+ -
{editingWorkflow.id !== undefined ? "Editing" : "New"} workflow
 setNewWorkflowName(event.target.value)} @@ -1111,107 +1140,126 @@ const Workflows = (props) => { workflowViewStyle.display = "none" } - const workflowView = workflows.length > 0 ? -
-
-
-
-

Workflows

-
-
- - - - {/* - - - - */} - - - - - - - upload = ref} onChange={importFiles} /> -
-
- - -
- {workflows.map((data, index) => { - return ( - - ) - })} -
-
-
-
-
-

Executions: {selectedWorkflow.name}

-
-
- + + : null} + + + + upload = ref} onChange={importFiles} /> + {workflows.length > 0 ? + + -
-
- -
- -
-
-
-
-
-

Execution Timeline

-
-
- Collapse results
} - control={ {setCollapseJson(!collapseJson)}} />} - /> -
-
- -
- -
-
- - : -
- -
-

Welcome to Shuffle

-
-
-

- Shuffle is a flexible, easy to use, automation platform allowing users to integrate their services and devices freely. It's made to significantly reduce the amount of manual labor, and is focused on security applications. Click here to learn more. -

-
-
- If you want to jump straight into it, click here to create your first workflow: -
-
- -
- -
+ + : null} + + + + + const WorkflowView = () => { + if (workflows.length === 0) { + return ( +
+ +
+

Welcome to Shuffle

+
+
+

+ Shuffle is a flexible, easy to use, automation platform allowing users to integrate their services and devices freely. It's made to significantly reduce the amount of manual labor, and is focused on security applications. Click here to learn more. +

+
+
+ If you want to jump straight into it, click here to create your first workflow: +
+
+ + + + ..OR + + {workflowButtons} + +
+ +
+ ) + } + + return ( +
+
+
+
+

Workflows

+
+
+ {workflowButtons} +
+
+ + +
+ {workflows.map((data, index) => { + return ( + + ) + })} +
+
+
+
+
+

Executions: {selectedWorkflow.name}

+
+
+ +
+
+ +
+ +
+
+
+
+
+

Execution Timeline

+
+
+ Collapse results
} + control={ {setCollapseJson(!collapseJson)}} />} + /> +
+
+ +
+ +
+
+ + ) + } const importWorkflowsFromUrl = (url) => { console.log("IMPORT WORKFLOWS FROM ", downloadUrl) @@ -1384,13 +1432,17 @@ const Workflows = (props) => { const loadedCheck = isLoaded && isLoggedIn && workflowDone ?
- {workflowView} + {modalView} {deleteModal} {workflowDownloadModalOpen}
: -
+
+ + + Loading Workflows +